Merge remote-tracking branch 'upstream/main' into ES-12677

# Conflicts:
#	server/src/main/resources/transport/upper_bounds/9.3.csv

Author: alexey-ivanov-es

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -46,6 +46,7 @@ steps:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
+        BENCHMARK_TYPE: baseline
   - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
@@ -56,6 +57,7 @@ steps:
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+        BENCHMARK_TYPE: contender
   - wait: ~
   - label: Update PR comment and Buildkite annotation
     command: |

.gitignore

@@ -46,6 +46,7 @@ build/
 **/.local*
 .vagrant/
 /logs/
+**/target/
 
 # osx stuff
 .DS_Store

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -119,7 +119,7 @@ public void setup() {
     }
 
     private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
-        var parsed = parser.createStatement(query, new QueryParams(), telemetry, config);
+        var parsed = parser.createStatement(query, new QueryParams(), telemetry);
         var analyzed = analyzer.analyze(parsed);
         var optimized = optimizer.optimize(analyzed);
         return optimized;

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy

@@ -151,7 +151,11 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
             tasks.named('checkstyleMain').configure { enabled = false }
             tasks.named('loggerUsageCheck').configure { enabled = false }
             // tested elsewhere
-            tasks.named('thirdPartyAudit').configure { enabled = false }
+            tasks.named('thirdPartyAudit').configure {
+                getRuntimeJavaVersion().set(JavaVersion.VERSION_21)
+                getTargetCompatibility().set(JavaVersion.VERSION_21)
+                enabled = false
+            }
             """
         when:
         def result = gradleRunner("check").build()
@@ -167,6 +171,25 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
         result.task(":loggerUsageCheck").outcome == TaskOutcome.SKIPPED
     }
 
+    def "can generate dependency infos file"() {
+        given:
+        repository.generateJar("junit", "junit", "4.12", 'org.acme.JunitMock')
+        repository.configureBuild(buildFile)
+        file("licenses/junit-4.12.jar.sha1").text = "2973d150c0dc1fefe998f834810d68f278ea58ec"
+        file("licenses/junit-LICENSE.txt").text = EXAMPLE_LICENSE
+        file("licenses/junit-NOTICE.txt").text = "mock notice"
+        buildFile << """
+        dependencies {
+            api "junit:junit:4.12"
+        }
+        """
+        when:
+        def result = gradleRunner("dependenciesInfo").build()
+        then:
+        result.task(":dependenciesInfo").outcome == TaskOutcome.SUCCESS
+        file("build/reports/dependencies/dependencies.csv").text == "junit:junit,4.12,https://repo1.maven.org/maven2/junit/junit/4.12,BSD-3-Clause,\n"
+    }
+
     def assertValidJar(File jar) {
         try (ZipFile zipFile = new ZipFile(jar)) {
             ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt")

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -311,4 +311,43 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "modification checks use PR base branch in CI"() {
+        given:
+        // create 9.1 branch
+        execute("git checkout -b 9.1")
+        // setup 9.1 upper bound as if 9.2 was just branched, so no patches yet
+        unreferableTransportVersion("initial_9.1.0", "8013000")
+        transportVersionUpperBound("9.1", "initial_9.1.0", "8013000")
+        execute("git add .")
+        execute("git commit -m initial")
+
+        // advance main branch
+        execute("git checkout main")
+        unreferableTransportVersion("initial_9.1.0", "8013000")
+        referableAndReferencedTransportVersion("new_tv", "8124000,8013001")
+        transportVersionUpperBound("9.2", "new_tv", "8124000")
+        transportVersionUpperBound("9.1", "new_tv", "8013001")
+        execute("git add .")
+        execute("git commit -m update")
+
+        // create a 9.1 backport
+        execute("git checkout 9.1")
+        execute("git checkout -b test_9.1")
+        referableAndReferencedTransportVersion("new_tv", "8124000,8013001")
+        transportVersionUpperBound("9.2", "new_tv", "8124000")
+        transportVersionUpperBound("9.1", "new_tv", "8013001")
+        file("myserver/build.gradle") << """
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.1'
+            }
+        """
+
+        when:
+        def result = gradleRunner("validateTransportVersionResources")
+            .withEnvironment(Map.of("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "9.1")).build()
+
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -7,15 +7,13 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import java.lang.management.ManagementFactory;
-import java.time.LocalDateTime;
 
-import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
-import static org.elasticsearch.gradle.internal.util.CiUtils.safeName
-
 import java.lang.management.ManagementFactory
 import java.time.LocalDateTime
+import org.elasticsearch.gradle.Architecture
+
+import static org.elasticsearch.gradle.internal.util.CiUtils.safeName
 
 // Resolving this early to avoid issues with the build scan plugin in combination with the configuration cache usage
 def taskNames = gradle.startParameter.taskNames.join(' ')
@@ -32,10 +30,10 @@ develocity {
     // Automatically publish scans from Elasticsearch CI
     if (onCI) {
       publishing.onlyIf { true }
-      if(server.isPresent() == false) {
+      if (server.isPresent() == false) {
         server = 'https://gradle-enterprise.elastic.co'
       }
-    } else if( server.isPresent() == false) {
+    } else if (server.isPresent() == false) {
       publishing.onlyIf { false }
     }
 
@@ -99,6 +97,15 @@ develocity {
           tag 'pull-request'
           link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
           link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
+        } else if (System.getenv('ELASTICSEARCH_PR_NUMBER')) {
+          // For tracking es-pr-check failures in the serverless repo caused by elasticsearch PRs
+          def esPr = System.getenv('ELASTICSEARCH_PR_NUMBER')
+          def esCommit = System.getenv('ELASTICSEARCH_SUBMODULE_COMMIT')
+          value 'Git Commit ID', esCommit
+          tag "pr/${esPr}"
+          tag 'pull-request'
+          link 'Source', "https://github.com/elastic/elasticsearch/pull/${esPr}/commits/${esCommit}"
+          link 'Pull Request', "https://github.com/elastic/elasticsearch/pull/${esPr}"
         } else {
           value 'Git Commit ID', gitRevision.get()
           link 'Source', "https://github.com/${repository}/tree/${gitRevision.get()}"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/AbstractDependenciesTask.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Optional;
+
+import java.util.Map;
+
+public abstract class AbstractDependenciesTask extends DefaultTask {
+
+    @Input
+    @Optional
+    public abstract MapProperty<String, String> getMappings();
+
+    /**
+     * Add a mapping from a regex pattern for the jar name, to a prefix to find
+     * the LICENSE and NOTICE file for that jar.
+     */
+    public void mapping(Map<String, String> props) {
+        String from = props.get("from");
+        if (from == null) {
+            throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
+        }
+        String to = props.get("to");
+        if (to == null) {
+            throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
+        }
+        if (props.size() > 2) {
+            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
+        }
+        getMappings().put(from, to);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java

@@ -17,6 +17,8 @@
 import org.gradle.api.attributes.Usage;
 import org.gradle.api.plugins.JavaPlugin;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createNonTransitiveArtifactsView;
+
 public class DependenciesInfoPlugin implements Plugin<Project> {
 
     public static String USAGE_ATTRIBUTE = "DependenciesInfo";
@@ -25,16 +27,15 @@ public class DependenciesInfoPlugin implements Plugin<Project> {
     public void apply(final Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
-
         depsInfo.configure(t -> {
             var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
+            t.getRuntimeArtifacts()
+                .set(project.getProviders().provider(() -> createNonTransitiveArtifactsView(runtimeConfiguration).getArtifacts()));
             t.getClasspath().from(runtimeConfiguration);
             var compileOnlyConfiguration = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
             t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
             t.getClasspath().from(compileOnlyConfiguration);
-
         });
         Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
         dependenciesInfoFilesConfiguration.setCanBeResolved(false);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java

@@ -17,19 +17,21 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.ProjectLayout;
-import org.gradle.api.internal.ConventionTask;
 import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.internal.component.external.model.ModuleComponentArtifactIdentifier;
 
 import java.io.File;
 import java.io.IOException;
@@ -55,7 +57,8 @@
  *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
  * </ul>
  */
-public abstract class DependenciesInfoTask extends ConventionTask {
+@CacheableTask
+public abstract class DependenciesInfoTask extends AbstractDependenciesTask {
 
     @Inject
     public abstract ProviderFactory getProviderFactory();
@@ -86,16 +89,17 @@ public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
      * artifact transforms that might be applied and fail due to missing task dependency to jar
      * generating tasks.
      * */
-    @InputFiles
+    @Classpath
     abstract ConfigurableFileCollection getClasspath();
 
     private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
         return getProviderFactory().provider(
             () -> artifacts.getArtifacts()
                 .stream()
                 .map(r -> r.getId())
-                .filter(id -> id instanceof ModuleComponentIdentifier)
-                .map(id -> (ModuleComponentIdentifier) id)
+                .filter(mcaId -> mcaId instanceof ModuleComponentArtifactIdentifier)
+                .map(mcaId -> (ModuleComponentArtifactIdentifier) mcaId)
+                .map(it -> it.getComponentIdentifier())
                 .collect(Collectors.toSet())
         );
     }
@@ -111,6 +115,7 @@ private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers
      * Directory to read license files
      */
     @Optional
+    @PathSensitive(PathSensitivity.RELATIVE)
     @InputDirectory
     public File getLicensesDir() {
         File asFile = licensesDir.get().getAsFile();
@@ -143,7 +148,6 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
 
     @TaskAction
     public void generateDependenciesInfo() throws IOException {
-
         final Set<String> compileOnlyIds = getCompileOnlyModules().map(
             set -> set.stream()
                 .map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
@@ -166,18 +170,13 @@ public void generateDependenciesInfo() throws IOException {
             final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
             final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
             getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
-
             final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
             output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
         }
 
         Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
     }
 
-    @Input
-    @Optional
-    public abstract MapProperty<String, String> getMappings();
-
     /**
      * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
      * based on dependency coordinates.