Merge branch 'main' into validate-logstash-pipeline-id

Author: mashhurs

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java

@@ -52,6 +52,23 @@ public void apply(Project project) {
             // Spotless resolves required dependencies from project repositories, so we need maven central
             project.getRepositories().mavenCentral();
 
+            // we cannot update to a newer spotless plugin version yet but we want to use the latest eclipse formatter to be compatible
+            // with latest java versions
+            project.getConfigurations().matching(it -> it.getName().startsWith("spotless")).configureEach(conf -> {
+                project.getDependencies().constraints(constraints -> {
+                    constraints.add(conf.getName(), "org.eclipse.jdt:org.eclipse.jdt.core:3.42.0", dependencyConstraint -> {
+                        dependencyConstraint.because(
+                            "We want to use a recent version of the Eclipse formatter libraries to support latest Java"
+                        );
+                    });
+                    constraints.add(conf.getName(), "org.eclipse.jdt:ecj:3.42.0", dependencyConstraint -> {
+                        dependencyConstraint.because(
+                            "We want to use a recent version of the Eclipse formatter libraries to support latest Java"
+                        );
+                    });
+                });
+            });
+
             project.getExtensions().getByType(SpotlessExtension.class).java(java -> {
                 File elasticsearchWorkspace = Util.locateElasticsearchWorkspace(project.getGradle());
                 String importOrderPath = "build-conventions/elastic.importorder";
@@ -74,7 +91,7 @@ public void apply(Project project) {
                 // When running build benchmarks we alter the source in some scenarios.
                 // The gradle-profiler unfortunately does not generate compliant formatted
                 // sources so we ignore that altered file when running build benchmarks
-                if(Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
+                if (Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
                     java.targetExclude("src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java");
                 }
             });

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/ResolveTransportVersionConflictFuncTest.groovy

@@ -87,4 +87,13 @@ class ResolveTransportVersionConflictFuncTest extends AbstractTransportVersionFu
         assertUpperBound("9.1", "new_tv,8012002")
         assertUpperBound("8.19", "new_tv,7123002")
     }
+
+    def "no new transport version is idempotent"() {
+        when:
+        def result = runResolveAndValidateTask().build()
+
+        then:
+        assertResolveAndValidateSuccess(result)
+        assertUpperBound("9.2", "existing_92,8123000")
+    }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -428,7 +428,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
 
     def "upper bounds files must exist for backport branches"() {
         when:
-        def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
+        def result = runGenerateTask("--name", "new_tv", "--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
 
         then:
         assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [8.19, 9.0, 9.1, 9.2]")

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -294,4 +294,21 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "only current upper bound validated on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.0'
+            }
+        """
+        referableAndReferencedTransportVersion("some_tv", "8124000,8012004")
+        transportVersionUpperBound("9.1", "some_tv", "8012004")
+
+        when:
+        def result = gradleRunner("validateTransportVersionResources").build()
+
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -113,6 +113,9 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
             String versionProperty = System.getProperty("tests.bwc.main.version");
             // We use this phony version as a placeholder for the real version
             if (distribution.getVersion().equals("0.0.0")) {
+                if (versionProperty == null) {
+                    throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
+                }
                 BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
                     Version.fromString(versionProperty),
                     "main",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -98,14 +98,14 @@ public void run() throws IOException {
         List<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
         String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
 
-        List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
-        Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
-
         getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
         if (targetDefinitionName.isEmpty()) {
             // TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
             // resetAllUpperBounds(resources);
         } else {
+            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
+            Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
+
             List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
             // (Re)write the definition file.
             resources.writeDefinition(new TransportVersionDefinition(targetDefinitionName, ids, true));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -83,6 +83,7 @@ public void validateTransportVersions() throws IOException {
         Map<String, TransportVersionDefinition> allDefinitions = collectAllDefinitions(referableDefinitions, unreferableDefinitions);
         Map<Integer, List<IdAndDefinition>> idsByBase = collectIdsByBase(allDefinitions.values());
         Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
+        TransportVersionUpperBound currentUpperBound = upperBounds.get(getCurrentUpperBoundName().get());
         boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
 
         for (var definition : referableDefinitions.values()) {
@@ -94,14 +95,21 @@ public void validateTransportVersions() throws IOException {
         }
 
         for (var entry : idsByBase.entrySet()) {
-            validateBase(entry.getKey(), entry.getValue());
+            int baseId = entry.getKey();
+            // on main we validate all bases, but on release branches we only validate up to the current upper bound
+            if (onReleaseBranch == false || baseId <= currentUpperBound.definitionId().base()) {
+                validateBase(baseId, entry.getValue());
+            }
         }
 
-        for (var upperBound : upperBounds.values()) {
-            validateUpperBound(upperBound, allDefinitions, idsByBase);
-        }
+        if (onReleaseBranch) {
+            // on release branches we only check the current upper bound, others may be inaccurate
+            validateUpperBound(currentUpperBound, allDefinitions, idsByBase);
+        } else {
+            for (var upperBound : upperBounds.values()) {
+                validateUpperBound(upperBound, allDefinitions, idsByBase);
+            }
 
-        if (onReleaseBranch == false) {
             validatePrimaryIds(resources, upperBounds, allDefinitions);
         }
     }

build-tools/src/main/java/org/elasticsearch/gradle/util/OsUtils.java

@@ -38,7 +38,7 @@ private OsUtils() {}
       * This method returns true if the given version of the JDK is known to be incompatible
       */
     public static boolean jdkIsIncompatibleWithOS(Version version) {
-        return version.onOrBefore("8.10.4") && isUbuntu2404OrLater();
+        return version.after("0.0.0") && version.onOrBefore("8.10.4") && isUbuntu2404OrLater();
     }
 
     private static boolean isUbuntu2404OrLater() {

build.gradle

@@ -338,18 +338,13 @@ allprojects {
   tasks.register('resolveAllDependencies', ResolveAllDependencies) {
     def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"]
     configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false }
+
     if (project.path == ':') {
       resolveJavaToolChain = true
-
-      // ensure we have best possible caching of bwc builds
-      dependsOn ":distribution:bwc:major1:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:major2:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:major3:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:major4:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:minor1:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:minor2:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:minor3:buildBwcLinuxTar"
-      dependsOn ":distribution:bwc:minor4:buildBwcLinuxTar"
+    }
+    // ensure we have best possible caching of bwc builds
+    if(project.path.startsWith(":distribution:bwc:")) {
+      dependsOn project.tasks.matching { it.name == 'buildBwcLinuxTar' }
     }
     if (project.path.contains("fixture")) {
       dependsOn tasks.withType(ComposePull)

docs/changelog/134585.yaml

@@ -0,0 +1,5 @@
+pr: 134585
+summary: Add classes to represent raw docs sampling configs
+area: Ingest Node
+type: feature
+issues: []