WIP handling multiple releases in the docs, and update release notes for 9.0.1

brianseeders · brianseeders · commit 80d16ecfb322 · 2025-05-02T15:35:57.000-04:00
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BundleChangelogsTask.java
@@ -14,6 +14,7 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
 
+import org.elasticsearch.gradle.Version;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.Directory;
@@ -28,6 +29,7 @@
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.options.Option;
 import org.gradle.process.ExecOperations;
 
 import java.io.File;
@@ -38,6 +40,7 @@
 import java.util.List;
 import java.util.Properties;
 
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 
 import static java.util.stream.Collectors.toList;
@@ -52,6 +55,29 @@ public class BundleChangelogsTask extends DefaultTask {
 
     private final GitWrapper gitWrapper;
 
+    @Nullable
+    private String branch;
+    private boolean updateExisting;
+    private boolean finalize;
+
+    @Option(
+        option = "update-existing",
+        description = "Only update entries that are already in the bundle. Useful for updating the bundle after a BC has been cut."
+    )
+    public void setUpdateExisting(boolean updateExisting) {
+        this.updateExisting = updateExisting;
+    }
+
+    @Option(option = "branch", description = "Branch (or other ref) to use for generating the changelog bundle.")
+    public void setBranch(String branch) {
+        this.branch = branch;
+    }
+
+    @Option(option = "finalize", description = "Specify that the bundle is finalized, i.e. that the version has been released.")
+    public void setFinalize(boolean finalize) {
+        this.finalize = finalize;
+    }
+
     private static final ObjectMapper yamlMapper = new ObjectMapper(
         new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)
             .disable(YAMLGenerator.Feature.SPLIT_LINES)
@@ -72,13 +98,16 @@ public BundleChangelogsTask(ObjectFactory objectFactory, ExecOperations execOper
 
     @TaskAction
     public void executeTask() throws IOException {
+        if (branch == null) {
+            throw new IllegalArgumentException("'branch' not specified.");
+        }
+
         final String upstreamRemote = gitWrapper.getUpstream();
 
-        String ref = "9.0-fix-release-notes";
         try {
-            checkoutChangelogs(gitWrapper, upstreamRemote, ref);
+            checkoutChangelogs(gitWrapper, upstreamRemote, branch);
             Properties props = new Properties();
-            props.load(new StringReader(gitWrapper.runCommand("git", "show", ref + ":build-tools-internal/version.properties")));
+            props.load(new StringReader(gitWrapper.runCommand("git", "show", branch + ":build-tools-internal/version.properties")));
             String version = props.getProperty("elasticsearch");
 
             LOGGER.info("Finding changelog files...");
@@ -90,7 +119,27 @@ public void executeTask() throws IOException {
                 .sorted(Comparator.comparing(ChangelogEntry::getPr))
                 .collect(toList());
 
-            ChangelogBundle bundle = new ChangelogBundle(version, Instant.now().toString(), entries);
+            ChangelogBundle existingBundle = null;
+
+            if (updateExisting) {
+                var existingBundleFile = new File("docs/release-notes/changelog-bundles/" + version + ".yml");
+                if (existingBundleFile.exists()) {
+                    var bundle = ChangelogBundle.parse(existingBundleFile);
+                    existingBundle = bundle;
+                    entries = entries.stream()
+                        .filter(e -> bundle.changelogs().stream().anyMatch(c -> c.getPr().equals(e.getPr())))
+                        .toList();
+                }
+            }
+
+            var isReleased = false;
+            if (finalize) {
+                isReleased = true;
+            } else if (existingBundle != null) {
+                isReleased = existingBundle.released();
+            }
+
+            ChangelogBundle bundle = new ChangelogBundle(version, isReleased, Instant.now().toString(), entries);
 
             yamlMapper.writeValue(new File("docs/release-notes/changelog-bundles/" + version + ".yml"), bundle);
         } finally {
@@ -102,7 +151,13 @@ private static void checkoutChangelogs(GitWrapper gitWrapper, String upstream, S
         gitWrapper.updateRemote(upstream);
         // TODO check for changes first
         gitWrapper.runCommand("rm", "-rf", "docs/changelog");
-        gitWrapper.runCommand("git", "checkout", ref, "--", "docs/changelog");
+        var refSpec = upstream + "/" + ref;
+        if (ref.contains("upstream/")) {
+            refSpec = ref.replace("upstream/", upstream + "/");
+        } else if (ref.matches("^[0-9a-f]+$")) {
+            refSpec = ref;
+        }
+        gitWrapper.runCommand("git", "checkout", refSpec, "--", "docs/changelog");
     }
 
     @InputDirectory
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogBundle.java
@@ -21,13 +21,17 @@
 import java.io.UncheckedIOException;
 import java.util.List;
 
-public record ChangelogBundle(String version, String generated, List<ChangelogEntry> changelogs) {
+public record ChangelogBundle(String version, boolean released, String generated, List<ChangelogEntry> changelogs) {
 
     private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class);
     private static final ObjectMapper yamlMapper = new ObjectMapper(
         new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES).disable(YAMLGenerator.Feature.SPLIT_LINES)
     );
 
+    public ChangelogBundle(String version, String generated, List<ChangelogEntry> changelogs) {
+        this(version, false, generated, changelogs);
+    }
+
     public static ChangelogBundle parse(File file) {
         try {
             return yamlMapper.readValue(file, ChangelogBundle.class);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java
@@ -97,42 +97,23 @@ public void executeTask() throws IOException {
             .getFiles()
             .stream()
             .map(ChangelogBundle::parse)
-            .sorted(Comparator.comparing(ChangelogBundle::generated).reversed())
+            .sorted(Comparator.comparing(ChangelogBundle::released).reversed().thenComparing(ChangelogBundle::generated).reversed())
             .toList();
 
         // Ensure that each changelog/PR only shows up once, in its earliest release
-        // TODO: This should only be for unreleased/non-final bundles
         var uniquePrs = new HashSet<Integer>();
         for (int i = bundles.size() - 1; i >= 0; i--) {
             var bundle = bundles.get(i);
-            bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList());
+            if (!bundle.released()) {
+                bundle.changelogs().removeAll(bundle.changelogs().stream().filter(c -> uniquePrs.contains(c.getPr())).toList());
+            }
             uniquePrs.addAll(bundle.changelogs().stream().map(ChangelogEntry::getPr).toList());
         }
 
         LOGGER.info("Generating release notes...");
         ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), bundles);
         ReleaseNotesGenerator.update(this.breakingChangesTemplate.get().getAsFile(), this.breakingChangesFile.get().getAsFile(), bundles);
         ReleaseNotesGenerator.update(this.deprecationsTemplate.get().getAsFile(), this.deprecationsFile.get().getAsFile(), bundles);
-
-        // Only update breaking changes and deprecations for new minors
-        // if (qualifiedVersion.revision() == 0) {
-        // LOGGER.info("Generating breaking changes / deprecations notes...");
-        // ReleaseNotesGenerator.update(
-        // this.breakingChangesTemplate.get().getAsFile(),
-        // this.breakingChangesFile.get().getAsFile(),
-        // qualifiedVersion,
-        // changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()),
-        // bundles
-        // );
-        //
-        // ReleaseNotesGenerator.update(
-        // this.deprecationsTemplate.get().getAsFile(),
-        // this.deprecationsFile.get().getAsFile(),
-        // qualifiedVersion,
-        // changelogsByVersion.getOrDefault(qualifiedVersion, Set.of()),
-        // bundles
-        // );
-        // }
     }
 
     /**
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java
@@ -149,7 +149,7 @@ static Stream<QualifiedVersion> findPreviousVersion(GitWrapper gitWrapper, Quali
         final String currentMajorPattern = "v" + version.major() + ".*";
         final String previousMajorPattern = "v" + (version.major() - 1) + ".*";
 
-        return Stream.concat(gitWrapper.listVersions(currentMajorPattern), gitWrapper.listVersions(previousMajorPattern))
+        return gitWrapper.listVersions(previousMajorPattern)
             .filter(v -> v.isBefore(version));
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java
@@ -73,6 +73,7 @@ public void apply(Project project) {
             task.setChangelogs(yamlFiles);
             task.setChangelogDirectory(changeLogDirectory);
             task.setBundleFile(projectDirectory.file("docs/release-notes/changelogs-" + version.toString() + ".yml"));
+            task.getOutputs().upToDateWhen(o -> false);
         };
 
         final Function<Boolean, Action<GenerateReleaseNotesTask>> configureGenerateTask = shouldConfigureYamlFiles -> task -> {
@@ -98,6 +99,8 @@ public void apply(Project project) {
 
             task.setChangelogBundleDirectory(projectDirectory.dir("docs/release-notes/changelog-bundles"));
 
+            task.getOutputs().upToDateWhen(o -> false);
+
             task.dependsOn(validateChangelogsTask);
         };
 
diff --git a/build-tools-internal/src/main/resources/templates/index.md b/build-tools-internal/src/main/resources/templates/index.md
@@ -23,16 +23,24 @@ To check for security updates, go to [Security announcements for the Elastic sta
 <%
 for(bundle in changelogBundles) {
     def version = bundle.version
-    def versionWithoutSeparator = bundle.versionWithoutSeparator
+    def versionForIds = bundle.version.toString().equals('9.0.0') ? bundle.versionWithoutSeparator : bundle.version
     def changelogsByTypeByArea = bundle.changelogsByTypeByArea
     def notableHighlights = bundle.notableHighlights
     def nonNotableHighlights = bundle.nonNotableHighlights
     def unqualifiedVersion = bundle.unqualifiedVersion
+    def coming = !bundle.bundle.released
+
+    if (coming) {
+        print "\n"
+        print "```{applies_to}\n"
+        print "stack: coming ${version}\n"
+        print "```"
+    }
 %>
-## ${unqualifiedVersion} [elasticsearch-${versionWithoutSeparator}-release-notes]
+## ${unqualifiedVersion} [elasticsearch-${versionForIds}-release-notes]
 <%
 if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) {
-    print "\n### Highlights [elasticsearch-${versionWithoutSeparator}-highlights]\n"
+    print "\n### Highlights [elasticsearch-${versionForIds}-highlights]\n"
 }
 
 for (highlights in [notableHighlights, nonNotableHighlights]) {
@@ -50,7 +58,7 @@ for (changeType in ['features-enhancements', 'fixes', 'regression']) {
         continue;
     }
 %>
-### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionWithoutSeparator}-${changeType}]
+### ${ TYPE_LABELS.getOrDefault(changeType, 'No mapping for TYPE_LABELS[' + changeType + ']') } [elasticsearch-${versionForIds}-${changeType}]
 <% for (team in changelogsByTypeByArea[changeType].keySet()) {
     print "\n${team}:\n";
 
diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md
@@ -10,8 +10,6 @@ Breaking changes can impact your Elastic applications, potentially disrupting no
 
 If you are migrating from a version prior to version 9.0, you must first upgrade to the last 8.x version available. To learn how to upgrade, check out [Upgrade](docs-content://deploy-manage/upgrade.md).
 
-To learn how to upgrade, check out <upgrade docs>.
-
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
 ## 9.0.1 [elasticsearch-901-breaking-changes]
diff --git a/docs/release-notes/changelog-bundles/9.0.0.yml b/docs/release-notes/changelog-bundles/9.0.0.yml
@@ -1,4 +1,5 @@
 version: 9.0.0
+released: true
 generated: 2025-04-14T18:16:08.465456Z
 changelogs:
   - pr: 90529
diff --git a/docs/release-notes/changelog-bundles/9.0.1.yml b/docs/release-notes/changelog-bundles/9.0.1.yml
@@ -1,11 +1,17 @@
 version: 9.0.1
-generated: 2025-04-28T21:41:06.757953Z
+released: false
+generated: 2025-05-02T19:34:24.151692Z
 changelogs:
   - pr: 125694
     summary: LTR score bounding
     area: Ranking
     type: bug
     issues: []
+  - pr: 125922
+    summary: Fix text structure NPE when fields in list have null value
+    area: Machine Learning
+    type: bug
+    issues: []
   - pr: 126273
     summary: Fix LTR rescorer with model alias
     area: Ranking
@@ -130,3 +136,13 @@ changelogs:
     area: Ingest Node
     type: upgrade
     issues: []
+  - pr: 127414
+    summary: Fix npe when using source confirmed text query against missing field
+    area: Search
+    type: bug
+    issues: []
+  - pr: 127527
+    summary: "No, line noise isn't a valid ip"
+    area: ES|QL
+    type: bug
+    issues: []
diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md
@@ -21,9 +21,12 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
-## 9.0.1 [elasticsearch-901-release-notes]
+```{applies_to}
+stack: coming 9.0.1
+```
+## 9.0.1 [elasticsearch-9.0.1-release-notes]
 
-### Features and enhancements [elasticsearch-901-features-enhancements]
+### Features and enhancements [elasticsearch-9.0.1-features-enhancements]
 
 Infra/Core:
 * Validation checks on paths allowed for 'files' entitlements. Restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories. [#126852](https://github.com/elastic/elasticsearch/pull/126852)
@@ -37,7 +40,7 @@ Search:
 Security:
 * Add Issuer to failed SAML Signature validation logs when available [#126310](https://github.com/elastic/elasticsearch/pull/126310) (issue: [#111022](https://github.com/elastic/elasticsearch/issues/111022))
 
-### Fixes [elasticsearch-901-fixes]
+### Fixes [elasticsearch-9.0.1-fixes]
 
 Aggregations:
 * Rare terms aggregation false **positive** fix [#126884](https://github.com/elastic/elasticsearch/pull/126884)
@@ -55,6 +58,7 @@ ES|QL:
 * Fix count optimization with pushable union types [#127225](https://github.com/elastic/elasticsearch/pull/127225) (issue: [#127200](https://github.com/elastic/elasticsearch/issues/127200))
 * Fix join masking eval [#126614](https://github.com/elastic/elasticsearch/pull/126614)
 * Fix sneaky bug in single value query [#127146](https://github.com/elastic/elasticsearch/pull/127146)
+* No, line noise isn't a valid ip [#127527](https://github.com/elastic/elasticsearch/pull/127527)
 
 ILM+SLM:
 * Fix equality bug in `WaitForIndexColorStep` [#126605](https://github.com/elastic/elasticsearch/pull/126605)
@@ -69,12 +73,16 @@ Infra/Core:
 
 Machine Learning:
 * Adding missing `onFailure` call for Inference API start model request [#126930](https://github.com/elastic/elasticsearch/pull/126930)
+* Fix text structure NPE when fields in list have null value [#125922](https://github.com/elastic/elasticsearch/pull/125922)
 * Leverage threadpool schedule for inference api to avoid long running thread [#126858](https://github.com/elastic/elasticsearch/pull/126858) (issue: [#126853](https://github.com/elastic/elasticsearch/issues/126853))
 
 Ranking:
 * Fix LTR rescorer with model alias [#126273](https://github.com/elastic/elasticsearch/pull/126273)
 * LTR score bounding [#125694](https://github.com/elastic/elasticsearch/pull/125694)
 
+Search:
+* Fix npe when using source confirmed text query against missing field [#127414](https://github.com/elastic/elasticsearch/pull/127414)
+
 TSDB:
 * Improve resiliency of `UpdateTimeSeriesRangeService` [#126637](https://github.com/elastic/elasticsearch/pull/126637)
 

Original file line number	Diff line number	Diff line change
`@@ -149,7 +149,7 @@ static Stream<QualifiedVersion> findPreviousVersion(GitWrapper gitWrapper, Quali`
`149`	`149`	`final String currentMajorPattern = "v" + version.major() + ".*";`
`150`	`150`	`final String previousMajorPattern = "v" + (version.major() - 1) + ".*";`
`151`	`151`
`152`		`- return Stream.concat(gitWrapper.listVersions(currentMajorPattern), gitWrapper.listVersions(previousMajorPattern))`
	`152`	`+ return gitWrapper.listVersions(previousMajorPattern)`
`153`	`153`	`.filter(v -> v.isBefore(version));`
`154`	`154`	`}`
`155`	`155`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`version: 9.0.0`
	`2`	`+released: true`
`2`	`3`	`generated: 2025-04-14T18:16:08.465456Z`
`3`	`4`	`changelogs:`
`4`	`5`	`- pr: 90529`