ESQL: Fail build docs aren't up to date

x-pack/plugin/esql/build.gradle

@@ -94,37 +94,58 @@ interface Injected {
 }
 
 tasks.named("test").configure {
-  if (buildParams.ci == false) {
-    systemProperty 'generateDocs', true
-    def injected = project.objects.newInstance(Injected)
-    // Define the folder to delete and recreate
-    def tempDir = file("build/testrun/test/temp/esql")
-    def commandsExamplesFile = new File(tempDir, "commands.examples")
-    // Find all matching .md files for commands examples
-    def mdFiles = fileTree("${rootDir}/docs/reference/query-languages/esql/_snippets/commands/examples/") {
-      include("**/*.csv-spec/*.md")
+  def injected = project.objects.newInstance(Injected)
+  // Define the folder to delete and recreate
+  def tempDir = file("build/testrun/test/temp/esql")
+  def commandsExamplesFile = new File(tempDir, "commands.examples")
+  def existingFiles = new File(tempDir, "existing.files")
+  // Find all matching .md files for commands examples
+  def mdFiles = fileTree("${rootDir}/docs/reference/query-languages/esql/_snippets/commands/examples/") {
+    include("**/*.csv-spec/*.md")
+  }
+  def esqlDocFolder = file("${rootDir}/docs/reference/query-languages/esql").toPath()
+  def imagesDocFolder = file("${esqlDocFolder}/images")
+  def snippetsDocFolder = file("${esqlDocFolder}/_snippets")
+  def kibanaDocFolder = file("${esqlDocFolder}/kibana")
+  File imagesFolder = file("build/testrun/test/temp/esql/images")
+  File snippetsFolder = file("build/testrun/test/temp/esql/_snippets")
+  File kibanaFolder = file("build/testrun/test/temp/esql/kibana")
+
+  doFirst {
+    injected.fs.delete {
+      it.delete(tempDir)
     }
-    doFirst {
-      injected.fs.delete {
-        it.delete(tempDir)
-      }
-      // Re-create this folder so we can save a table of generated examples to extract from csv-spec tests
-      tempDir.mkdirs() // Recreate the folder
+    // Re-create this folder so we can save a table of generated examples to extract from csv-spec tests
+    tempDir.mkdirs() // Recreate the folder
 
-      // Write directory name and filename of each .md file to the output file
-      commandsExamplesFile.withWriter { writer ->
-        mdFiles.each { file ->
-          writer.writeLine("${file.parentFile.name}/${file.name}")
+    // Write directory name and filename of each .md file to the output file
+    commandsExamplesFile.withWriter { writer ->
+      mdFiles.each { file ->
+        writer.writeLine("${file.parentFile.name}/${file.name}")
+      }
+    }
+    println "File 'commands.examples' created with ${mdFiles.size()} example specifications from csv-spec files."
+    if (buildParams.ci) {
+      existingFiles.withWriter { writer ->
+        injected.fs.sync {
+          from snippetsDocFolder
+          into snippetsFolder
+        }
+        injected.fs.sync {
+          from imagesDocFolder
+          into imagesFolder
+        }
+        injected.fs.sync {
+          from kibanaDocFolder
+          into kibanaFolder
         }
       }
-      println "File 'commands.examples' created with ${mdFiles.size()} example specifications from csv-spec files."
     }
-    File imagesFolder = file("build/testrun/test/temp/esql/images")
-    File snippetsFolder = file("build/testrun/test/temp/esql/_snippets")
-    File kibanaFolder = file("build/testrun/test/temp/esql/kibana")
-    def imagesDocFolder = file("${rootDir}/docs/reference/query-languages/esql/images")
-    def snippetsDocFolder = file("${rootDir}/docs/reference/query-languages/esql/_snippets")
-    def kibanaDocFolder = file("${rootDir}/docs/reference/query-languages/esql/kibana")
+  }
+  if (buildParams.ci) {
+    systemProperty 'generateDocs', 'assert'
+  } else {
+    systemProperty 'generateDocs', 'write'
     def snippetsTree = fileTree(snippetsFolder).matching {
       include "**/types/*.md"  // Recursively include all types/*.md files (effectively counting functions and operators)
     }
@@ -229,8 +250,10 @@ tasks.named("test").configure {
 // This is similar to the test task above, but needed for the LookupJoinTypesIT which runs in the internalClusterTest task
 // and generates a types table for the LOOKUP JOIN command. It is possible in future we might have move tests that do this.
 tasks.named("internalClusterTest").configure {
-  if (buildParams.ci == false) {
-    systemProperty 'generateDocs', true
+  if (buildParams.ci) {
+    systemProperty 'generateDocs', 'assert'
+  } else {
+    systemProperty 'generateDocs', 'write'
     def injected = project.objects.newInstance(Injected)
     // Define the folder to delete and recreate
     def tempDir = file("build/testrun/internalClusterTest/temp/esql")

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java

@@ -768,6 +768,9 @@ private boolean isValidDataType(DataType dataType) {
     }
 
     private static void saveJoinTypes(Supplier<Map<List<DataType>, DataType>> signatures) throws Exception {
+        if (System.getProperty("generateDocs") == null) {
+            return;
+        }
         ArrayList<EsqlFunctionRegistry.ArgSignature> args = new ArrayList<>();
         args.add(new EsqlFunctionRegistry.ArgSignature("field from the left index", null, null, false, false));
         args.add(new EsqlFunctionRegistry.ArgSignature("field from the lookup index", null, null, false, false));
@@ -776,7 +779,8 @@ private static void saveJoinTypes(Supplier<Map<List<DataType>, DataType>> signat
             LookupJoinTypesIT.class,
             null,
             args,
-            signatures
+            signatures,
+            DocsV3Support.callbacksFromSystemProperty()
         );
         docs.renderDocs();
     }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CommandDocsTests.java

@@ -8,6 +8,8 @@
 package org.elasticsearch.xpack.esql;
 
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.DocsV3Support;
@@ -50,8 +52,10 @@ public static Map<List<DataType>, DataType> signatures() {
      * in the documentation.
      */
     public static class TestDocsV3Support extends DocsV3Support {
+        private static final Logger logger = LogManager.getLogger(TestDocsV3Support.class);
+
         public TestDocsV3Support() {
-            super("commands", "commands", CommandDocsTests.class, null);
+            super("commands", "commands", CommandDocsTests.class, null, callbacksFromSystemProperty());
         }
 
         @Override
@@ -108,22 +112,19 @@ protected boolean renderExample(String lineSpec) throws IOException {
             }
             String rendered = builder.toString();
             logger.info("Writing example for [{}]:\n{}", name, rendered);
-            writeExampleFile(csvFile, tagFile, rendered);
+            writeExampleFile(csvFile, tag, rendered);
             return true;
         }
 
-        protected void writeExampleFile(String csvFile, String tagFile, String str) throws IOException {
+        protected void writeExampleFile(String csvFile, String tag, String str) throws IOException {
             // We have to write to a tempdir because it’s all test are allowed to write to. Gradle can move them.
             Path dir = PathUtils.get(System.getProperty("java.io.tmpdir"))
                 .resolve("esql")
                 .resolve("_snippets")
                 .resolve(category)
                 .resolve("examples")
                 .resolve(csvFile);
-            Files.createDirectories(dir);
-            Path file = dir.resolve(tagFile);
-            Files.writeString(file, str);
-            logger.info("Wrote to file: {}", file);
+            callbacks.write(dir, tag, "md", str, false);
         }
 
         @SuppressWarnings("SameParameterValue")