[cli] Batch: support multiple/nested !include directive (#7354)

Co-authored-by: Jim Schubert <[email protected]>

Author: dtiller

docs/usage.md

@@ -602,7 +602,7 @@ openapi-generator generate -i petstore.yaml -g typescript-fetch -o out \
 The `batch` command allows you to move all CLI arguments supported by the `generate` command into a YAML or JSON file.
 
 *NOTE*: This command supports an additional `!include` property which may point to another "shared" file, the base path to which can be
-modified by `--includes-base-dir`.
+modified by `--includes-base-dir`. Starting with 5.0.0, the `!batch` command supports multiple `!include` properties, either sequential or nested. In order to support multiple `!include` properties in a JSON file, the property name can have a suffix, e.g. `!include1`, `!include2`, etc. The suffix have no meaning other than providing unique property names.
 
 ```bash
 openapi-generator help batch

modules/openapi-generator-cli/src/main/java/org/openapitools/codegen/cmd/GenerateBatch.java

@@ -18,13 +18,19 @@
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.core.TreeNode;
 import com.fasterxml.jackson.databind.*;
 import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier;
 import com.fasterxml.jackson.databind.deser.std.DelegatingDeserializer;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.util.TokenBuffer;
+
 import io.airlift.airline.Arguments;
 import io.airlift.airline.Command;
 import io.airlift.airline.Option;
@@ -268,35 +274,51 @@ protected JsonDeserializer<?> newDelegatingInstance(JsonDeserializer<?> newDeleg
 
         @Override
         public Object deserialize(JsonParser p, DeserializationContext ctx) throws IOException {
-            TreeNode node = p.readValueAsTree();
-            JsonNode include = (JsonNode) node.get(INCLUDE);
             ObjectMapper codec = (ObjectMapper) ctx.getParser().getCodec();
-
-            if (include != null) {
-                String ref = include.textValue();
-                if (ref != null) {
-                    File includeFile = scanDir != null ? new File(scanDir, ref) : new File(ref);
-                    if (includeFile.exists()) {
-                        // load the file into the tree node and continue parsing as normal
-                        ((ObjectNode) node).remove(INCLUDE);
-
-                        TreeNode includeNode;
-                        try (JsonParser includeParser = codec.getFactory().createParser(includeFile)) {
-                            includeNode = includeParser.readValueAsTree();
+            TokenBuffer buffer = new TokenBuffer(p);
+            
+            recurse(buffer, p, codec, false);
+            
+            JsonParser newParser = buffer.asParser(codec);
+            newParser.nextToken();
+            
+            return super.deserialize(newParser, ctx);
+        }
+        
+        private void recurse(TokenBuffer buffer, JsonParser p, ObjectMapper codec, boolean skipOuterbraces) throws IOException {
+            boolean firstToken = true;
+            JsonToken token; 
+            
+            while ((token = p.nextToken()) != null) {
+                String name = p.currentName();
+                
+                if (skipOuterbraces && firstToken && JsonToken.START_OBJECT.equals(token)) {
+                    continue;
+                }
+                
+                if (skipOuterbraces && p.getParsingContext().inRoot() && JsonToken.END_OBJECT.equals(token)) {
+                    continue;
+                }
+                
+                if (JsonToken.VALUE_NULL.equals(token)) {
+                    continue;
+                }
+                
+                if (name != null && JsonToken.FIELD_NAME.equals(token) && name.startsWith(INCLUDE)) {
+                    p.nextToken();
+                    String fileName = p.getText();
+                    if (fileName != null) {
+                        File includeFile = scanDir != null ? new File(scanDir, fileName) : new File(fileName);
+                        if (includeFile.exists()) {
+                            recurse(buffer, codec.getFactory().createParser(includeFile), codec, true);
                         }
-
-                        ObjectReader reader = codec.readerForUpdating(node);
-                        TreeNode updated = reader.readValue(includeNode.traverse());
-                        JsonParser updatedParser = updated.traverse();
-                        updatedParser.nextToken();
-                        return super.deserialize(updatedParser, ctx);
                     }
+                } else {
+                    buffer.copyCurrentEvent(p);
                 }
+                
+                firstToken = false;
             }
-
-            JsonParser newParser = node.traverse();
-            newParser.nextToken();
-            return super.deserialize(newParser, ctx);
         }
     }
 }

modules/openapi-generator-cli/src/test/java/org/openapitools/codegen/cmd/GenerateBatchTest.java

@@ -28,6 +28,12 @@ public class GenerateBatchTest {
     private static final String JAXRS_DATELIB_J8_YAML = "jaxrs-datelib-j8.yaml";
     private static final String JAXRS_DATELIB_J8_YAML_INCLUDE_JSON = "jaxrs-datelib-j8-yaml-include.json";
     private static final String JAXRS_DATELIB_J8_JSON_INCLUDE_YAML = "jaxrs-datelib-j8-json-include.yaml";
+    private static final String JAXRS_DATELIB_J8_DOUBLE_JSON = "jaxrs-datelib-j8-double.json";
+    private static final String JAXRS_DATELIB_J8_DOUBLE_YAML = "jaxrs-datelib-j8-double.yaml";
+    private static final String JAXRS_DATELIB_J8_NESTED_JSON = "jaxrs-datelib-j8-nested.json";
+    private static final String JAXRS_DATELIB_J8_NESTED_YAML = "jaxrs-datelib-j8-nested.yaml";
+    private static final String JAXRS_DATELIB_J8_NESTED_PROPERTY_MERGE_YAML = "jaxrs-datelib-j8-nested-property-merge.yaml";
+
     Path workingDirectory;
 
     @BeforeTest
@@ -40,7 +46,12 @@ public Object[][] customIncludeDeserializerFiles() {
         return new Object[][] {
                 {JAXRS_DATELIB_J8_JSON},
                 {JAXRS_DATELIB_J8_YAML},
-                {JAXRS_DATELIB_J8_JSON_INCLUDE_YAML}
+                {JAXRS_DATELIB_J8_JSON_INCLUDE_YAML},
+                {JAXRS_DATELIB_J8_DOUBLE_JSON},
+                {JAXRS_DATELIB_J8_DOUBLE_YAML},
+                {JAXRS_DATELIB_J8_NESTED_JSON},
+                {JAXRS_DATELIB_J8_NESTED_YAML},
+                {JAXRS_DATELIB_J8_NESTED_PROPERTY_MERGE_YAML}
         };
     }
 

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-double.json

@@ -0,0 +1,4 @@
+{
+  "!include": "jaxrs-datelib-j8.json",
+  "!include1": "common/jaxrs-datelib-j8.json"
+}

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-double.yaml

@@ -0,0 +1,2 @@
+"!include": jaxrs-datelib-j8.yaml
+"!include": common/jaxrs-datelib-j8.yaml

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-nested-include-property-merge.yaml

@@ -0,0 +1,11 @@
+---
+"!include": common/jaxrs-datelib-j8.yaml
+generatorName: jaxrs-jersey
+# We expect this property to be ignored because it exists in the importing file
+outputDir: outputDir-should-be-ignored
+inputSpec: batch/specs/petstore.yaml
+# This map should be ignored since it exists in the importing file
+# We may want to consider merging additionalProperties, but that would depend on user need
+additionalProperties:
+  hideGenerationTimestamp: true
+  serverPort: 'should-be-ignored'

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-nested-include.json

@@ -0,0 +1,9 @@
+{
+  "!include": "common/jaxrs-datelib-j8.json",
+  "generatorName": "jaxrs-jersey",
+  "outputDir": "outputDir",
+  "additionalProperties": {
+    "hideGenerationTimestamp": true,
+    "serverPort": "8082"
+  }
+}

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-nested-include.yaml

@@ -0,0 +1,7 @@
+---
+"!include": common/jaxrs-datelib-j8.yaml
+generatorName: jaxrs-jersey
+outputDir: outputDir
+additionalProperties:
+  hideGenerationTimestamp: true
+  serverPort: '8082'

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-nested-property-merge.yaml

@@ -0,0 +1,7 @@
+---
+"!include": jaxrs-datelib-j8-nested-include-property-merge.yaml
+# We expect this one to "win"
+outputDir: outputDir
+additionalProperties:
+  hideGenerationTimestamp: true
+  serverPort: '8082'

modules/openapi-generator-cli/src/test/resources/batch/jaxrs-datelib-j8-nested.json

@@ -0,0 +1,4 @@
+{
+	"!include": "jaxrs-datelib-j8-nested-include.json",
+	"inputSpec": "batch/specs/petstore.yaml"
+}