feat: [OpenAPI] Exclusion of paths and properties (#745)

Co-authored-by: Alexander Dümont <[email protected]>
Co-authored-by: Roshin Rajan Panackal <[email protected]>
Co-authored-by: Roshin Rajan Panackal <[email protected]>

Author: 3 people

datamodel/openapi/openapi-generator/src/main/java/com/sap/cloud/sdk/datamodel/openapi/generator/CustomJavaClientCodegen.java

@@ -1,10 +1,16 @@
 package com.sap.cloud.sdk.datamodel.openapi.generator;
 
 import static com.sap.cloud.sdk.datamodel.openapi.generator.GeneratorCustomProperties.FIX_REDUNDANT_IS_BOOLEAN_PREFIX;
+import static com.sap.cloud.sdk.datamodel.openapi.generator.GeneratorCustomProperties.FIX_REMOVE_UNUSED_COMPONENTS;
+import static com.sap.cloud.sdk.datamodel.openapi.generator.GeneratorCustomProperties.USE_EXCLUDE_PATHS;
+import static com.sap.cloud.sdk.datamodel.openapi.generator.GeneratorCustomProperties.USE_EXCLUDE_PROPERTIES;
 import static com.sap.cloud.sdk.datamodel.openapi.generator.GeneratorCustomProperties.USE_FLOAT_ARRAYS;
 import static com.sap.cloud.sdk.datamodel.openapi.generator.GeneratorCustomProperties.USE_ONE_OF_CREATORS;
 
+import java.util.Arrays;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -23,9 +29,11 @@
 
 import com.sap.cloud.sdk.datamodel.openapi.generator.model.GenerationConfiguration;
 
+import io.swagger.v3.oas.models.OpenAPI;
 import io.swagger.v3.oas.models.media.Schema;
 import lombok.extern.slf4j.Slf4j;
 
+@SuppressWarnings( "PMD.TooManyStaticImports" )
 @Slf4j
 class CustomJavaClientCodegen extends JavaClientCodegen
 {
@@ -38,6 +46,33 @@ public CustomJavaClientCodegen( @Nonnull final GenerationConfiguration config )
         this.config = config;
     }
 
+    @Override
+    public void preprocessOpenAPI( @Nonnull final OpenAPI openAPI )
+    {
+        if( USE_EXCLUDE_PROPERTIES.isEnabled(config) ) {
+            final String[] exclusions = USE_EXCLUDE_PROPERTIES.getValue(config).trim().split("[,\\s]+");
+            for( final String exclusion : exclusions ) {
+                final String[] split = exclusion.split("\\.", 2);
+                preprocessRemoveProperty(openAPI, split[0], split[1]);
+            }
+        }
+
+        if( USE_EXCLUDE_PATHS.isEnabled(config) ) {
+            final String[] exclusions = USE_EXCLUDE_PATHS.getValue(config).trim().split("[,\\s]+");
+            for( final String exclusion : exclusions ) {
+                if( !openAPI.getPaths().keySet().remove(exclusion) ) {
+                    log.error("Could not remove path {}", exclusion);
+                }
+            }
+        }
+
+        super.preprocessOpenAPI(openAPI);
+
+        if( FIX_REMOVE_UNUSED_COMPONENTS.isEnabled(config) ) {
+            preprocessRemoveRedundancies(openAPI);
+        }
+    }
+
     @Override
     protected
         void
@@ -104,6 +139,124 @@ protected void updateModelForComposedSchema(
         }
     }
 
+    /**
+     * Remove property from specification.
+     *
+     * @param openAPI
+     *            The OpenAPI specification to update.
+     * @param schemaName
+     *            The name of the schema to update.
+     * @param propertyName
+     *            The name of the property to remove.
+     */
+    @SuppressWarnings( { "rawtypes", "unchecked", "ReplaceInefficientStreamCount" } )
+    private void preprocessRemoveProperty(
+        @Nonnull final OpenAPI openAPI,
+        @Nonnull final String schemaName,
+        @Nonnull final String propertyName )
+    {
+        final var schema = openAPI.getComponents().getSchemas().get(schemaName);
+        if( schema == null ) {
+            log.error("Could not find schema {} to remove property {} from.", schemaName, propertyName);
+            return;
+        }
+        boolean removed = false;
+
+        final Predicate<Schema> remove =
+            s -> s != null && s.getProperties() != null && s.getProperties().remove(propertyName) != null;
+        final var schemasQueued = new LinkedList<Schema>();
+        final var schemasDone = new HashSet<Schema>();
+        schemasQueued.add(schema);
+
+        while( !schemasQueued.isEmpty() ) {
+            final var s = schemasQueued.remove();
+            if( s == null || !schemasDone.add(s) ) {
+                continue;
+            }
+            // check removal of direct schema property
+            removed |= remove.test(s);
+
+            // check for allOf, anyOf, oneOf
+            for( final List<Schema> list : Arrays.asList(s.getAllOf(), s.getAnyOf(), s.getOneOf()) ) {
+                if( list != null ) {
+                    schemasQueued.addAll(list);
+                }
+            }
+        }
+        if( !removed ) {
+            log.error("Could not remove property {} from schema {}.", propertyName, schemaName);
+        }
+    }
+
+    /**
+     * Remove unused schema components.
+     *
+     * @param openAPI
+     *            The OpenAPI specification to update.
+     */
+    @SuppressWarnings( { "rawtypes", "unchecked" } )
+    private void preprocessRemoveRedundancies( @Nonnull final OpenAPI openAPI )
+    {
+        final var queue = new LinkedList<Schema>();
+        final var done = new HashSet<Schema>();
+        final var refs = new LinkedHashSet<String>();
+        final var pattern = Pattern.compile("\\$ref: #/components/schemas/(\\w+)");
+
+        // find and queue schemas nested in paths
+        for( final var path : openAPI.getPaths().values() ) {
+            final var m = pattern.matcher(path.toString());
+            while( m.find() ) {
+                final var name = m.group(1);
+                final var schema = openAPI.getComponents().getSchemas().get(name);
+                queue.add(schema);
+                refs.add(m.group(0).split(" ")[1]);
+            }
+        }
+
+        while( !queue.isEmpty() ) {
+            final var s = queue.remove();
+            if( s == null || !done.add(s) ) {
+                continue;
+            }
+
+            // check for $ref attribute
+            final var ref = s.get$ref();
+            if( ref != null ) {
+                refs.add(ref);
+                final var refName = ref.substring(ref.lastIndexOf('/') + 1);
+                queue.add(openAPI.getComponents().getSchemas().get(refName));
+            }
+
+            // check for direct properties
+            if( s.getProperties() != null ) {
+                for( final var s1 : s.getProperties().values() ) {
+                    queue.add((Schema) s1);
+                }
+            }
+
+            // check for array items
+            if( s.getItems() != null ) {
+                queue.add(s.getItems());
+            }
+
+            // check for allOf, anyOf, oneOf
+            for( final List<Schema> list : Arrays.asList(s.getAllOf(), s.getAnyOf(), s.getOneOf()) ) {
+                if( list != null ) {
+                    queue.addAll(list);
+                }
+            }
+        }
+
+        // remove all schemas that have not been marked "used"
+        openAPI.getComponents().getSchemas().keySet().removeIf(schema -> {
+            if( !refs.contains("#/components/schemas/" + schema) ) {
+                log.info("Removing unused schema {}", schema);
+                return true;
+            }
+            return false;
+        });
+    }
+
     /**
      * Use JsonCreator for interface sub-types in case there are any primitives.
      *

datamodel/openapi/openapi-generator/src/main/java/com/sap/cloud/sdk/datamodel/openapi/generator/GeneratorCustomProperties.java

@@ -25,7 +25,22 @@ enum GeneratorCustomProperties
     /**
      * Use float arrays instead of big-decimal lists.
      */
-    USE_FLOAT_ARRAYS("useFloatArrays", "false");
+    USE_FLOAT_ARRAYS("useFloatArrays", "false"),
+
+    /**
+     * Exclude generation of properties, e.g. `schemaName1.propertyNameA, schemaName2.propertyNameB`.
+     */
+    USE_EXCLUDE_PROPERTIES("excludeProperties", "false"),
+
+    /**
+     * Exclude generation of APIs that match a provided path, e.g. `/api/v1/health, /deployments/{id}/completions`.
+     */
+    USE_EXCLUDE_PATHS("excludePaths", "false"),
+
+    /**
+     * Remove schema components that are unused, before generating them.
+     */
+    FIX_REMOVE_UNUSED_COMPONENTS("removeUnusedComponents", "false");
 
     private final String key;
     private final String defaultValue;

datamodel/openapi/openapi-generator/src/test/java/com/sap/cloud/sdk/datamodel/openapi/generator/DataModelGeneratorIntegrationTest.java

@@ -1,5 +1,7 @@
 package com.sap.cloud.sdk.datamodel.openapi.generator;
 
+import static java.util.Map.entry;
+
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.nio.file.Files;
@@ -55,6 +57,20 @@ private enum TestCase
             true,
             6,
             Map.of()),
+        PARTIAL_GENERATION(
+            "partial-generation",
+            "sodastore.json",
+            "com.sap.cloud.sdk.services.builder.api",
+            "com.sap.cloud.sdk.services.builder.model",
+            ApiMaturity.RELEASED,
+            true,
+            true,
+            4,
+            Map
+                .ofEntries(
+                    entry("excludePaths", "/sodas,/foobar/{baz}"),
+                    entry("excludeProperties", "Foo.bar,Soda.embedding,Soda.flavor,UpdateSoda.flavor,SodaWithFoo.foo"),
+                    entry("removeUnusedComponents", "true"))),
         INPUT_SPEC_WITH_UPPERCASE_FILE_EXTENSION(
             "input-spec-with-uppercase-file-extension",
             "sodastore.JSON",
@@ -207,7 +223,7 @@ void generateDataModelForComparison( final TestCase testCase )
         testCase.additionalProperties.forEach(generationConfiguration::additionalProperty);
 
         GenerationConfiguration build = generationConfiguration.build();
-        new DataModelGenerator().generateDataModel(build);
+        new DataModelGenerator().generateDataModel(build).onFailure(Throwable::printStackTrace);
     }
 
     private static Path getInputDirectory( final TestCase testCase )