#4852 fix: allow recursive models to process without StackOverflow (#5004)

This PR adds a cycle-detection safeguard to ModelResolver to prevent infinite recursion when resolving array schemas that reference subclasses or self-referential types.

It introduces a dedicated helper method resolveArraySchemaWithCycleGuard(...), which tracks types currently being resolved in a typesBeingResolved set and ensures they are safely added and removed using a try/finally block.

This change fixes StackOverflowError occurrences (issue #4852) when generating schemas for models containing recursive array structures, and adds regression tests with JSON fixtures for both OpenAPI 3.0 and 3.1 to verify correct schema output.

Author: Yaytay

modules/swagger-core/src/main/java/io/swagger/v3/core/jackson/ModelResolver.java

@@ -141,6 +141,8 @@ public class ModelResolver extends AbstractModelConverter implements ModelConver
 
     protected ValidatorProcessor validatorProcessor;
 
+    protected Set<AnnotatedType> typesBeingResolved = new HashSet<>();
+
     public ModelResolver(ObjectMapper mapper) {
         super(mapper);
     }
@@ -829,7 +831,8 @@ public Schema resolve(AnnotatedType annotatedType, ModelConverterContext context
                     if (reResolvedProperty.isPresent()) {
                         property = reResolvedProperty.get();
                     }
-                    reResolvedProperty = AnnotationsUtils.getArraySchema(ctxArraySchema, annotatedType.getComponents(), null, openapi31, property, true);
+
+                    reResolvedProperty = resolveArraySchemaWithCycleGuard(ctxArraySchema, annotatedType, openapi31, property);
                     if (reResolvedProperty.isPresent()) {
                         property = reResolvedProperty.get();
                     }
@@ -3608,4 +3611,24 @@ protected boolean applySchemaResolution() {
                 (Boolean.parseBoolean(System.getProperty(Schema.APPLY_SCHEMA_RESOLUTION_PROPERTY, "false")) ||
                         Boolean.parseBoolean(System.getenv(Schema.APPLY_SCHEMA_RESOLUTION_PROPERTY)));
     }
+
+    private Optional<Schema> resolveArraySchemaWithCycleGuard(
+            io.swagger.v3.oas.annotations.media.ArraySchema ctxArraySchema,
+            AnnotatedType annotatedType,
+            boolean openapi31,
+            Schema<?> property) {
+        boolean processSchemaImplementation = !typesBeingResolved.contains(annotatedType);
+        Optional<Schema> reResolvedProperty;
+        if (processSchemaImplementation) {
+            typesBeingResolved.add(annotatedType);
+        } try {
+            reResolvedProperty = AnnotationsUtils.getArraySchema(ctxArraySchema, annotatedType.getComponents(), null,
+                    openapi31, property, processSchemaImplementation );
+        } finally {
+            if (processSchemaImplementation) {
+                    typesBeingResolved.remove(annotatedType);
+                }
+        }
+        return reResolvedProperty;
+    }
 }

modules/swagger-core/src/test/java/io/swagger/v3/core/converting/ArrayOfSubclassTest.java

@@ -0,0 +1,43 @@
+package io.swagger.v3.core.converting;
+
+import io.swagger.v3.core.converter.ModelConverters;
+import io.swagger.v3.core.converter.ResolvedSchema;
+import io.swagger.v3.core.oas.models.ModelWithArrayOfSubclasses;
+import io.swagger.v3.core.util.Json31;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import org.testng.annotations.Test;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JsonNode;
+
+
+public class ArrayOfSubclassTest {
+
+    @Test
+    public void extractSubclassArray_oas31() throws Exception {
+        ResolvedSchema schema = ModelConverters.getInstance(true).readAllAsResolvedSchema(ModelWithArrayOfSubclasses.Holder.class);
+        assertNotNull(schema);
+        String expectedJson = new String(Files.readAllBytes(Paths.get("src/test/java/io/swagger/v3/core/converting/ArrayOfSubclassTest_expected31.json")));
+        String actualJson = Json31.pretty(schema);
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode expectedNode = mapper.readTree(expectedJson);
+        JsonNode actualNode = mapper.readTree(actualJson);
+        assertEquals(actualNode, expectedNode);
+    }
+
+    @Test
+    public void extractSubclassArray_oas30() throws Exception {
+        ResolvedSchema schema = ModelConverters.getInstance(false).readAllAsResolvedSchema(ModelWithArrayOfSubclasses.Holder.class);
+        assertNotNull(schema);
+        String expectedJson = new String(Files.readAllBytes(Paths.get("src/test/java/io/swagger/v3/core/converting/ArrayOfSubclassTest_expected30.json")));
+        String actualJson = Json31.pretty(schema);
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode expectedNode = mapper.readTree(expectedJson);
+        JsonNode actualNode = mapper.readTree(actualJson);
+        assertEquals(actualNode, expectedNode);
+    }
+}

modules/swagger-core/src/test/java/io/swagger/v3/core/converting/ArrayOfSubclassTest_expected30.json

@@ -0,0 +1,82 @@
+{
+  "schema" : {
+    "description" : "The holder",
+    "properties" : {
+      "name" : {
+        "type" : "string"
+      },
+      "friend" : {
+        "type" : "string"
+      },
+      "baseArray" : {
+        "type" : "array",
+        "description" : "Thingy",
+        "items" : {
+          "$ref" : "#/components/schemas/Base"
+        },
+        "minItems" : 0,
+        "uniqueItems" : true
+      }
+    }
+  },
+  "referencedSchemas" : {
+    "Base" : {
+      "description" : "Stuff",
+      "discriminator" : {
+        "propertyName" : "name",
+        "mapping" : {
+          "a" : "#/components/schemas/SubA",
+          "b" : "#/components/schemas/SubB"
+        }
+      },
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        }
+      }
+    },
+    "Holder" : {
+      "description" : "The holder",
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        },
+        "friend" : {
+          "type" : "string"
+        },
+        "baseArray" : {
+          "type" : "array",
+          "description" : "Thingy",
+          "items" : {
+            "$ref" : "#/components/schemas/Base"
+          },
+          "minItems" : 0,
+          "uniqueItems" : true
+        }
+      }
+    },
+    "SubA" : {
+      "description" : "The SubA class",
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        },
+        "count" : {
+          "type" : "integer",
+          "format" : "int64"
+        }
+      }
+    },
+    "SubB" : {
+      "description" : "The SubB class",
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        },
+        "friend" : {
+          "type" : "string"
+        }
+      }
+    }
+  }
+}

modules/swagger-core/src/test/java/io/swagger/v3/core/converting/ArrayOfSubclassTest_expected31.json

@@ -0,0 +1,106 @@
+{
+  "schema" : {
+    "type" : "object",
+    "description" : "The holder",
+    "properties" : {
+      "name" : {
+        "type" : "string"
+      },
+      "friend" : {
+        "type" : "string"
+      },
+      "baseArray" : {
+        "type" : "array",
+        "description" : "Thingy",
+        "items" : {
+          "$ref" : "#/components/schemas/Base",
+          "description" : "Thingy",
+          "minItems" : 0,
+          "uniqueItems" : true
+        },
+        "minItems" : 0,
+        "uniqueItems" : true
+      }
+    }
+  },
+  "referencedSchemas" : {
+    "Base" : {
+      "type" : "object",
+      "description" : "Stuff",
+      "discriminator" : {
+        "propertyName" : "name",
+        "mapping" : {
+          "a" : "#/components/schemas/SubA",
+          "b" : "#/components/schemas/SubB"
+        }
+      },
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        }
+      }
+    },
+    "Holder" : {
+      "type" : "object",
+      "description" : "The holder",
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        },
+        "friend" : {
+          "type" : "string"
+        },
+        "baseArray" : {
+          "type" : "array",
+          "description" : "Thingy",
+          "items" : {
+            "$ref" : "#/components/schemas/Base",
+            "description" : "Thingy",
+            "minItems" : 0,
+            "uniqueItems" : true
+          },
+          "minItems" : 0,
+          "uniqueItems" : true
+        }
+      }
+    },
+    "SubA" : {
+      "type" : "object",
+      "description" : "The SubA class",
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        },
+        "count" : {
+          "type" : "integer",
+          "format" : "int64"
+        }
+      }
+    },
+    "SubB" : {
+      "type" : "object",
+      "description" : "The SubB class",
+      "properties" : {
+        "name" : {
+          "type" : "string"
+        },
+        "friend" : {
+          "type" : "string"
+        },
+        "baseArray" : {
+          "type" : "array",
+          "description" : "Thingy",
+          "items" : {
+            "$ref" : "#/components/schemas/Base",
+            "description" : "Thingy",
+            "minItems" : 0,
+            "uniqueItems" : true
+          },
+          "minItems" : 0,
+          "uniqueItems" : true
+        }
+      }
+    }
+  }
+}
+

modules/swagger-core/src/test/java/io/swagger/v3/core/oas/models/ModelWithArrayOfSubclasses.java

@@ -0,0 +1,63 @@
+package io.swagger.v3.core.oas.models;
+
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.DiscriminatorMapping;
+import io.swagger.v3.oas.annotations.media.Schema;
+
+public class ModelWithArrayOfSubclasses {
+    
+    @Schema(description = "The holder")
+    public class Holder extends SubB {
+    }
+  
+    @Schema(
+            discriminatorProperty = "name"
+            , discriminatorMapping = {
+              @DiscriminatorMapping(schema = SubA.class, value = "a") 
+              , @DiscriminatorMapping(schema = SubB.class, value = "b") 
+            }
+            , description = "Stuff"
+    )
+    public class Base {
+      
+        private String name;
+      
+        public String getName() {
+            return name;
+        }
+    }
+
+    @Schema(description = "The SubA class")
+    public class SubA extends Base {
+      
+        private Long count;
+      
+        public Long getCount() {
+            return count;
+        }
+    }
+
+    @Schema(description = "The SubB class")
+    public class SubB extends Base {
+      
+        private String friend;
+        private Base[] baseArray;
+
+        public String getFriend() {
+            return friend;
+        }
+
+        @ArraySchema(
+                schema = @Schema(implementation = Base.class)
+                , arraySchema = @Schema(
+                        type = "array"
+                        , description = "Thingy"
+                )
+                , minItems = 0
+                , uniqueItems = true
+        )
+        public Base[] getBaseArray() {
+            return baseArray;
+        }
+    }
+}