Skip to content

Commit cb7c5dd

Browse files
daniel-kmiecikdaniel-kmiecik
committed
fix: parse large file with resolve option set to true
1 parent 5625506 commit cb7c5dd

File tree

6 files changed

+86259
-102
lines changed

6 files changed

+86259
-102
lines changed

‎modules/swagger-parser-v3/src/main/java/io/swagger/v3/parser/reference/ReferenceVisitor.java‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@
1515
import io.swagger.v3.oas.models.security.SecurityScheme;
1616
import io.swagger.v3.parser.core.models.AuthorizationValue;
1717
import io.swagger.v3.parser.urlresolver.PermittedUrlsChecker;
18+
import io.swagger.v3.parser.util.DeserializationUtils;
1819
import io.swagger.v3.parser.util.RemoteUrl;
1920
import org.apache.commons.lang3.StringUtils;
2021
import org.slf4j.LoggerFactory;
22+
import org.yaml.snakeyaml.LoaderOptions;
23+
import org.yaml.snakeyaml.Yaml;
2124

2225
import java.util.HashMap;
2326
import java.util.HashSet;
@@ -35,6 +38,8 @@ public class ReferenceVisitor extends AbstractVisitor {
3538
protected Reference reference;
3639
protected DereferencerContext context;
3740
private PermittedUrlsChecker permittedUrlsChecker;
41+
private final LoaderOptions loaderOptions;
42+
3843

3944
public ReferenceVisitor(
4045
Reference reference,
@@ -46,6 +51,8 @@ public ReferenceVisitor(
4651
this.visited = visited;
4752
this.visitedMap = visitedMap;
4853
this.context = null;
54+
this.loaderOptions = DeserializationUtils.buildLoaderOptions();
55+
4956
}
5057

5158
public ReferenceVisitor(
@@ -61,6 +68,7 @@ public ReferenceVisitor(
6168
this.context = context;
6269
this.permittedUrlsChecker = new PermittedUrlsChecker(context.getParseOptions().getRemoteRefAllowList(),
6370
context.getParseOptions().getRemoteRefBlockList());
71+
this.loaderOptions = DeserializationUtils.buildLoaderOptions();
6472
}
6573

6674
public String toBaseURI(String uri) throws Exception{
@@ -301,7 +309,25 @@ public JsonNode findAnchor(JsonNode root, String anchor) {
301309

302310
public JsonNode deserializeIntoTree(String content) throws Exception {
303311
boolean isJson = content.trim().startsWith("{");
304-
return isJson ? Json31.mapper().readTree(content) : Yaml31.mapper().readTree(content);
312+
if (isJson) {
313+
return Json31.mapper().readTree(content);
314+
}
315+
Yaml yaml = getYaml();
316+
317+
Object yamlObject = yaml.load(content);
318+
return Yaml31.mapper().valueToTree(yamlObject);
319+
}
320+
321+
private Yaml getYaml() {
322+
Yaml yaml;
323+
String yamlCodePoints = System.getProperty("maxYamlCodePoints");
324+
if (yamlCodePoints != null && !yamlCodePoints.isEmpty() && StringUtils.isNumeric(yamlCodePoints)) {
325+
loaderOptions.setCodePointLimit(Integer.parseInt(yamlCodePoints));
326+
yaml = new Yaml(loaderOptions);
327+
} else {
328+
yaml = new Yaml();
329+
}
330+
return yaml;
305331
}
306332

307333
public JsonNode parse(String absoluteUri, List<AuthorizationValue> auths) throws Exception {

‎modules/swagger-parser-v3/src/main/java/io/swagger/v3/parser/util/DeserializationUtils.java‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,11 @@ public void setYamlCycleCheck(boolean yamlCycleCheck) {
7979
}
8080

8181
public Integer getMaxYamlCodePoints() {
82-
return maxYamlCodePoints;
82+
return maxYamlCodePoints;
8383
}
8484

8585
public void setMaxYamlCodePoints(Integer maxYamlCodePoints) {
86-
this.maxYamlCodePoints = maxYamlCodePoints;
86+
this.maxYamlCodePoints = maxYamlCodePoints;
8787
}
8888

8989
public Integer getMaxYamlAliasesForCollections() {
@@ -103,7 +103,7 @@ public void setYamlAllowRecursiveKeys(boolean yamlAllowRecursiveKeys) {
103103
}
104104
}
105105

106-
private static Options options = new Options();
106+
private static final Options options = new Options();
107107

108108
private static final Logger LOGGER = LoggerFactory.getLogger(DeserializationUtils.class);
109109

@@ -145,7 +145,6 @@ protected void addImplicitResolvers() {
145145
addImplicitResolver(Tag.MERGE, MERGE, "<");
146146
addImplicitResolver(Tag.NULL, NULL, "~nN\0");
147147
addImplicitResolver(Tag.NULL, EMPTY, null);
148-
// addImplicitResolver(Tag.TIMESTAMP, TIMESTAMP, "0123456789");
149148
}
150149
}
151150

@@ -249,15 +248,15 @@ public static JsonNode readYamlTree(String contents, ParseOptions parseOptions,
249248
return Json.mapper().convertValue(yaml.load(contents), JsonNode.class);
250249
}
251250
try {
252-
org.yaml.snakeyaml.Yaml yaml = null;
251+
org.yaml.snakeyaml.Yaml yaml;
253252
if (options.isValidateYamlInput()) {
254253
yaml = buildSnakeYaml(new CustomSnakeYamlConstructor());
255254
} else {
256255
yaml = buildSnakeYaml(new SafeConstructor(buildLoaderOptions()));
257256
}
258257
Object o = yaml.load(contents);
259258
if (options.isValidateYamlInput()) {
260-
boolean res = exceedsLimits(o, null, new Integer(0), new IdentityHashMap<Object, Long>(), deserializationUtilsResult);
259+
boolean res = exceedsLimits(o, null, 0, new IdentityHashMap<>(), deserializationUtilsResult);
261260
if (res) {
262261
LOGGER.warn("Error converting snake-parsed yaml to JsonNode");
263262
return getYaml30Mapper().readTree(contents);
@@ -270,8 +269,8 @@ public static JsonNode readYamlTree(String contents, ParseOptions parseOptions,
270269
//
271270
}
272271
return Json.mapper().convertValue(o, JsonNode.class);
273-
} catch (Throwable e) {
274-
LOGGER.warn("Error snake-parsing yaml content", e);
272+
} catch (Exception e) {
273+
LOGGER.warn(e.getMessage(), e);
275274
if (deserializationUtilsResult != null) {
276275
deserializationUtilsResult.message(e.getMessage());
277276
}
@@ -302,8 +301,7 @@ public static org.yaml.snakeyaml.Yaml buildSnakeYaml(BaseConstructor constructor
302301
}
303302
try {
304303
LoaderOptions loaderOptions = buildLoaderOptions();
305-
org.yaml.snakeyaml.Yaml yaml = new org.yaml.snakeyaml.Yaml(constructor, new Representer(new DumperOptions()), new DumperOptions(), loaderOptions, new CustomResolver());
306-
return yaml;
304+
return new org.yaml.snakeyaml.Yaml(constructor, new Representer(new DumperOptions()), new DumperOptions(), loaderOptions, new CustomResolver());
307305
} catch (Exception e) {
308306
//
309307
LOGGER.error("error building snakeYaml", e);
@@ -479,8 +477,8 @@ public Object getSingleData(Class<?> type) {
479477
throw new SnakeException("StackOverflow safe-checking yaml content (maxDepth " + options.getMaxYamlDepth() + ")", e);
480478
} catch (DuplicateKeyException e) {
481479
throw new SnakeException(e.getProblem().replace("found duplicate key", "Duplicate field"));
482-
} catch (Throwable e) {
483-
throw new SnakeException("Exception safe-checking yaml content (maxDepth " + options.getMaxYamlDepth() + ", maxYamlAliasesForCollections " + options.getMaxYamlAliasesForCollections() + ")", e);
480+
} catch (Exception e) {
481+
throw new SnakeException(e.getMessage() + "; Max code points: " + options.getMaxYamlCodePoints(), e);
484482
}
485483
}
486484
}

‎modules/swagger-parser-v3/src/test/java/io/swagger/v3/parser/OpenAPIV3ParserLargeFilesTest.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package io.swagger.v3.parser;
2+
3+
import io.swagger.v3.oas.models.OpenAPI;
4+
import io.swagger.v3.parser.core.models.ParseOptions;
5+
import io.swagger.v3.parser.core.models.SwaggerParseResult;
6+
import org.testng.annotations.AfterClass;
7+
import org.testng.annotations.BeforeClass;
8+
import org.testng.annotations.Test;
9+
10+
import static org.testng.Assert.assertFalse;
11+
import static org.testng.Assert.assertNotNull;
12+
13+
public class OpenAPIV3ParserLargeFilesTest {
14+
15+
@BeforeClass
16+
public static void init() {
17+
System.setProperty("maxYamlCodePoints", "10000000");
18+
}
19+
20+
@AfterClass
21+
public void cleanUpAfterAllTests() {
22+
System.clearProperty("maxYamlCodePoints");
23+
}
24+
25+
@Test
26+
public void issue2059() {
27+
OpenAPIV3Parser openApiParser = new OpenAPIV3Parser();
28+
ParseOptions options = new ParseOptions();
29+
options.setResolve(true);
30+
SwaggerParseResult parseResult = openApiParser.readLocation("issue2059/largeFile.yaml", null, options);
31+
OpenAPI openAPI = parseResult.getOpenAPI();
32+
33+
assertNotNull(openAPI);
34+
assertFalse(parseResult.getMessages().stream().anyMatch(message -> message.contains("exceeds the limit: 3145728")));
35+
}
36+
}

‎modules/swagger-parser-v3/src/test/java/io/swagger/v3/parser/reference/ReferenceVisitorTest.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package io.swagger.v3.parser.reference;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import org.testng.annotations.Test;
5+
import org.yaml.snakeyaml.error.YAMLException;
6+
7+
import java.io.IOException;
8+
import java.io.InputStream;
9+
import java.nio.charset.StandardCharsets;
10+
11+
import static org.junit.Assert.assertFalse;
12+
import static org.testng.Assert.assertNotNull;
13+
14+
public class ReferenceVisitorTest {
15+
16+
@Test
17+
public void largeFileShouldBeParsedByJacksonLibraryWhenCodeLimitIsSet() throws Exception {
18+
System.setProperty("maxYamlCodePoints", "10000000");
19+
ReferenceVisitor visitor = new ReferenceVisitor(null, null, null, null);
20+
String resourceName = "/issue2059/largeFile.yaml";
21+
String content = readResourceAsString(resourceName);
22+
23+
JsonNode jsonNode = visitor.deserializeIntoTree(content);
24+
25+
assertNotNull(content);
26+
assertFalse(content.isEmpty());
27+
assertNotNull(jsonNode);
28+
System.clearProperty("maxYamlCodePoints");
29+
}
30+
31+
@Test
32+
public void largeFileShouldNotBeParsedByJacksonLibraryWhenCodeLimitIsNotSet() throws Exception {
33+
ReferenceVisitor visitor = new ReferenceVisitor(null, null, null, null);
34+
String resourceName = "/issue2059/largeFile.yaml";
35+
String content = readResourceAsString(resourceName);
36+
37+
try {
38+
visitor.deserializeIntoTree(content);
39+
// Fail if no exception is thrown
40+
org.testng.Assert.fail("Expected YAMLException was not thrown");
41+
} catch (YAMLException ex) {
42+
org.testng.Assert.assertTrue(
43+
ex.getMessage() != null &&
44+
ex.getMessage().contains("The incoming YAML document exceeds the limit: 3145728 code points."),
45+
"Unexpected YAMLException message: " + ex.getMessage()
46+
);
47+
}
48+
}
49+
50+
private String readResourceAsString(String resourceName) throws IOException {
51+
try (InputStream is = this.getClass().getResourceAsStream(resourceName)) {
52+
if (is == null) {
53+
throw new IOException("Resource not found: " + resourceName);
54+
}
55+
java.io.ByteArrayOutputStream buffer = new java.io.ByteArrayOutputStream();
56+
byte[] data = new byte[4096];
57+
int nRead;
58+
while ((nRead = is.read(data, 0, data.length)) != -1) {
59+
buffer.write(data, 0, nRead);
60+
}
61+
return new String(buffer.toByteArray(), StandardCharsets.UTF_8);
62+
}
63+
}
64+
}

0 commit comments

Comments
 (0)