fix(jackson): increase max serialized string length default (#8053)

Author: RyanHolstien

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java

@@ -1,6 +1,7 @@
 package com.linkedin.datahub.graphql.resolvers;
 
 import com.datahub.authentication.Authentication;
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
@@ -29,6 +30,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static com.linkedin.metadata.Constants.*;
+
 
 public class ResolverUtils {
 
@@ -37,6 +40,10 @@ public class ResolverUtils {
         "_entityType"
     );
     private static final ObjectMapper MAPPER = new ObjectMapper();
+    static {
+        int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+        MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build());
+    }
 
     private static final Logger _logger = LoggerFactory.getLogger(ResolverUtils.class.getName());
 

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSQualificationStep.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.upgrade.common.steps;
 
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -17,6 +18,8 @@
 import java.util.function.Function;
 import lombok.RequiredArgsConstructor;
 
+import static com.linkedin.metadata.Constants.*;
+
 
 @RequiredArgsConstructor
 public class GMSQualificationStep implements UpgradeStep {
@@ -78,6 +81,10 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
         String responseString = convertStreamToString(response);
 
         ObjectMapper mapper = new ObjectMapper();
+        int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH,
+            MAX_JACKSON_STRING_SIZE));
+        mapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+            .maxStringLength(maxSize).build());
         JsonNode configJson = mapper.readTree(responseString);
         if (isEligible((ObjectNode) configJson)) {
           return new DefaultUpgradeStepResult(

entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java

@@ -1,5 +1,6 @@
 package com.linkedin.metadata.models.registry;
 
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.linkedin.data.schema.DataSchema;
@@ -31,6 +32,7 @@
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 
+import static com.linkedin.metadata.Constants.*;
 import static com.linkedin.metadata.models.registry.EntityRegistryUtils.*;
 
 
@@ -49,6 +51,10 @@ public class ConfigEntityRegistry implements EntityRegistry {
   private final Map<String, AspectSpec> _aspectNameToSpec;
 
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory());
+  static {
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build());
+  }
 
   public ConfigEntityRegistry(Pair<Path, Path> configFileClassPathPair) throws IOException {
     this(DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()), configFileClassPathPair.getFirst());

entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java

@@ -1,5 +1,6 @@
 package com.linkedin.metadata.models.registry;
 
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.linkedin.data.schema.DataSchema;
@@ -31,6 +32,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.maven.artifact.versioning.ComparableVersion;
 
+import static com.linkedin.metadata.Constants.*;
 import static com.linkedin.metadata.models.registry.EntityRegistryUtils.*;
 
 
@@ -51,6 +53,10 @@ public class PatchEntityRegistry implements EntityRegistry {
   private final String identifier;
 
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory());
+  static {
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build());
+  }
 
   @Override
   public String toString() {

entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/util/TemplateUtil.java

@@ -1,11 +1,14 @@
 package com.linkedin.metadata.models.registry.template.util;
 
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.fge.jsonpatch.Patch;
 import java.util.ArrayList;
 import java.util.List;
 
+import static com.linkedin.metadata.Constants.*;
+
 
 public class TemplateUtil {
 
@@ -14,9 +17,13 @@ private TemplateUtil() {
   }
 
   public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+  static {
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build());
+  }
 
   public static List<String> getPaths(Patch jsonPatch) {
-    JsonNode patchNode = new ObjectMapper().valueToTree(jsonPatch);
+    JsonNode patchNode = OBJECT_MAPPER.valueToTree(jsonPatch);
     List<String> paths = new ArrayList<>();
     patchNode.elements().forEachRemaining(node -> {
       paths.add(node.get("path").asText());

li-utils/src/main/java/com/linkedin/metadata/Constants.java

@@ -16,6 +16,12 @@ public class Constants {
   public static final Long ASPECT_LATEST_VERSION = 0L;
   public static final String UNKNOWN_DATA_PLATFORM = "urn:li:dataPlatform:unknown";
 
+  // !!!!!!! IMPORTANT !!!!!!!
+  // This effectively sets the max aspect size to 16 MB. Used in deserialization of messages. Without this the limit is
+  // whatever Jackson is defaulting to (5 MB currently).
+  public static final String MAX_JACKSON_STRING_SIZE = "16000000";
+  public static final String INGESTION_MAX_SERIALIZED_STRING_LENGTH = "INGESTION_MAX_SERIALIZED_STRING_LENGTH";
+
   /**
    * System Metadata
    */

metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitter.java

@@ -1,5 +1,6 @@
 package datahub.client.file;
 
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import java.io.BufferedWriter;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -25,14 +26,16 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import lombok.extern.slf4j.Slf4j;
 
+import static com.linkedin.metadata.Constants.*;
+
+
 @Slf4j
 public class FileEmitter implements Emitter {
 
   private final EventFormatter eventFormatter;
   private final FileEmitterConfig config;
-  private final ObjectMapper objectMapper = new ObjectMapper()
-      .setSerializationInclusion(JsonInclude.Include.NON_NULL);
-  private final JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
+  private final ObjectMapper objectMapper;
+  private final JacksonDataTemplateCodec dataTemplateCodec;
 
   private final BufferedWriter writer;
   private final Future<MetadataWriteResponse> cachedSuccessFuture;
@@ -46,6 +49,11 @@ public class FileEmitter implements Emitter {
    * @param config
    */
   public FileEmitter(FileEmitterConfig config) {
+    objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    objectMapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+        .maxStringLength(maxSize).build());
+    dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
 
     this.config = config;
     this.eventFormatter = this.config.getEventFormatter();

metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java

@@ -1,5 +1,6 @@
 package datahub.client.rest;
 
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -47,6 +48,8 @@
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 
+import static com.linkedin.metadata.Constants.*;
+
 
 @ThreadSafe
 @Slf4j
@@ -77,8 +80,8 @@ public class RestEmitter implements Emitter {
   private final String ingestOpenApiUrl;
   private final String configUrl;
 
-  private final ObjectMapper objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);
-  private final JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
+  private final ObjectMapper objectMapper;
+  private final JacksonDataTemplateCodec dataTemplateCodec;
   private final CloseableHttpAsyncClient httpClient;
   private final EventFormatter eventFormatter;
 
@@ -87,6 +90,12 @@ public class RestEmitter implements Emitter {
    * @param config
    */
   public RestEmitter(RestEmitterConfig config) {
+    objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    objectMapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+        .maxStringLength(maxSize).build());
+    dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
+
     this.config = config;
     // Override httpClient settings with RestEmitter configs if present
     if (config.getTimeoutSec() != null) {

metadata-integration/java/datahub-client/src/main/java/datahub/event/EventFormatter.java

@@ -1,6 +1,7 @@
 package datahub.event;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.ByteString;
@@ -13,19 +14,26 @@
 import java.nio.charset.StandardCharsets;
 import lombok.SneakyThrows;
 
+import static com.linkedin.metadata.Constants.*;
+
 
 /**
  * A class that helps to format Metadata events for transport
  */
 public class EventFormatter {
 
-  private final ObjectMapper objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);
+  private final ObjectMapper objectMapper;
 
-  private final JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
+  private final JacksonDataTemplateCodec dataTemplateCodec;
   private final Format serializationFormat;
 
   public EventFormatter(Format serializationFormat) {
     this.serializationFormat = serializationFormat;
+    objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    objectMapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+        .maxStringLength(maxSize).build());
+    dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
   }
 
   public EventFormatter() {

metadata-integration/java/datahub-client/src/test/java/datahub/client/file/FileEmitterTest.java

@@ -1,5 +1,7 @@
 package datahub.client.file;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.StreamReadConstraints;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -27,9 +29,20 @@
 import datahub.client.MetadataWriteResponse;
 import datahub.event.MetadataChangeProposalWrapper;
 
+import static com.linkedin.metadata.Constants.*;
+
+
 public class FileEmitterTest {
-  private final ObjectMapper objectMapper = new ObjectMapper();
-  private final JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
+  private final ObjectMapper objectMapper;
+  private final JacksonDataTemplateCodec dataTemplateCodec;
+
+  public FileEmitterTest() {
+    objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE));
+    objectMapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+        .maxStringLength(maxSize).build());
+    dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory());
+  }
 
   @Rule
   public TemporaryFolder tempFolder = new TemporaryFolder();