Merge branch 'master' into ygree/snakeyaml-engine-migration

Author: ygree

.gitlab-ci.yml

@@ -377,8 +377,8 @@ muzzle-dep-report:
   needs: [ build_tests ]
   stage: tests
   variables:
-    KUBERNETES_MEMORY_REQUEST: 16Gi
-    KUBERNETES_MEMORY_LIMIT: 16Gi
+    KUBERNETES_MEMORY_REQUEST: 17Gi
+    KUBERNETES_MEMORY_LIMIT: 17Gi
     KUBERNETES_CPU_REQUEST: 10
     GRADLE_WORKERS: 4
     GRADLE_MEM: 3G
@@ -402,7 +402,7 @@ muzzle-dep-report:
       export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
       fi
     - *prepare_test_env
-    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M"
     - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
   after_script:
     - *restore_pretest_env

components/yaml/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-  id("me.champeau.jmh")
+  `java-library`
 }
 
 apply(from = "$rootDir/gradle/java.gradle")

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java

@@ -1,5 +1,7 @@
 package datadog.trace.bootstrap.instrumentation.decorator;
 
+import static datadog.context.Context.root;
+import static datadog.context.propagation.Propagators.defaultPropagator;
 import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS;
 import static datadog.trace.api.datastreams.DataStreamsContext.fromTags;
 import static datadog.trace.api.gateway.Events.EVENTS;
@@ -8,6 +10,7 @@
 import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR;
 
 import datadog.appsec.api.blocking.BlockingException;
+import datadog.context.Context;
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTags;
 import datadog.trace.api.function.TriConsumer;
@@ -124,6 +127,7 @@ protected AgentTracer.TracerAPI tracer() {
     return AgentTracer.get();
   }
 
+  /** Deprecated. Use {@link #extractContext(REQUEST_CARRIER)} instead. */
   public AgentSpanContext.Extracted extract(REQUEST_CARRIER carrier) {
     AgentPropagation.ContextVisitor<REQUEST_CARRIER> getter = getter();
     if (null == carrier || null == getter) {
@@ -132,7 +136,18 @@ public AgentSpanContext.Extracted extract(REQUEST_CARRIER carrier) {
     return extractContextAndGetSpanContext(carrier, getter);
   }
 
-  /** Deprecated. Use {@link #startSpan(String, Object, AgentSpanContext.Extracted)} instead. */
+  /**
+   * Will be renamed to #extract(REQUEST_CARRIER) when refactoring of instrumentation's is complete
+   */
+  public Context extractContext(REQUEST_CARRIER carrier) {
+    AgentPropagation.ContextVisitor<REQUEST_CARRIER> getter = getter();
+    if (null == carrier || null == getter) {
+      return root();
+    }
+    return defaultPropagator().extract(root(), carrier, getter);
+  }
+
+  /** Deprecated. Use {@link #startSpanFromContext(String, Object, Context)} instead. */
   @Deprecated
   public AgentSpan startSpan(REQUEST_CARRIER carrier, AgentSpanContext.Extracted context) {
     return startSpan("http-server", carrier, context);
@@ -155,6 +170,20 @@ public AgentSpan startSpan(
     return span;
   }
 
+  /**
+   * Will be renamed to #startSpan(String, REQUEST_CARRIER, Context) when refactoring of
+   * instrumentation's is complete
+   */
+  public AgentSpan startSpanFromContext(
+      String instrumentationName, REQUEST_CARRIER carrier, Context context) {
+    return startSpan(instrumentationName, carrier, getSpanContext(context));
+  }
+
+  public AgentSpanContext.Extracted getSpanContext(Context context) {
+    AgentSpan extractedSpan = AgentSpan.fromContext(context);
+    return extractedSpan == null ? null : (AgentSpanContext.Extracted) extractedSpan.context();
+  }
+
   public AgentSpan onRequest(
       final AgentSpan span,
       final CONNECTION connection,

dd-java-agent/appsec/src/main/java/com/datadog/appsec/event/data/ObjectIntrospection.java

@@ -1,6 +1,8 @@
 package com.datadog.appsec.event.data;
 
+import com.datadog.appsec.gateway.AppSecRequestContext;
 import datadog.trace.api.Platform;
+import datadog.trace.api.telemetry.WafMetricCollector;
 import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
@@ -16,6 +18,7 @@
 public final class ObjectIntrospection {
   private static final int MAX_DEPTH = 20;
   private static final int MAX_ELEMENTS = 256;
+  private static final int MAX_STRING_LENGTH = 4096;
   private static final Logger log = LoggerFactory.getLogger(ObjectIntrospection.class);
 
   private static final Method trySetAccessible;
@@ -60,20 +63,32 @@ private ObjectIntrospection() {}
    * <p>Certain instance fields are excluded. Right now, this includes metaClass fields in Groovy
    * objects and this$0 fields in inner classes.
    *
-   * <p>Only string values are preserved. Numbers or booleans are removed, since we do not expect
-   * rules to detect malicious payloads in these types. An exception to this are map keys, which are
-   * always converted to strings.
-   *
    * @param obj an arbitrary object
+   * @param requestContext the request context
    * @return the converted object
    */
-  public static Object convert(Object obj) {
-    return guardedConversion(obj, 0, new State());
+  public static Object convert(Object obj, AppSecRequestContext requestContext) {
+    State state = new State(requestContext);
+    Object converted = guardedConversion(obj, 0, state);
+    if (state.stringTooLong || state.listMapTooLarge || state.objectTooDeep) {
+      requestContext.setWafTruncated();
+      WafMetricCollector.get()
+          .wafInputTruncated(state.stringTooLong, state.listMapTooLarge, state.objectTooDeep);
+    }
+    return converted;
   }
 
   private static class State {
     int elemsLeft = MAX_ELEMENTS;
     int invalidKeyId;
+    boolean objectTooDeep = false;
+    boolean listMapTooLarge = false;
+    boolean stringTooLong = false;
+    AppSecRequestContext requestContext;
+
+    private State(AppSecRequestContext requestContext) {
+      this.requestContext = requestContext;
+    }
   }
 
   private static Object guardedConversion(Object obj, int depth, State state) {
@@ -94,31 +109,48 @@ private static String keyConversion(Object key, State state) {
       return "null";
     }
     if (key instanceof String) {
-      return (String) key;
+      return checkStringLength((String) key, state);
     }
     if (key instanceof Number
         || key instanceof Boolean
         || key instanceof Character
         || key instanceof CharSequence) {
-      return key.toString();
+      return checkStringLength(key.toString(), state);
     }
     return "invalid_key:" + (++state.invalidKeyId);
   }
 
   private static Object doConversion(Object obj, int depth, State state) {
+    if (obj == null) {
+      return null;
+    }
     state.elemsLeft--;
-    if (state.elemsLeft <= 0 || obj == null || depth > MAX_DEPTH) {
+    if (state.elemsLeft <= 0) {
+      state.listMapTooLarge = true;
+      return null;
+    }
+
+    if (depth > MAX_DEPTH) {
+      state.objectTooDeep = true;
       return null;
     }
 
-    // strings, booleans and numbers are preserved
-    if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) {
+    // booleans and numbers are preserved
+    if (obj instanceof Boolean || obj instanceof Number) {
       return obj;
     }
 
+    // strings are preserved, but we need to check the length
+    if (obj instanceof String) {
+      return checkStringLength((String) obj, state);
+    }
+
     // char sequences are transformed just in case they are not immutable,
+    if (obj instanceof CharSequence) {
+      return checkStringLength(obj.toString(), state);
+    }
     // single char sequences are transformed to strings for ddwaf compatibility.
-    if (obj instanceof CharSequence || obj instanceof Character) {
+    if (obj instanceof Character) {
       return obj.toString();
     }
 
@@ -147,6 +179,7 @@ private static Object doConversion(Object obj, int depth, State state) {
       }
       for (Object o : ((Iterable<?>) obj)) {
         if (state.elemsLeft <= 0) {
+          state.listMapTooLarge = true;
           break;
         }
         newList.add(guardedConversion(o, depth + 1, state));
@@ -178,6 +211,7 @@ private static Object doConversion(Object obj, int depth, State state) {
     for (Field[] fields : allFields) {
       for (Field f : fields) {
         if (state.elemsLeft <= 0) {
+          state.listMapTooLarge = true;
           break outer;
         }
         if (Modifier.isStatic(f.getModifiers())) {
@@ -239,4 +273,12 @@ private static boolean setAccessible(Field field) {
       return false;
     }
   }
+
+  private static String checkStringLength(final String str, final State state) {
+    if (str.length() > MAX_STRING_LENGTH) {
+      state.stringTooLong = true;
+      return str.substring(0, MAX_STRING_LENGTH);
+    }
+    return str;
+  }
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java

@@ -472,7 +472,7 @@ private Flow<Void> onGrpcServerRequestMessage(RequestContext ctx_, Object obj) {
       if (subInfo == null || subInfo.isEmpty()) {
         return NoopFlow.INSTANCE;
       }
-      Object convObj = ObjectIntrospection.convert(obj);
+      Object convObj = ObjectIntrospection.convert(obj, ctx);
       DataBundle bundle =
           new SingletonDataBundle<>(KnownAddresses.GRPC_SERVER_REQUEST_MESSAGE, convObj);
       try {
@@ -574,7 +574,7 @@ private Flow<Void> onRequestBodyProcessed(RequestContext ctx_, Object obj) {
       }
       DataBundle bundle =
           new SingletonDataBundle<>(
-              KnownAddresses.REQUEST_BODY_OBJECT, ObjectIntrospection.convert(obj));
+              KnownAddresses.REQUEST_BODY_OBJECT, ObjectIntrospection.convert(obj, ctx));
       try {
         GatewayContext gwCtx = new GatewayContext(false);
         return producerService.publishDataEvent(subInfo, ctx, bundle, gwCtx);