Expand SSRF support in IAST to java.net.http.HttpClient (#7877)

Author: Mariovido

dd-java-agent/instrumentation/java-http-client/src/main/java11/datadog/trace/instrumentation/httpclient/JavaNetClientDecorator.java

@@ -36,6 +36,11 @@ protected URI url(HttpRequest httpRequest) throws URISyntaxException {
     return httpRequest.uri();
   }
 
+  @Override
+  protected Object sourceUrl(final HttpRequest request) {
+    return request.uri();
+  }
+
   @Override
   protected int status(HttpResponse<?> httpResponse) {
     return httpResponse.statusCode();

dd-smoke-tests/iast-util/iast-util-11/build.gradle

@@ -0,0 +1,35 @@
+plugins {
+  id 'idea'
+  id 'java-test-fixtures'
+}
+
+
+apply from: "$rootDir/gradle/java.gradle"
+
+description = 'iast-smoke-tests-utils-java-11'
+
+idea {
+  module {
+    jdkName = '11'
+  }
+}
+
+dependencies {
+  api project(':dd-smoke-tests')
+  compileOnly group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.2.0.RELEASE'
+
+  testFixturesImplementation testFixtures(project(":dd-smoke-tests:iast-util"))
+}
+
+project.tasks.withType(AbstractCompile).configureEach {
+  setJavaVersion(it, 11)
+  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_11
+  if (it instanceof JavaCompile) {
+    it.options.release.set(11)
+  }
+}
+
+forbiddenApisMain {
+  failOnMissingClasses = false
+}

dd-smoke-tests/iast-util/iast-util-11/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java

@@ -0,0 +1,40 @@
+package datadog.smoketest.springboot.controller;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/ssrf")
+public class SsrfController {
+
+  @PostMapping("/java-net")
+  public String javaNet(
+      @RequestParam(value = "url", required = false) final String url,
+      @RequestParam(value = "async", required = false) final boolean async,
+      @RequestParam(value = "promise", required = false) final boolean promise) {
+    HttpClient httpClient = HttpClient.newBuilder().build();
+    try {
+      HttpRequest httpRequest = HttpRequest.newBuilder().uri(new URI(url)).build();
+      if (async) {
+        if (promise) {
+          httpClient.sendAsync(
+              httpRequest,
+              HttpResponse.BodyHandlers.ofString(),
+              (initiatingRequest, pushPromiseRequest, acceptor) -> {});
+        } else {
+          httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString());
+        }
+      } else {
+        httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
+      }
+    } catch (Exception e) {
+    }
+    return "ok";
+  }
+}

dd-smoke-tests/iast-util/iast-util-11/src/testFixtures/groovy/datadog/smoketest/AbstractIast11SpringBootTest.groovy

@@ -0,0 +1,68 @@
+package datadog.smoketest
+
+import okhttp3.FormBody
+import okhttp3.Request
+
+import static datadog.trace.api.config.IastConfig.IAST_DEBUG_ENABLED
+import static datadog.trace.api.config.IastConfig.IAST_DETECTION_MODE
+import static datadog.trace.api.config.IastConfig.IAST_ENABLED
+
+abstract class AbstractIast11SpringBootTest extends AbstractIastServerSmokeTest {
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    String springBootShadowJar = System.getProperty('datadog.smoketest.springboot.shadowJar.path')
+
+    List<String> command = []
+    command.add(javaPath())
+    command.addAll(defaultJavaProperties)
+    command.addAll(iastJvmOpts())
+    command.addAll((String[]) ['-jar', springBootShadowJar, "--server.port=${httpPort}"])
+    ProcessBuilder processBuilder = new ProcessBuilder(command)
+    processBuilder.directory(new File(buildDirectory))
+    // Spring will print all environment variables to the log, which may pollute it and affect log assertions.
+    processBuilder.environment().clear()
+    return processBuilder
+  }
+
+  protected List<String> iastJvmOpts() {
+    return [
+      withSystemProperty(IAST_ENABLED, true),
+      withSystemProperty(IAST_DETECTION_MODE, 'FULL'),
+      withSystemProperty(IAST_DEBUG_ENABLED, true),
+    ]
+  }
+
+  void 'ssrf is present (#path)'() {
+    setup:
+    final url = "http://localhost:${httpPort}/ssrf/${path}"
+    final body = new FormBody.Builder()
+    .add(parameter, value)
+    .add("async", async)
+    .add("promise", promise).build()
+    final request = new Request.Builder().url(url).post(body).build()
+
+    when:
+    client.newCall(request).execute()
+
+    then:
+    hasVulnerability { vul ->
+      if (vul.type != 'SSRF') {
+        return false
+      }
+      final parts = vul.evidence.valueParts
+      if (parameter == 'url') {
+        return parts.size() == 1
+        && parts[0].value == value && parts[0].source.origin == 'http.request.parameter' && parts[0].source.name == parameter
+      } else {
+        throw new IllegalArgumentException("Parameter $parameter not supported")
+      }
+    }
+
+    where:
+    path       | parameter | value                     | async   | promise
+    "java-net" | "url"     | "https://dd.datad0g.com/" | "false" | "false"
+    "java-net" | "url"     | "https://dd.datad0g.com/" | "true"  | "false"
+    "java-net" | "url"     | "https://dd.datad0g.com/" | "true"  | "true"
+  }
+}

dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastServerSmokeTest.groovy

@@ -119,6 +119,7 @@ abstract class AbstractIastServerSmokeTest extends AbstractServerSmokeTest {
         }
         final vulnerabilities = parseVulnerabilities(json)
         found.addAll(vulnerabilities)
+        return vulnerabilities.find(matcher) != null
       }
     } catch (SpockTimeoutError toe) {
       throw new AssertionError("No matching vulnerability found. Vulnerabilities found: ${new JsonBuilder(found).toPrettyString()}")

dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastSpringBootTest.groovy

@@ -753,12 +753,12 @@ abstract class AbstractIastSpringBootTest extends AbstractIastServerSmokeTest {
     }
 
     where:
-    path                  | parameter | value                     | method               | protocolSecure
-    "apache-httpclient4"  | "url"     | "https://dd.datad0g.com/" | "apacheHttpClient4"  | false
-    "apache-httpclient4"  | "host"    | "dd.datad0g.com"          | "apacheHttpClient4"  | false
-    "commons-httpclient2" | "url"     | "https://dd.datad0g.com/" | "commonsHttpClient2" | false
-    "okHttp2"             | "url"     | "https://dd.datad0g.com/" | "okHttp2"            | false
-    "okHttp3"             | "url"     | "https://dd.datad0g.com/" | "okHttp3"            | false
+    path                  | parameter | value                     | protocolSecure
+    "apache-httpclient4"  | "url"     | "https://dd.datad0g.com/" | true
+    "apache-httpclient4"  | "host"    | "dd.datad0g.com"          | false
+    "commons-httpclient2" | "url"     | "https://dd.datad0g.com/" | true
+    "okHttp2"             | "url"     | "https://dd.datad0g.com/" | true
+    "okHttp3"             | "url"     | "https://dd.datad0g.com/" | true
   }
 
   void 'test iast metrics stored in spans'() {

dd-smoke-tests/springboot-java-11/build.gradle

@@ -0,0 +1,45 @@
+plugins {
+  id 'java'
+  id 'org.springframework.boot' version '2.7.15'
+  id 'io.spring.dependency-management' version '1.0.15.RELEASE'
+  id 'java-test-fixtures'
+}
+
+ext {
+  minJavaVersionForTests = JavaVersion.VERSION_11
+}
+
+apply from: "$rootDir/gradle/java.gradle"
+description = 'SpringBoot Java 11 Smoke Tests.'
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.2.0.RELEASE'
+
+  testImplementation project(':dd-smoke-tests')
+  testImplementation testFixtures(project(":dd-smoke-tests:iast-util:iast-util-11"))
+  testImplementation testFixtures(project(':dd-smoke-tests:iast-util'))
+
+  implementation project(':dd-smoke-tests:iast-util:iast-util-11')
+}
+
+project.tasks.withType(AbstractCompile).configureEach {
+  setJavaVersion(it, 11)
+  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_11
+  if (it instanceof JavaCompile) {
+    it.options.release.set(11)
+  }
+}
+
+forbiddenApisMain {
+  failOnMissingClasses = false
+}
+
+tasks.withType(Test).configureEach {
+  dependsOn "bootJar"
+  jvmArgs "-Ddatadog.smoketest.springboot.shadowJar.path=${tasks.bootJar.archiveFile.get()}"
+}

dd-smoke-tests/springboot-java-11/src/main/java/datadog/smoketest/springboot/SpringbootApplication.java

@@ -0,0 +1,14 @@
+package datadog.smoketest.springboot;
+
+import java.lang.management.ManagementFactory;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class SpringbootApplication {
+
+  public static void main(final String[] args) {
+    SpringApplication.run(SpringbootApplication.class, args);
+    System.out.println("Started in " + ManagementFactory.getRuntimeMXBean().getUptime() + "ms");
+  }
+}

dd-smoke-tests/springboot-java-11/src/test/groovy/datadog/smoketest/springboot/IastSpringBootSmokeTest.groovy

@@ -0,0 +1,6 @@
+package datadog.smoketest.springboot
+
+import datadog.smoketest.AbstractIast11SpringBootTest
+
+class IastSpringBootSmokeTest extends AbstractIast11SpringBootTest {
+}

settings.gradle

@@ -139,6 +139,7 @@ include ':dd-smoke-tests:spring-security'
 include ':dd-smoke-tests:springboot'
 include ':dd-smoke-tests:springboot-freemarker'
 include ':dd-smoke-tests:springboot-grpc'
+include ':dd-smoke-tests:springboot-java-11'
 include ':dd-smoke-tests:springboot-jetty-jsp'
 include ':dd-smoke-tests:springboot-mongo'
 include ':dd-smoke-tests:springboot-openliberty-20'
@@ -162,6 +163,7 @@ include ':dd-smoke-tests:debugger-integration-tests'
 include ':dd-smoke-tests:datastreams:kafkaschemaregistry'
 include ':dd-smoke-tests:iast-propagation'
 include ':dd-smoke-tests:iast-util'
+include ':dd-smoke-tests:iast-util:iast-util-11'
 // TODO this fails too often with a jgit failure, so disable until fixed
 //include ':dd-smoke-tests:debugger-integration-tests:latest-jdk-app'
 
@@ -500,4 +502,3 @@ include ':dd-java-agent:benchmark'
 include ':dd-java-agent:benchmark-integration'
 include ':dd-java-agent:benchmark-integration:jetty-perftest'
 include ':dd-java-agent:benchmark-integration:play-perftest'
-