Upgrade libddwaf java to 14.0.0

Author: sezen-datadog

dd-java-agent/appsec/build.gradle

@@ -15,7 +15,7 @@ dependencies {
   implementation project(':internal-api')
   implementation project(':communication')
   implementation project(':telemetry')
-  implementation group: 'io.sqreen', name: 'libsqreen', version: '13.0.1'
+  implementation group: 'io.sqreen', name: 'libsqreen', version: '14.0.0-SNAPSHOT'
   implementation libs.moshi
 
   testImplementation libs.bytebuddy

dd-java-agent/appsec/src/jmh/java/datadog/appsec/benchmark/WafBenchmark.java

@@ -7,8 +7,8 @@
 import com.datadog.appsec.config.AppSecConfigDeserializer;
 import com.datadog.appsec.event.data.KnownAddresses;
 import com.datadog.ddwaf.Waf;
+import com.datadog.ddwaf.WafBuilder;
 import com.datadog.ddwaf.WafContext;
-import com.datadog.ddwaf.WafHandle;
 import com.datadog.ddwaf.WafMetrics;
 import com.datadog.ddwaf.exception.AbstractWafException;
 import java.io.IOException;
@@ -44,14 +44,14 @@ public class WafBenchmark {
     BenchmarkUtil.initializeWaf();
   }
 
-  WafHandle ctx;
+  WafBuilder wafBuilder;
   Map<String, Object> wafData = new HashMap<>();
   Waf.Limits limits = new Waf.Limits(50, 500, 1000, 5000000, 5000000);
 
   @Benchmark
   public void withMetrics() throws Exception {
-    WafMetrics metricsCollector = ctx.createMetrics();
-    WafContext add = ctx.openContext();
+    WafMetrics metricsCollector = new WafMetrics();
+    WafContext add = new WafContext(wafBuilder);
     try {
       add.run(wafData, limits, metricsCollector);
     } finally {
@@ -61,7 +61,7 @@ public void withMetrics() throws Exception {
 
   @Benchmark
   public void withoutMetrics() throws Exception {
-    WafContext add = ctx.openContext();
+    WafContext add = new WafContext(wafBuilder);
     try {
       add.run(wafData, limits, null);
     } finally {
@@ -75,7 +75,8 @@ public void setUp() throws AbstractWafException, IOException {
     Map<String, AppSecConfig> cfg =
         Collections.singletonMap("waf", AppSecConfigDeserializer.INSTANCE.deserialize(stream));
     AppSecConfig waf = cfg.get("waf");
-    ctx = Waf.createHandle("waf", waf.getRawConfig());
+    wafBuilder = new WafBuilder();
+    wafBuilder.addOrUpdateConfig("waf", waf.getRawConfig());
 
     wafData.put(KnownAddresses.REQUEST_METHOD.getKey(), "POST");
     wafData.put(
@@ -112,6 +113,8 @@ public void setUp() throws AbstractWafException, IOException {
 
   @TearDown(Level.Trial)
   public void teardown() {
-    ctx.close();
+    if (wafBuilder != null && !wafBuilder.isOnline()) {
+      wafBuilder.destroy();
+    }
   }
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/AppSecModule.java

@@ -3,10 +3,12 @@
 import com.datadog.appsec.config.AppSecModuleConfigurer;
 import com.datadog.appsec.event.DataListener;
 import com.datadog.appsec.event.data.Address;
+import com.datadog.ddwaf.WafBuilder;
 import java.util.Collection;
 
 public interface AppSecModule {
-  void config(AppSecModuleConfigurer appSecConfigService) throws AppSecModuleActivationException;
+  void config(AppSecModuleConfigurer appSecConfigService, WafBuilder wafBuilder)
+      throws AppSecModuleActivationException;
 
   String getName();
 

dd-java-agent/appsec/src/main/java/com/datadog/appsec/AppSecSystem.java

@@ -7,11 +7,14 @@
 import com.datadog.appsec.config.AppSecConfigService;
 import com.datadog.appsec.config.AppSecConfigServiceImpl;
 import com.datadog.appsec.ddwaf.WAFModule;
+import com.datadog.appsec.ddwaf.WafInitialization;
 import com.datadog.appsec.event.EventDispatcher;
 import com.datadog.appsec.event.ReplaceableEventProducerService;
 import com.datadog.appsec.gateway.GatewayBridge;
 import com.datadog.appsec.util.AbortStartupException;
 import com.datadog.appsec.util.StandardizedLogging;
+import com.datadog.ddwaf.WafBuilder;
+import com.datadog.ddwaf.WafConfig;
 import datadog.appsec.api.blocking.Blocking;
 import datadog.appsec.api.blocking.BlockingService;
 import datadog.communication.ddagent.SharedCommunicationObjects;
@@ -43,6 +46,7 @@ public class AppSecSystem {
   private static ReplaceableEventProducerService REPLACEABLE_EVENT_PRODUCER; // testing
   private static Runnable STOP_SUBSCRIPTION_SERVICE;
   private static Runnable RESET_SUBSCRIPTION_SERVICE;
+  private static WafBuilder wafBuilder;
 
   public static void start(SubscriptionService gw, SharedCommunicationObjects sco) {
     try {
@@ -64,7 +68,10 @@ private static void doStart(SubscriptionService gw, SharedCommunicationObjects s
       return;
     }
     log.debug("AppSec is starting ({})", appSecEnabledConfig);
-
+    if (!WafInitialization.ONLINE) {
+      log.debug("In-app WAF initialization failed. See previous log entries");
+      return;
+    }
     REPLACEABLE_EVENT_PRODUCER = new ReplaceableEventProducerService();
     EventDispatcher eventDispatcher = new EventDispatcher();
     REPLACEABLE_EVENT_PRODUCER.replaceEventProducerService(eventDispatcher);
@@ -86,7 +93,8 @@ private static void doStart(SubscriptionService gw, SharedCommunicationObjects s
     APP_SEC_CONFIG_SERVICE =
         new AppSecConfigServiceImpl(
             config, configurationPoller, () -> reloadSubscriptions(REPLACEABLE_EVENT_PRODUCER));
-    APP_SEC_CONFIG_SERVICE.init();
+    wafBuilder = new WafBuilder(createWafConfig(config));
+    APP_SEC_CONFIG_SERVICE.init(wafBuilder);
 
     sco.createRemaining(config);
 
@@ -105,7 +113,7 @@ private static void doStart(SubscriptionService gw, SharedCommunicationObjects s
 
     setActive(appSecEnabledConfig == ProductActivation.FULLY_ENABLED);
 
-    APP_SEC_CONFIG_SERVICE.maybeSubscribeConfigPolling();
+    APP_SEC_CONFIG_SERVICE.maybeSubscribeConfigPolling(wafBuilder);
 
     Blocking.setBlockingService(new BlockingServiceImpl(REPLACEABLE_EVENT_PRODUCER));
 
@@ -143,8 +151,8 @@ public static void stop() {
       RESET_SUBSCRIPTION_SERVICE = null;
     }
     Blocking.setBlockingService(BlockingService.NOOP);
-
     APP_SEC_CONFIG_SERVICE.close();
+    wafBuilder.destroy();
   }
 
   private static void loadModules(EventDispatcher eventDispatcher, Monitoring monitoring) {
@@ -157,7 +165,7 @@ private static void loadModules(EventDispatcher eventDispatcher, Monitoring moni
       try {
         AppSecConfigService.TransactionalAppSecModuleConfigurer cfgObject;
         cfgObject = APP_SEC_CONFIG_SERVICE.createAppSecModuleConfigurer();
-        module.config(cfgObject);
+        module.config(cfgObject, wafBuilder);
         cfgObject.commit();
       } catch (RuntimeException | AppSecModule.AppSecModuleActivationException t) {
         log.error("Startup of appsec module {} failed", module.getName(), t);
@@ -209,4 +217,21 @@ public static Set<String> getStartedModulesInfo() {
       return Collections.emptySet();
     }
   }
+
+  private static WafConfig createWafConfig(Config config) {
+    WafConfig wafConfig = new WafConfig();
+    String keyRegexp = config.getAppSecObfuscationParameterKeyRegexp();
+    if (keyRegexp != null) {
+      wafConfig.obfuscatorKeyRegex = keyRegexp;
+    } else { // reset
+      wafConfig.obfuscatorKeyRegex = WafConfig.DEFAULT_KEY_REGEX;
+    }
+    String valueRegexp = config.getAppSecObfuscationParameterValueRegexp();
+    if (valueRegexp != null) {
+      wafConfig.obfuscatorValueRegex = valueRegexp;
+    } else { // reset
+      wafConfig.obfuscatorValueRegex = WafConfig.DEFAULT_VALUE_REGEX;
+    }
+    return wafConfig;
+  }
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigService.java

@@ -1,12 +1,13 @@
 package com.datadog.appsec.config;
 
+import com.datadog.ddwaf.WafBuilder;
 import java.io.Closeable;
 
 public interface AppSecConfigService extends Closeable {
-  void init();
-
   void close();
 
+  void init(WafBuilder wafBuilder);
+
   TransactionalAppSecModuleConfigurer createAppSecModuleConfigurer();
 
   interface TransactionalAppSecModuleConfigurer extends AppSecModuleConfigurer {

dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java

@@ -27,6 +27,8 @@
 import com.datadog.appsec.config.CurrentAppSecConfig.DirtyStatus;
 import com.datadog.appsec.util.AbortStartupException;
 import com.datadog.appsec.util.StandardizedLogging;
+import com.datadog.ddwaf.WafBuilder;
+import com.datadog.ddwaf.exception.InvalidRuleSetException;
 import datadog.remoteconfig.ConfigurationEndListener;
 import datadog.remoteconfig.ConfigurationPoller;
 import datadog.remoteconfig.Product;
@@ -75,6 +77,8 @@ public class AppSecConfigServiceImpl implements AppSecConfigService {
       this::applyRemoteConfigListener;
 
   private boolean hasUserWafConfig;
+  private boolean defaultConfigActivated;
+  private final String DEFAULT_WAF_CONFIG_RULE = "DEFAULT_WAF_CONFIG";
 
   public AppSecConfigServiceImpl(
       Config tracerConfig,
@@ -85,12 +89,12 @@ public AppSecConfigServiceImpl(
     this.reconfiguration = reconfig;
   }
 
-  private void subscribeConfigurationPoller() {
+  private void subscribeConfigurationPoller(WafBuilder wafBuilder) {
     // see also close() method
     subscribeAsmFeatures();
 
     if (!hasUserWafConfig) {
-      subscribeRulesAndData();
+      subscribeRulesAndData(wafBuilder);
     } else {
       log.debug("Will not subscribe to ASM, ASM_DD and ASM_DATA (AppSec custom rules in use)");
     }
@@ -125,7 +129,7 @@ private void subscribeConfigurationPoller() {
     this.configurationPoller.addCapabilities(capabilities);
   }
 
-  private void subscribeRulesAndData() {
+  private void subscribeRulesAndData(WafBuilder wafBuilder) {
     this.configurationPoller.addListener(
         Product.ASM_DD,
         AppSecConfigDeserializer.INSTANCE,
@@ -134,13 +138,20 @@ private void subscribeRulesAndData() {
           if (!initialized) {
             throw new IllegalStateException();
           }
-          if (newConfig == null) {
-            if (DEFAULT_WAF_CONFIG == null) {
-              throw new IllegalStateException("Expected default waf config to be available");
+          if (newConfig == null || newConfig.getRawConfig() == null) {
+            log.debug("AppSec config given by remote config was pulled. Removing WAF config");
+            wafBuilder.removeConfig(configKey);
+          } else {
+            if (defaultConfigActivated) {
+              log.debug("Removing default config");
+              wafBuilder.removeConfig(DEFAULT_WAF_CONFIG_RULE);
+              defaultConfigActivated = false;
+            }
+            try {
+              wafBuilder.addOrUpdateConfig(configKey, newConfig.getRawConfig());
+            } catch (InvalidRuleSetException e) {
+              throw new RuntimeException(e);
             }
-            log.debug(
-                "AppSec config given by remote config was pulled. Restoring default WAF config");
-            newConfig = DEFAULT_WAF_CONFIG;
           }
           this.currentAppSecConfig.setDdConfig(newConfig);
           // base rules can contain all rules/data/exclusions/etc
@@ -155,8 +166,14 @@ private void subscribeRulesAndData() {
           }
           if (newConfig == null) {
             currentAppSecConfig.mergedAsmData.removeConfig(configKey);
+            wafBuilder.removeConfig(configKey);
           } else {
             currentAppSecConfig.mergedAsmData.addConfig(configKey, newConfig);
+            try {
+              wafBuilder.addOrUpdateConfig(configKey, newConfig.getRawConfig());
+            } catch (InvalidRuleSetException e) {
+              throw new RuntimeException(e);
+            }
           }
           this.currentAppSecConfig.dirtyStatus.data = true;
         });
@@ -170,9 +187,15 @@ private void subscribeRulesAndData() {
           DirtyStatus dirtyStatus;
           if (newConfig == null) {
             dirtyStatus = currentAppSecConfig.userConfigs.removeConfig(configKey);
+            wafBuilder.removeConfig(configKey);
           } else {
             AppSecUserConfig userCfg = newConfig.build(configKey);
             dirtyStatus = currentAppSecConfig.userConfigs.addConfig(userCfg);
+            try {
+              wafBuilder.addOrUpdateConfig(configKey, newConfig.getRawConfig());
+            } catch (InvalidRuleSetException e) {
+              throw new RuntimeException(e);
+            }
           }
 
           this.currentAppSecConfig.dirtyStatus.mergeFrom(dirtyStatus);
@@ -218,7 +241,7 @@ private void distributeSubConfigurations(
   }
 
   @Override
-  public void init() {
+  public void init(WafBuilder wafBuilder) {
     AppSecConfig wafConfig;
     hasUserWafConfig = false;
     try {
@@ -242,17 +265,22 @@ public void init() {
     this.lastConfig.put("waf", this.currentAppSecConfig);
     this.mergedAsmFeatures = new MergedAsmFeatures();
     this.initialized = true;
+
+    if (DEFAULT_WAF_CONFIG == null) {
+      throw new IllegalStateException("Expected default waf config to be available");
+    }
+    defaultConfigActivated = true;
   }
 
-  public void maybeSubscribeConfigPolling() {
+  public void maybeSubscribeConfigPolling(WafBuilder wafBuilder) {
     if (this.configurationPoller != null) {
       if (hasUserWafConfig
           && tracerConfig.getAppSecActivation() == ProductActivation.FULLY_ENABLED) {
         log.info(
             "AppSec will not use remote config because "
                 + "there is a custom user configuration and AppSec is explicitly enabled");
       } else {
-        subscribeConfigurationPoller();
+        subscribeConfigurationPoller(wafBuilder);
       }
     } else {
       log.info("Remote config is disabled; AppSec will not be able to use it");

dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecData.java

@@ -12,6 +12,8 @@ public class AppSecData {
   @Json(name = "exclusion_data")
   private List<Map<String, Object>> exclusion;
 
+  private Map<String, Object> rawConfig;
+
   public List<Map<String, Object>> getRules() {
     return rules;
   }
@@ -27,4 +29,12 @@ public List<Map<String, Object>> getExclusion() {
   public void setExclusion(List<Map<String, Object>> exclusion) {
     this.exclusion = exclusion;
   }
+
+  public Map<String, Object> getRawConfig() {
+    return rawConfig;
+  }
+
+  public void setRawConfig(Map<String, Object> rawConfig) {
+    this.rawConfig = rawConfig;
+  }
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecDataDeserializer.java

@@ -7,6 +7,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Map;
 import okio.Okio;
 
 public class AppSecDataDeserializer implements ConfigurationDeserializer<AppSecData> {
@@ -18,10 +19,29 @@ private AppSecDataDeserializer() {}
 
   @Override
   public AppSecData deserialize(byte[] content) throws IOException {
+    if (content == null || content.length == 0) {
+      return null;
+    }
     return deserialize(new ByteArrayInputStream(content));
   }
 
+  @SuppressWarnings("unchecked")
   private AppSecData deserialize(InputStream is) throws IOException {
-    return ADAPTER.fromJson(Okio.buffer(Okio.source(is)));
+    AppSecData appSecData = ADAPTER.fromJson(Okio.buffer(Okio.source(is)));
+    is.reset();
+    if (appSecData != null && is.available() > 0) {
+      appSecData.setRawConfig(MOSHI.adapter(Map.class).fromJson(Okio.buffer(Okio.source(is))));
+      if (appSecData.getRawConfig().containsKey("rules_data")) {
+        appSecData.getRawConfig().put("rules", appSecData.getRawConfig().get("rules_data"));
+        appSecData.getRawConfig().remove("rules_data");
+      }
+      if (appSecData.getRawConfig().containsKey("exclusion_data")) {
+        appSecData
+            .getRawConfig()
+            .put("exclusions", appSecData.getRawConfig().get("exclusion_data"));
+        appSecData.getRawConfig().remove("exclusion_data");
+      }
+    }
+    return appSecData;
   }
 }