Skip to content

Commit a2d6de5

Browse files
sezen-datadogsezen-datadog
committed
send to RC and telemetry logs
1 parent 9c88d82 commit a2d6de5

File tree

3 files changed

+147
-17
lines changed

3 files changed

+147
-17
lines changed

‎dd-java-agent/appsec/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ apply from: "$rootDir/gradle/java.gradle"
1111
apply from: "$rootDir/gradle/version.gradle"
1212

1313
dependencies {
14+
api project(':remote-config:remote-config-core')
1415
api libs.slf4j
1516
implementation project(':internal-api')
1617
implementation project(':communication')

‎dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java‎

Lines changed: 145 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_TRUSTED_IPS;
2222
import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_USER_BLOCKING;
2323
import static datadog.remoteconfig.Capabilities.CAPABILITY_ENDPOINT_FINGERPRINT;
24+
import static datadog.trace.logging.LogLevel.ERROR;
2425

2526
import com.datadog.appsec.AppSecModule;
2627
import com.datadog.appsec.AppSecSystem;
@@ -47,6 +48,7 @@
4748
import datadog.trace.api.Config;
4849
import datadog.trace.api.ProductActivation;
4950
import datadog.trace.api.UserIdCollectionMode;
51+
import datadog.trace.api.telemetry.LogCollector;
5052
import java.io.ByteArrayInputStream;
5153
import java.io.FileInputStream;
5254
import java.io.FileNotFoundException;
@@ -60,8 +62,6 @@
6062
import java.util.Map;
6163
import java.util.Set;
6264
import java.util.concurrent.ConcurrentHashMap;
63-
64-
import datadog.trace.api.telemetry.LogCollector;
6565
import okio.Okio;
6666
import org.slf4j.Logger;
6767
import org.slf4j.LoggerFactory;
@@ -101,6 +101,10 @@ public class AppSecConfigServiceImpl implements AppSecConfigService {
101101
private final String DEFAULT_WAF_CONFIG_RULE = "DEFAULT_WAF_CONFIG";
102102
private String currentRuleVersion;
103103
private List<AppSecModule> modulesToUpdateVersionIn;
104+
private final LogCollector telemetryLogger = LogCollector.get();
105+
106+
Moshi moshi = new Moshi.Builder().build();
107+
JsonAdapter<Map> mapToJsonAdapter;
104108

105109
public AppSecConfigServiceImpl(
106110
Config tracerConfig,
@@ -113,6 +117,7 @@ public AppSecConfigServiceImpl(
113117
if (tracerConfig.isAppSecWafMetrics()) {
114118
traceSegmentPostProcessors.add(statsReporter);
115119
}
120+
mapToJsonAdapter = moshi.adapter(Map.class);
116121
}
117122

118123
private void subscribeConfigurationPoller() {
@@ -156,11 +161,11 @@ private void subscribeConfigurationPoller() {
156161
}
157162

158163
private void subscribeRulesAndData() {
159-
this.configurationPoller.addListener(Product.ASM_DD, new AsmDDTypedListener());
164+
this.configurationPoller.addListener(Product.ASM_DD, new AsmDDTypedListener(Product.ASM_DD));
160165
this.configurationPoller.addListener(
161-
Product.ASM_DATA, new AppSecConfigConfigurationChangesTypedListener());
166+
Product.ASM_DATA, new AppSecConfigConfigurationChangesTypedListener(Product.ASM_DATA));
162167
this.configurationPoller.addListener(
163-
Product.ASM, new AppSecConfigConfigurationChangesTypedListener());
168+
Product.ASM, new AppSecConfigConfigurationChangesTypedListener(Product.ASM));
164169
}
165170

166171
public void modulesToUpdateVersionIn(List<AppSecModule> modules) {
@@ -172,6 +177,12 @@ public String getCurrentRuleVersion() {
172177
}
173178

174179
private class AppSecConfigConfigurationChangesTypedListener implements ProductListener {
180+
private Product productType;
181+
182+
public AppSecConfigConfigurationChangesTypedListener(Product product) {
183+
this.productType = product;
184+
}
185+
175186
@Override
176187
public void accept(ConfigKey configKey, byte[] content, PollingRateHinter pollingRateHinter)
177188
throws IOException {
@@ -189,7 +200,7 @@ public void accept(ConfigKey configKey, byte[] content, PollingRateHinter pollin
189200
Map<String, Object> contentMap =
190201
ADAPTER.fromJson(Okio.buffer(Okio.source(new ByteArrayInputStream(content))));
191202
try {
192-
handleWafUpdateResultReport(configKey.toString(), contentMap);
203+
handleWafUpdateResultReport(configKey.toString(), contentMap, productType);
193204
} catch (AppSecModule.AppSecModuleActivationException e) {
194205
throw new RuntimeException(e);
195206
}
@@ -209,6 +220,10 @@ public void commit(PollingRateHinter pollingRateHinter) {
209220
}
210221

211222
private class AsmDDTypedListener extends AppSecConfigConfigurationChangesTypedListener {
223+
public AsmDDTypedListener(Product product) {
224+
super(product);
225+
}
226+
212227
@Override
213228
public void accept(ConfigKey configKey, byte[] content, PollingRateHinter pollingRateHinter)
214229
throws IOException {
@@ -222,7 +237,11 @@ public void accept(ConfigKey configKey, byte[] content, PollingRateHinter pollin
222237
defaultConfigActivated = false;
223238
}
224239
super.accept(configKey, content, pollingRateHinter);
225-
usedDDWafConfigKeys.add(configKey.toString());
240+
if (content == null) {
241+
usedDDWafConfigKeys.remove(configKey.toString());
242+
} else {
243+
usedDDWafConfigKeys.add(configKey.toString());
244+
}
226245
}
227246

228247
@Override
@@ -233,7 +252,8 @@ public void remove(ConfigKey configKey, PollingRateHinter pollingRateHinter)
233252
}
234253
}
235254

236-
private void handleWafUpdateResultReport(String configKey, Map<String, Object> rawConfig)
255+
private void handleWafUpdateResultReport(
256+
String configKey, Map<String, Object> rawConfig, Product productType)
237257
throws AppSecModule.AppSecModuleActivationException {
238258
wafBuilder = getWafBuilder();
239259
if (modulesToUpdateVersionIn != null
@@ -247,9 +267,6 @@ private void handleWafUpdateResultReport(String configKey, Map<String, Object> r
247267
StandardizedLogging.numLoadedRules(log, configKey, countRules(rawConfig));
248268
}
249269

250-
// TODO: Send diagnostics via telemetry
251-
final LogCollector telemetryLogger = LogCollector.get();
252-
253270
initReporter.setReportForPublication(wafDiagnostics);
254271
if (wafDiagnostics.rulesetVersion != null
255272
&& !wafDiagnostics.rulesetVersion.isEmpty()
@@ -261,13 +278,17 @@ private void handleWafUpdateResultReport(String configKey, Map<String, Object> r
261278
modulesToUpdateVersionIn.forEach(module -> module.setRuleVersion(currentRuleVersion));
262279
}
263280
}
281+
if (wafDiagnostics.getNumConfigError() > 0) {
282+
addTelemetryErrorLog(wafDiagnostics);
283+
}
264284
} catch (InvalidRuleSetException e) {
265285
log.debug(
266286
"Invalid rule during waf config update for config key {}: {}",
267287
configKey,
268288
e.wafDiagnostics);
269289

270-
// TODO: Propagate diagostics back to remote config apply_error
290+
addTelemetryErrorLog(e.wafDiagnostics);
291+
sendErrorToRemoteConfig(e.wafDiagnostics, productType);
271292

272293
initReporter.setReportForPublication(e.wafDiagnostics);
273294
throw new RuntimeException(e);
@@ -277,6 +298,117 @@ private void handleWafUpdateResultReport(String configKey, Map<String, Object> r
277298
}
278299
}
279300

301+
private void sendErrorToRemoteConfig(WafDiagnostics wafDiagnostics, Product productType) {
302+
if (wafDiagnostics.rules != null) {
303+
getRemoteConfigErrorLogFor("rules", wafDiagnostics.rules.getErrors(), productType);
304+
}
305+
if (wafDiagnostics.customRules != null) {
306+
getRemoteConfigErrorLogFor(
307+
"customRules", wafDiagnostics.customRules.getErrors(), productType);
308+
}
309+
if (wafDiagnostics.rulesData != null) {
310+
getRemoteConfigErrorLogFor("rulesData", wafDiagnostics.rulesData.getErrors(), productType);
311+
}
312+
if (wafDiagnostics.rulesOverride != null) {
313+
getRemoteConfigErrorLogFor(
314+
"rulesOverride", wafDiagnostics.rulesOverride.getErrors(), productType);
315+
}
316+
if (wafDiagnostics.exclusions != null) {
317+
getRemoteConfigErrorLogFor("exclusions", wafDiagnostics.exclusions.getErrors(), productType);
318+
}
319+
if (wafDiagnostics.exclusionData != null) {
320+
getRemoteConfigErrorLogFor(
321+
"exclusionData", wafDiagnostics.exclusionData.getErrors(), productType);
322+
}
323+
if (wafDiagnostics.actions != null) {
324+
getRemoteConfigErrorLogFor("actions", wafDiagnostics.actions.getErrors(), productType);
325+
}
326+
if (wafDiagnostics.processors != null) {
327+
getRemoteConfigErrorLogFor("processors", wafDiagnostics.processors.getErrors(), productType);
328+
}
329+
if (wafDiagnostics.scanners != null) {
330+
getRemoteConfigErrorLogFor("scanners", wafDiagnostics.scanners.getErrors(), productType);
331+
}
332+
}
333+
334+
private void getRemoteConfigErrorLogFor(
335+
String configType, Map<String, List<String>> errors, Product productType) {
336+
if (productType == null) {
337+
return; // no need for RC error upon initialization
338+
}
339+
340+
String error = "{";
341+
Set<String> messageKeySet = errors.keySet();
342+
for (String key : messageKeySet) {
343+
error = error + "\"message\": \"" + key + "\" : [";
344+
List<String> errorsPerKey = errors.get(key);
345+
for (int i = 0; i < errorsPerKey.size(); i++) {
346+
error = error + "\"" + errorsPerKey.get(i) + "\"";
347+
if (i < errorsPerKey.size() - 1) {
348+
error = error + ",";
349+
}
350+
}
351+
error = error + "],";
352+
353+
error = error + "\"level\": \"ERROR\",";
354+
355+
error =
356+
error
357+
+ "\"tags\" : {"
358+
+ "\"log_type\": \"rc::"
359+
+ productType.name().toLowerCase()
360+
+ "::diagnostic\""
361+
+ "\"appsec_config_key\": \""
362+
+ configType
363+
+ "\""
364+
+ "\"rc_config_id\": \"\"}";
365+
error = error + "},";
366+
}
367+
368+
error = error.substring(0, error.length() - 1); // remove last comma
369+
370+
throw new RuntimeException(error);
371+
}
372+
373+
private void addTelemetryErrorLog(WafDiagnostics wafDiagnostics) {
374+
if (wafDiagnostics.rules != null) {
375+
addTelemetryErrorLogFor("rules", wafDiagnostics.rules.getErrors());
376+
}
377+
if (wafDiagnostics.customRules != null) {
378+
addTelemetryErrorLogFor("customRules", wafDiagnostics.customRules.getErrors());
379+
}
380+
if (wafDiagnostics.rulesData != null) {
381+
addTelemetryErrorLogFor("rulesData", wafDiagnostics.rulesData.getErrors());
382+
}
383+
if (wafDiagnostics.rulesOverride != null) {
384+
addTelemetryErrorLogFor("rulesOverride", wafDiagnostics.rulesOverride.getErrors());
385+
}
386+
if (wafDiagnostics.exclusions != null) {
387+
addTelemetryErrorLogFor("exclusions", wafDiagnostics.exclusions.getErrors());
388+
}
389+
if (wafDiagnostics.exclusionData != null) {
390+
addTelemetryErrorLogFor("exclusionData", wafDiagnostics.exclusionData.getErrors());
391+
}
392+
if (wafDiagnostics.actions != null) {
393+
addTelemetryErrorLogFor("actions", wafDiagnostics.actions.getErrors());
394+
}
395+
if (wafDiagnostics.processors != null) {
396+
addTelemetryErrorLogFor("processors", wafDiagnostics.processors.getErrors());
397+
}
398+
if (wafDiagnostics.scanners != null) {
399+
addTelemetryErrorLogFor("scanners", wafDiagnostics.scanners.getErrors());
400+
}
401+
}
402+
403+
private void addTelemetryErrorLogFor(String section, Map<String, List<String>> errors) {
404+
if (!errors.isEmpty()) {
405+
String error =
406+
"{\"" + section + "\" : {\"errors\" :" + mapToJsonAdapter.toJson(errors) + "}}";
407+
telemetryLogger.addLogMessage(
408+
ERROR.name(), error, new AppSecModule.AppSecModuleActivationException(error));
409+
}
410+
}
411+
280412
private void subscribeAsmFeatures() {
281413
this.configurationPoller.addListener(
282414
Product.ASM_FEATURES,
@@ -348,10 +480,7 @@ public void init() {
348480
throw new IllegalStateException("Expected default waf config to be available");
349481
}
350482
try {
351-
handleWafUpdateResultReport(
352-
DEFAULT_WAF_CONFIG_RULE,
353-
wafConfig,
354-
defaultConfigActivated ? DEFAULT_CONFIG_LOCATION : tracerConfig.getAppSecRulesFile());
483+
handleWafUpdateResultReport(DEFAULT_WAF_CONFIG_RULE, wafConfig, null);
355484
} catch (AppSecModule.AppSecModuleActivationException e) {
356485
throw new RuntimeException(e);
357486
}

‎dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/config/AppSecConfigServiceImplSpecification.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
122122
when:
123123
appSecConfigService.init()
124124
then:
125-
2 * config.getAppSecRulesFile() >> (p as String)
125+
1 * config.getAppSecRulesFile() >> (p as String)
126126

127127
when:
128128
appSecConfigService.maybeSubscribeConfigPolling()

0 commit comments

Comments
 (0)