Update libdatadog; fix two bugs

Author: cataphract

.gitlab/generate-appsec.php

@@ -145,6 +145,7 @@
   after_script:
     - mkdir -p "${CI_PROJECT_DIR}/artifacts"
     - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
+    - cp -r appsec/tests/integration/build/test-logs "${CI_PROJECT_DIR}/artifacts/" 2>/dev/null || true
     - .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec"
   artifacts:
     reports:

Cargo.lock

@@ -162,7 +162,9 @@ void service::metrics_impl::submit_log(const sidecar_settings &sc_settings,
         SPDLOG_INFO("Failed to enqueue telemetry log, error: {}",
             std::string_view{error_msg.ptr, error_msg.len});
         fn_ddog_Error_drop(&result.some);
-        return;
+    } else {
+        SPDLOG_DEBUG("Sent telemetry log via sidecar-ffi: {}: {}",
+            log.identifier, log.message);
     }
 }
 
@@ -189,6 +191,9 @@ void service::metrics_impl::register_metric_ffi(
         SPDLOG_INFO("Failed to register telemetry metric, error: {}",
             std::string_view{error_msg.ptr, error_msg.len});
         fn_ddog_Error_drop(&result.some);
+    } else {
+        SPDLOG_DEBUG(
+            "Sent telemetry metric via sidecar-ffi: {} of type {}", name, type);
     }
 }
 
@@ -222,14 +227,32 @@ void service::metrics_impl::submit_metric_ffi(
         SPDLOG_INFO("Failed to enqueue telemetry point, error: {}",
             std::string_view{error_msg.ptr, error_msg.len});
         fn_ddog_Error_drop(&result.some);
+    } else {
+        SPDLOG_DEBUG("Sent telemetry point via sidecar-ffi: {} of value {}",
+            name, value);
     }
 }
 
 void service::register_known_metrics(const sidecar_settings &sc_settings,
     const telemetry_settings &telemetry_settings)
 {
+    static constexpr std::chrono::minutes kRegisterInterval = 25min;
+    // register known metrics every 25 minutes
+    // libdatadog has a cleanup task that deletes TelemetryCachedClients
+    // if they are 30 minutes without being used
+    auto registered = metrics_registered_at_.load(std::memory_order_relaxed);
+    auto now = std::chrono::steady_clock::now();
+    if (registered + kRegisterInterval > now) {
+        return;
+    }
+    if (!metrics_registered_at_.compare_exchange_strong(
+            registered, now, std::memory_order_relaxed)) {
+        SPDLOG_DEBUG("Metrics concurrent registration attempt");
+        return;
+    }
+
     for (auto &&metric : metrics::known_metrics) {
-        msubmitter_->register_metric_ffi(
+        metrics_impl::register_metric_ffi(
             sc_settings, telemetry_settings, metric.name, metric.type);
     }
 }
@@ -241,7 +264,7 @@ void service::handle_worker_count_metrics(const sidecar_settings &sc_settings)
         return;
     }
 
-    msubmitter_->submit_metric_ffi(sc_settings, telemetry_settings_,
+    metrics_impl::submit_metric_ffi(sc_settings, telemetry_settings_,
         metrics::helper_worker_count, static_cast<double>(cur_st.count),
         std::nullopt);
 

appsec/src/helper/service.cpp

@@ -261,11 +261,7 @@ class service {
 
     void drain_metrics(const sidecar_settings &sc_settings)
     {
-        auto registered = metrics_registered_.load(std::memory_order_relaxed);
-        if (!registered && metrics_registered_.compare_exchange_strong(
-                               registered, true, std::memory_order_relaxed)) {
-            register_known_metrics(sc_settings, telemetry_settings_);
-        }
+        register_known_metrics(sc_settings, telemetry_settings_);
 
         msubmitter_->drain_metrics(sc_settings, telemetry_settings_);
     }
@@ -274,8 +270,7 @@ class service {
     {
         msubmitter_->drain_logs(sc_settings, telemetry_settings_);
 
-        // take this opportunity to submit internal worker count metrics too
-        // this could be done at any other time when sc_settings is available
+        register_known_metrics(sc_settings, telemetry_settings_);
         handle_worker_count_metrics(sc_settings);
     }
 
@@ -337,7 +332,8 @@ class service {
     std::optional<sampler> schema_sampler_;
     std::string rc_path_;
     telemetry_settings telemetry_settings_;
-    std::atomic<bool> metrics_registered_;
+    std::atomic<std::chrono::time_point<std::chrono::steady_clock>>
+        metrics_registered_at_;
     std::shared_ptr<metrics_impl> msubmitter_;
 
     struct num_workers_t {

appsec/src/helper/service.hpp

@@ -36,9 +36,10 @@ dependencies {
     implementation 'org.msgpack:msgpack-core:0.9.9'
     implementation 'io.javalin:javalin:6.7.0'
 
-    implementation platform('org.testcontainers:testcontainers-bom:1.21.3')
-    implementation 'org.testcontainers:mysql'
-    implementation "org.testcontainers:junit-jupiter"
+    implementation platform('org.testcontainers:testcontainers-bom:2.0.3')
+    implementation 'org.testcontainers:testcontainers-mysql'
+    implementation "org.testcontainers:testcontainers-junit-jupiter"
+    implementation 'org.hamcrest:hamcrest:2.2'
     implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.16'
     implementation 'org.junit.jupiter:junit-jupiter-engine:5.12.2'
     implementation 'org.junit.jupiter:junit-jupiter-params:5.12.2'
@@ -93,6 +94,11 @@ def creationDateOf(String image) {
     Date volumeDate = Date.from(OffsetDateTime.parse(imageModifiedStr).toInstant())
     volumeDate.time
 }
+def libddwafCommit() {
+    def proc = ['git', '-C', "${projectDir}/../../third_party/libddwaf", 'rev-parse', 'HEAD'].execute()
+    proc.waitForOrKill(5_000)
+    proc.text.trim()
+}
 
 def createVolumeTask = { String volumeName ->
     String taskName = "createVolume-$volumeName"
@@ -346,17 +352,17 @@ def buildAppSecTask = { String version, String variant ->
             ],
             command: [
                     '-e', '-c',
-                    '''
-                    git config --global --add safe.directory '*'
+                    """
                     cd /appsec
                     test -f CMakeCache.txt || \\
                         cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\
                               -DCMAKE_INSTALL_PREFIX=/appsec \\
                               -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\
+                              -DGIT_COMMIT=${libddwafCommit()} \\
                               -DDD_APPSEC_TESTING=ON /project/appsec
                     make -j extension ddappsec-helper && \\
                         touch ddappsec.so libddappsec-helper.so
-                '''
+                """
             ]
     )
 }