RFC 1076: fallback on http.endpoint for schema sampler

Author: cataphract

appsec/src/extension/ddtrace.c

@@ -59,6 +59,8 @@ static zend_string *(*_ddtrace_ip_extraction_find)(zval *server);
 
 static struct telemetry_rc_info (*_ddtrace_get_telemetry_rc_info)(void);
 static void *(*nullable _ddtrace_emit_asm_event)(void);
+static zend_string *(*nullable _ddtrace_guess_endpoint_from_url)(
+    const char *nonnull url, size_t url_len);
 
 static void _test_ddtrace_metric_register_buffer(
     zend_string *nonnull name, ddtrace_metric_type type, ddtrace_metric_ns ns);
@@ -115,6 +117,8 @@ static void dd_trace_load_symbols(zend_module_entry *module)
         ddtrace_metric_register_buffer, "ddtrace_metric_register_buffer");
     ASSIGN_DLSYM(ddtrace_metric_add_point, "ddtrace_metric_add_point");
     ASSIGN_DLSYM(_ddtrace_emit_asm_event, "ddtrace_emit_asm_event");
+    ASSIGN_DLSYM(
+        _ddtrace_guess_endpoint_from_url, "ddtrace_guess_endpoint_from_url");
 
     if (manually_opened) {
         dlclose(handle);
@@ -462,6 +466,16 @@ void dd_trace_emit_asm_event(void)
     _asm_event_emitted = true;
 }
 
+zend_string *dd_trace_guess_endpoint_from_url(
+    const char *nonnull url, size_t url_len)
+{
+    if (UNEXPECTED(_ddtrace_guess_endpoint_from_url == NULL)) {
+        return NULL;
+    }
+
+    return _ddtrace_guess_endpoint_from_url(url, url_len);
+}
+
 static PHP_FUNCTION(datadog_appsec_testing_ddtrace_rshutdown)
 {
     if (zend_parse_parameters_none() == FAILURE) {

appsec/src/extension/ddtrace.h

@@ -98,6 +98,9 @@ struct telemetry_rc_info {
 };
 struct telemetry_rc_info dd_trace_get_telemetry_rc_info(void);
 
+zend_string *nullable dd_trace_guess_endpoint_from_url(
+    const char *nonnull url, size_t url_len);
+
 typedef enum {
     DDTRACE_METRIC_TYPE_GAUGE,
     DDTRACE_METRIC_TYPE_COUNT,

appsec/src/extension/request_lifecycle.c

@@ -942,24 +942,67 @@ static uint64_t _calc_sampling_key(zend_object *root_span, int status_code)
         return 0;
     }
 
+    zend_string *route_or_endpoint = NULL;
+    bool free_route_or_endpoint = false;
+
     zval *route = zend_hash_str_find(Z_ARRVAL_P(meta), ZEND_STRL("http.route"));
-    if (!route || Z_TYPE_P(route) != IS_STRING) {
-        mlog_g(dd_log_debug, "No http.route tag; not sampling");
-        return 0;
+    const bool has_http_route = route && Z_TYPE_P(route) == IS_STRING;
+    if (has_http_route) {
+        route_or_endpoint = Z_STR_P(route);
+    } else {
+        // http.route is absent, check for http.endpoint
+        zval *endpoint =
+            zend_hash_str_find(Z_ARRVAL_P(meta), ZEND_STRL("http.endpoint"));
+        const bool has_http_endpoint =
+            endpoint && Z_TYPE_P(endpoint) == IS_STRING;
+        if (has_http_endpoint) {
+            // http.endpoint is already computed
+            // Use it unless status code is 404
+#define HTTP_NOT_FOUND 404
+            if (status_code == HTTP_NOT_FOUND) {
+                mlog_g(dd_log_debug,
+                    "http.endpoint present but status is 404; not sampling");
+            } else {
+                route_or_endpoint = Z_STR_P(endpoint);
+            }
+        } else {
+            // http.endpoint not computed, compute it now without setting the
+            // tag
+            zval *url =
+                zend_hash_str_find(Z_ARRVAL_P(meta), ZEND_STRL("http.url"));
+            const bool has_http_url = url && Z_TYPE_P(url) == IS_STRING;
+            if (has_http_url) {
+                route_or_endpoint = dd_trace_guess_endpoint_from_url(
+                    Z_STRVAL_P(url), Z_STRLEN_P(url));
+                if (route_or_endpoint) {
+                    free_route_or_endpoint = true;
+                } else {
+                    mlog_g(dd_log_debug,
+                        "Failed to compute http.endpoint; not sampling");
+                }
+            } else {
+                mlog_g(dd_log_debug,
+                    "No http.route, http.endpoint, or http.url; not sampling");
+            }
+        }
+    }
+
+    if (!route_or_endpoint) {
+        goto error;
     }
 
     zval *method =
         zend_hash_str_find(Z_ARRVAL_P(meta), ZEND_STRL("http.method"));
     if (!method || Z_TYPE_P(method) != IS_STRING) {
         mlog_g(dd_log_debug, "No http.method tag; not sampling");
-        return 0;
+        goto error;
     }
 
-    // use fnv-1a hash with: <http.route tag> NULL <http.method tag> NULL
+    // use fnv-1a hash with: <route_or_endpoint> NULL <http.method tag> NULL
     // status_code
 
     uint64_t hash = FNV_OFFSET_BASIS;
-    hash = _hash_zend_string(hash, Z_STR_P(route));
+    hash = _hash_zend_string(hash, route_or_endpoint);
     hash *= FNV_PRIME; // hash NUL byte
     hash = _hash_zend_string(hash, Z_STR_P(method));
     hash *= FNV_PRIME; // hash NUL byte
@@ -970,7 +1013,22 @@ static uint64_t _calc_sampling_key(zend_object *root_span, int status_code)
         snprintf(status_str, sizeof(status_str), "%d", status_code);
     hash = _hash_string(hash, status_str, status_len);
 
+    mlog_g(dd_log_debug,
+        "Will use sampling key: %" PRIu64
+        " for route/endpoint %.*s, method %.*s, status %d",
+        hash, (int)ZSTR_LEN(route_or_endpoint), ZSTR_VAL(route_or_endpoint),
+        (int)ZSTR_LEN(Z_STR_P(method)), ZSTR_VAL(Z_STR_P(method)), status_code);
+
+    if (free_route_or_endpoint) {
+        zend_string_release(route_or_endpoint);
+    }
     return hash;
+
+error:
+    if (free_route_or_endpoint) {
+        zend_string_release(route_or_endpoint);
+    }
+    return 0;
 }
 
 PHP_FUNCTION(datadog_appsec_testing_dump_req_lifecycle_state)

appsec/tests/integration/build.gradle

@@ -489,9 +489,11 @@ task saveCaches(type: Exec) {
 }
 
 if (hasProperty('buildScan')) {
-    buildScan {
-        termsOfServiceUrl = 'https://gradle.com/terms-of-service'
-        termsOfServiceAgree = 'yes'
+    develocity {
+        buildScan {
+            termsOfUseUrl = "https://gradle.com/help/legal-terms-of-use"
+            termsOfUseAgree = "yes"
+        }
     }
 }
 

appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy

@@ -19,7 +19,7 @@ import static org.testcontainers.containers.Container.ExecResult
 @Testcontainers
 @Slf4j
 @DisabledIf('isZts')
-class Apache2FpmTests implements CommonTests, SamplingTestsInFpm {
+class Apache2FpmTests implements CommonTests, SamplingTestsInFpm, EndpointFallbackSamplingTests {
     static boolean zts = variant.contains('zts')
 
     @Container

appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/EndpointFallbackSamplingTests.groovy

@@ -0,0 +1,106 @@
+package com.datadog.appsec.php.integration
+
+import com.datadog.appsec.php.docker.AppSecContainer
+import com.datadog.appsec.php.model.Trace
+import org.junit.jupiter.api.Test
+
+import java.net.http.HttpResponse
+
+trait EndpointFallbackSamplingTests {
+
+    /**
+     * Test Requirement 1: If http.route is present, use it for sampling
+     * Expected: Schema sampling should work using http.route
+     */
+    @Test
+    void 'sampling uses http route when present'() {
+        def trace = container.traceFromRequest('/endpoint_fallback.php?case=with_route') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 200
+        }
+        assert trace != null
+        assert trace.first().meta."http.route" == "/users/{id}/profile"
+        assert trace.first().meta."_dd.appsec.s.res.body" != null // we sampled
+
+        trace = container.traceFromRequest('/endpoint_fallback.php?case=with_route') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 200
+        }
+        assert trace != null
+        assert trace.first().meta."_dd.appsec.s.res.body" == null // we did not sample again
+    }
+
+    /**
+     * Test Requirement 2a: If http.route is absent and http.endpoint is present (non-404),
+     * use http.endpoint for sampling
+     * Expected: Schema sampling should work using http.endpoint
+     */
+    @Test
+    void 'sampling uses http endpoint when http route absent and status is not 404'() {
+        def trace = container.traceFromRequest('/endpoint_fallback.php?case=with_endpoint') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 200
+        }
+        assert trace != null
+
+        assert trace.first().meta."http.route" == null
+        assert trace.first().meta."http.endpoint" == "/api/products/{param:int}"
+        assert trace.first().meta."_dd.appsec.s.res.body" != null // sampling happened
+
+        trace = container.traceFromRequest('/endpoint_fallback.php?case=with_endpoint') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 200
+        }
+        assert trace != null
+        assert trace.first().meta."_dd.appsec.s.res.body" == null // no sampling again
+    }
+
+    /**
+     * Test Requirement 2b: If http.route is absent and http.endpoint is present but status is 404,
+     * should NOT sample (failsafe)
+     * Expected: No schema sampling should occur
+     */
+    @Test
+    void 'sampling does not use http endpoint when status is 404'() {
+        def trace = container.traceFromRequest('/endpoint_fallback.php?case=404') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 404
+        }
+        assert trace != null
+        assert trace.first().meta."http.route" == null
+        assert trace.first().meta."http.endpoint" == "/api/notfound/{param:int}"
+        assert trace.first().meta."_dd.appsec.s.res.body" == null // we did not sample
+    }
+
+    /**
+     * Test Requirement 3: If neither http.route nor http.endpoint is present,
+     * compute http.endpoint on-demand and use for sampling, but do NOT set it on the span
+     * Expected: Schema sampling should work, but http.endpoint should not be in meta
+     */
+    @Test
+    void 'sampling computes endpoint on-demand when neither route nor endpoint present'() {
+        def trace = container.traceFromRequest('/endpoint_fallback.php?case=computed') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 200
+        }
+        assert trace != null
+
+        assert trace.first().meta."http.url" != null
+        assert trace.first().meta."http.url".contains("/endpoint_fallback_computed/users/123/orders/456")
+        assert trace.first().meta."http.route" == null
+        assert trace.first().meta."http.endpoint" == null
+        assert trace.first().meta."_dd.appsec.s.res.body" != null // we did sample
+
+        trace = container.traceFromRequest('/endpoint_fallback.php?case=computed') {
+            HttpResponse<InputStream> resp ->
+                assert resp.statusCode() == 200
+        }
+        assert trace != null
+        assert trace.first().meta."_dd.appsec.s.res.body" == null
+        assert trace.first().meta."http.endpoint" == null
+    }
+
+    private AppSecContainer getContainer() {
+        getClass().container
+    }
+}

appsec/tests/integration/src/test/www/base/public/endpoint_fallback.php

@@ -0,0 +1,81 @@
+<?php
+$rootSpan = \DDTrace\root_span();
+$case = $_GET['case'] ?? 'unknown';
+
+switch ($case) {
+    case 'with_route':
+        // Test case: http.route is present - should use it for sampling
+        $rootSpan->meta["http.route"] = "/users/{id}/profile";
+        $rootSpan->meta["http.method"] = "GET";
+
+        header("Content-Type: application/json");
+        http_response_code(200);
+
+        echo json_encode([
+            "status" => "ok",
+            "test_case" => "with_route",
+            "messages" => ["test", "data"]
+        ]);
+        break;
+
+    case 'with_endpoint':
+        // Test case: http.route is absent, http.endpoint is present - should use http.endpoint for sampling
+        // Do NOT set http.route
+        $rootSpan->meta["http.endpoint"] = "/api/products/{param:int}";
+        $rootSpan->meta["http.method"] = "GET";
+
+        header("Content-Type: application/json");
+        http_response_code(200);
+
+        echo json_encode([
+            "status" => "ok",
+            "test_case" => "with_endpoint",
+            "messages" => ["test", "data"]
+        ]);
+        break;
+
+    case '404':
+        // Test case: http.route is absent, http.endpoint is present, but status is 404 - should NOT sample
+        // Do NOT set http.route
+        $rootSpan->meta["http.endpoint"] = "/api/notfound/{param:int}";
+        $rootSpan->meta["http.method"] = "GET";
+
+        header("Content-Type: application/json");
+        http_response_code(404);
+
+        echo json_encode([
+            "status" => "error",
+            "test_case" => "404_with_endpoint",
+            "message" => "Not found"
+        ]);
+        break;
+
+    case 'computed':
+        // Test case: Neither http.route nor http.endpoint set - should compute endpoint on-demand
+        // The endpoint should be computed but NOT added as a tag on the span
+        // Do NOT set http.route or http.endpoint
+        // Set http.url so endpoint can be computed
+        $rootSpan->meta["http.url"] = "http://localhost:8080/endpoint_fallback_computed/users/123/orders/456";
+        $rootSpan->meta["http.method"] = "GET";
+
+        header("Content-Type: application/json");
+        http_response_code(200);
+
+        echo json_encode([
+            "status" => "ok",
+            "test_case" => "computed_on_demand",
+            "messages" => ["test", "data"],
+        ]);
+        break;
+
+    default:
+        header("Content-Type: application/json");
+        http_response_code(400);
+
+        echo json_encode([
+            "status" => "error",
+            "test_case" => "unknown",
+            "message" => "Invalid case parameter. Valid values: with_route, with_endpoint, 404, computed"
+        ]);
+        break;
+}

ddtrace.sym

@@ -16,6 +16,7 @@ ddtrace_metric_add_point
 ddtrace_emit_asm_event
 ddtrace_loaded_by_ssi
 ddtrace_ssi_forced_injection_enabled
+ddtrace_guess_endpoint_from_url
 ddog_remote_config_reader_for_path
 ddog_remote_config_read
 ddog_remote_config_reader_drop

ext/endpoint_guessing.c

@@ -185,6 +185,11 @@ static zend_string* guess_endpoint(const char* orig_url, size_t orig_url_len) {
     return smart_str_extract(&result);
 }
 
+DDTRACE_PUBLIC zend_string* ddtrace_guess_endpoint_from_url(const char* url, size_t url_len)
+{
+    return guess_endpoint(url, url_len);
+}
+
 void ddtrace_maybe_add_guessed_endpoint_tag(ddtrace_root_span_data *span)
 {
     zval *span_type = &span->property_type;

ext/endpoint_guessing.h

@@ -1,5 +1,7 @@
 #pragma once
 
 #include "ddtrace.h"
+#include "ddtrace_export.h"
 
 void ddtrace_maybe_add_guessed_endpoint_tag(ddtrace_root_span_data *span);
+DDTRACE_PUBLIC zend_string* ddtrace_guess_endpoint_from_url(const char* url, size_t url_len);