chore(appsec): add support for more http data (#3193)

# Goal
To run AppSec rules, we need to get some data from the incoming request. For instance, the parsed query string, the HTTP body or even the URL parameters from the framework. The goal of this PR is to provide a way to access data from instrumentations event when these are not available on spans.

# Main change

To reach this goal, `set_http_meta` is updated to allow more arguments related to the current HTTP request. These are stored in a new contextual API (which under the hood stores the data on the root span). The data is then retrieved on span finish in the AppSecSpanProcessor and run through ddwaf. 


## Future work
This PRs uses references to `tracer` and `span` to create a transaction/request store. This store is used to keep a state containing interesting data in term of AppSec. We use it at the end of the root span to run the in-app waf. In the future, this store will exist independently from `tracer` and/or `span` and the current state is a transition to a future mechanism.

# Other Changes
* We also need to collect a given list of header when an AppSec event is found
* We mark the origin of the span collection to AppSec when there is an AppSec event and the previous origin was unset
* We forward more request data to `set_http_meta` for Django, Flask and Pyramid
* Update rules to latest version

Author: vdeturckheim

ddtrace/appsec/CMakeLists.txt

@@ -4,7 +4,7 @@ include(ExternalProject)
 
 ExternalProject_Add(libddwaf
     GIT_REPOSITORY https://github.com/DataDog/libddwaf.git
-    GIT_TAG 5fe5f1b065bf9e7a1b58bfae2a1dcbb9f6ba4231
+    GIT_TAG 627d8958d72440eb020e55ecb8772503796a73af
     INSTALL_DIR ${CMAKE_SOURCE_DIR}
     CMAKE_ARGS
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}

ddtrace/appsec/_ddwaf.pyx

@@ -227,7 +227,7 @@ cdef class DDWaf(object):
         cdef ddwaf_object* rule_objects
         self._rules = _Wrapper(rules, max_objects=None)
         rule_objects = (<_Wrapper?>self._rules)._ptr;
-        self._handle = ddwaf_init(rule_objects, NULL)
+        self._handle = ddwaf_init(rule_objects, NULL, NULL)
         if <void *> self._handle == NULL:
             raise ValueError("invalid rules")
 

ddtrace/appsec/_libddwaf.pxd

@@ -27,6 +27,12 @@ cdef extern from "include/ddwaf.h":
         uint64_t nbEntries
         DDWAF_OBJ_TYPE type
 
+    ctypedef struct ddwaf_ruleset_info:
+        uint16_t loaded
+        uint16_t failed
+        ddwaf_object errors
+        const char *version
+
     ctypedef struct ddwaf_handle:
         pass
 
@@ -45,7 +51,7 @@ cdef extern from "include/ddwaf.h":
 
     ctypedef void (*ddwaf_object_free_fn)(ddwaf_object *object);
 
-    ddwaf_handle ddwaf_init(const ddwaf_object* rules, const ddwaf_config* config);
+    ddwaf_handle ddwaf_init(const ddwaf_object* rules, const ddwaf_config* config, ddwaf_ruleset_info *info);
     ddwaf_context ddwaf_context_init(const ddwaf_handle handle, ddwaf_object_free_fn obj_free);
     DDWAF_RET_CODE ddwaf_run(ddwaf_context context, ddwaf_object* data, ddwaf_result* result, uint64_t timeout);
     void ddwaf_context_destroy(ddwaf_context context);

ddtrace/appsec/processor.py

@@ -2,36 +2,91 @@
 import json
 import os
 import os.path
+from typing import Set
 from typing import TYPE_CHECKING
 
 import attr
 
-import ddtrace
 from ddtrace.appsec._ddwaf import DDWaf
 from ddtrace.constants import MANUAL_KEEP_KEY
+from ddtrace.constants import ORIGIN_KEY
+from ddtrace.contrib.trace_utils import _normalize_tag_name
 from ddtrace.ext import SpanTypes
+from ddtrace.internal import _context
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.processor import SpanProcessor
 
 
 if TYPE_CHECKING:
-    from ddtrace import Span
+    from typing import Dict
+
+    from ddtrace.span import Span
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_RULES = os.path.join(ROOT_DIR, "rules.json")
 
 log = get_logger(__name__)
 
 
+def _no_cookies(data):
+    # type: (Dict[str, str]) -> Dict[str, str]
+    return {key: value for key, value in data.items() if key.lower() not in ("cookie", "set-cookie")}
+
+
 def get_rules():
+    # type: () -> str
     return os.getenv("DD_APPSEC_RULES", default=DEFAULT_RULES)
 
 
+class _Addresses(object):
+    SERVER_REQUEST_BODY = "server.request.body"
+    SERVER_REQUEST_QUERY = "server.request.query"
+    SERVER_REQUEST_HEADERS_NO_COOKIES = "server.request.headers.no_cookies"
+    SERVER_REQUEST_URI_RAW = "server.request.uri.raw"
+    SERVER_REQUEST_METHOD = "server.request.method"
+    SERVER_REQUEST_PATH_PARAMS = "server.request.path_params"
+    SERVER_REQUEST_COOKIES = "server.request.cookies"
+    SERVER_RESPONSE_STATUS = "server.response.status"
+    SERVER_RESPONSE_HEADERS_NO_COOKIES = "server.response.headers.no_cookies"
+
+
+_COLLECTED_REQUEST_HEADERS = {
+    "accept",
+    "accept-encoding",
+    "accept-language",
+    "content-encoding",
+    "content-language",
+    "content-length",
+    "content-type",
+    "forwarded",
+    "forwarded-for",
+    "host",
+    "true-client-ip",
+    "user-agent",
+    "via",
+    "x-client-ip",
+    "x-cluster-client-ip",
+    "x-forwarded",
+    "x-forwarded-for",
+    "x-real-ip",
+}
+
+_COLLECTED_HEADER_PREFIX = "http.request.headers."
+
+
+def _set_headers(span, headers):
+    # type: (Span, Dict) -> None
+    for k in headers:
+        if k.lower() in _COLLECTED_REQUEST_HEADERS:
+            span._set_str_tag(_normalize_tag_name("request", k), headers[k])
+
+
 @attr.s(eq=False)
 class AppSecSpanProcessor(SpanProcessor):
 
     rules = attr.ib(type=str, factory=get_rules)
     _ddwaf = attr.ib(type=DDWaf, default=None)
+    _addresses_to_keep = attr.ib(type=Set[str], factory=set)
 
     @property
     def enabled(self):
@@ -67,26 +122,82 @@ def __attrs_post_init__(self):
                 # Partial of DDAS-0005-00
                 log.warning("[DDAS-0005-00] WAF initialization failed")
                 raise
+        for address in self._ddwaf.required_data:
+            self._mark_needed(address)
+        # we always need the request headers
+        self._mark_needed(_Addresses.SERVER_REQUEST_HEADERS_NO_COOKIES)
 
     def on_span_start(self, span):
         # type: (Span) -> None
         pass
 
+    def _mark_needed(self, address):
+        # type: (str) -> None
+        self._addresses_to_keep.add(address)
+
+    def _is_needed(self, address):
+        # type: (str) -> bool
+        return address in self._addresses_to_keep
+
     def on_span_finish(self, span):
         # type: (Span) -> None
         if span.span_type != SpanTypes.WEB:
             return
         span.set_metric("_dd.appsec.enabled", 1.0)
         span._set_str_tag("_dd.runtime_family", "python")
-        data = {
-            "server.request.uri.raw": span.get_tag(ddtrace.ext.http.URL),
-            "server.response.status": span.get_tag(ddtrace.ext.http.STATUS_CODE),
-        }
+
+        data = {}
+        if self._is_needed(_Addresses.SERVER_REQUEST_QUERY):
+            request_query = _context.get_item("http.request.query", span=span)
+            if request_query is not None:
+                data[_Addresses.SERVER_REQUEST_QUERY] = request_query
+
+        if self._is_needed(_Addresses.SERVER_REQUEST_HEADERS_NO_COOKIES):
+            request_headers = _context.get_item("http.request.headers", span=span)
+            if request_headers is not None:
+                data[_Addresses.SERVER_REQUEST_HEADERS_NO_COOKIES] = _no_cookies(request_headers)
+
+        if self._is_needed(_Addresses.SERVER_REQUEST_URI_RAW):
+            uri = _context.get_item("http.request.uri", span=span)
+            if uri is not None:
+                data[_Addresses.SERVER_REQUEST_URI_RAW] = uri
+
+        if self._is_needed(_Addresses.SERVER_REQUEST_METHOD):
+            request_method = _context.get_item("http.request.method", span=span)
+            if request_method is not None:
+                data[_Addresses.SERVER_REQUEST_METHOD] = request_method
+
+        if self._is_needed(_Addresses.SERVER_REQUEST_PATH_PARAMS):
+            path_params = _context.get_item("http.request.path_params", span=span)
+            if path_params is not None:
+                data[_Addresses.SERVER_REQUEST_PATH_PARAMS] = path_params
+
+        if self._is_needed(_Addresses.SERVER_REQUEST_COOKIES):
+            cookies = _context.get_item("http.request.cookies", span=span)
+            if cookies is not None:
+                data[_Addresses.SERVER_REQUEST_COOKIES] = cookies
+
+        if self._is_needed(_Addresses.SERVER_RESPONSE_STATUS):
+            status = _context.get_item("http.response.status", span=span)
+            if status is not None:
+                data[_Addresses.SERVER_RESPONSE_STATUS] = status
+
+        if self._is_needed(_Addresses.SERVER_RESPONSE_HEADERS_NO_COOKIES):
+            response_headers = _context.get_item("http.response.headers", span=span)
+            if response_headers is not None:
+                data[_Addresses.SERVER_RESPONSE_HEADERS_NO_COOKIES] = _no_cookies(response_headers)
+
         log.debug("[DDAS-001-00] Executing AppSec In-App WAF with parameters: %s", data)
         res = self._ddwaf.run(data)  # res is a serialized json
         if res is not None:
+            if _Addresses.SERVER_REQUEST_HEADERS_NO_COOKIES in data:
+                _set_headers(span, data[_Addresses.SERVER_REQUEST_HEADERS_NO_COOKIES])
             # Partial DDAS-011-00
             log.debug("[DDAS-011-00] AppSec In-App WAF returned: %s", res)
             span._set_str_tag("appsec.event", "true")
             span._set_str_tag("_dd.appsec.json", '{"triggers":%s}' % (res,))
+            # Right now, we overwrite any value that could be already there. We need to reconsider when ASM/AppSec's
+            # specs are updated.
             span.set_tag(MANUAL_KEEP_KEY)
+            if span.get_tag(ORIGIN_KEY) is None:
+                span._set_str_tag(ORIGIN_KEY, "appsec")