refactor(appsec): rename appsec submodules to make them private (#6861)

Refactors AppSec submodules to make them private:
 - ddtrace.appsec.ddwaf -> ddtrace.appsec._ddwaf
 - ddtrace.appsec.handlers -> ddtrace.appsec._handlers
 - ddtrace.appsec.iast -> ddtrace.appsec._iast
 - ddtrace.appsec.processor -> ddtrace.appsec._processor
 - ddtrace.appsec.trace_utils -> ddtrace.appsec._trace_utils
 - ddtrace.appsec.utils -> ddtrace.appsec._utils

Exports public functions as they were before:
 - ddtrace.appsec.iast:
    - ddtrace_iast_flask_patch
 - ddtrace.appsec.trace_utils:
    - block_request
    - block_request_if_user_blocked
    - should_block_user
    - track_custom_event
    - track_user_login_failure_event
    - track_user_login_success_event
    - track_user_signup_event

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [x] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [x] This PR doesn't touch any of that.

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Yun Kim <[email protected]>
Co-authored-by: Yun Kim <[email protected]>
Co-authored-by: Kyle Verhoog <[email protected]>
Co-authored-by: Gabriele N. Tornetta <[email protected]>
Co-authored-by: Alberto Vara <[email protected]>
Co-authored-by: Nir Rattner <[email protected]>
Co-authored-by: ZStriker19 <[email protected]>
Co-authored-by: Zachary Groves <[email protected]>
Co-authored-by: Munir Abdinur <[email protected]>
Co-authored-by: Emmett Butler <[email protected]>
Co-authored-by: Emmett Butler <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Christophe Papazian <[email protected]>
Co-authored-by: Teague Bick <[email protected]>
Co-authored-by: Tahir H. Butt <[email protected]>
Co-authored-by: Juanjo Alvarez Martinez <[email protected]>

Author: 14 people

.gitignore

@@ -25,7 +25,7 @@ ddtrace/internal/_tagset.c
 ddtrace/**/*.html
 
 # Dynamic binary libraries
-ddtrace/appsec/ddwaf/libddwaf/
+ddtrace/appsec/_ddwaf/libddwaf/
 libddwaf*.sha256
 ddtrace/internal/datadog/profiling/libdatadog/
 
@@ -138,9 +138,9 @@ ddtrace/_version.py
 artifacts/
 
 # IAST cpp
-ddtrace/appsec/iast/_taint_tracking/cmake-build-debug/*
-ddtrace/appsec/iast/_taint_tracking/_deps/*
-ddtrace/appsec/iast/_taint_tracking/CMakeFiles/*
+ddtrace/appsec/_iast/_taint_tracking/cmake-build-debug/*
+ddtrace/appsec/_iast/_taint_tracking/_deps/*
+ddtrace/appsec/_iast/_taint_tracking/CMakeFiles/*
 
 # CircleCI generated config
 .circleci/config.gen.yml

benchmarks/appsec_iast_propagation/scenario.py

@@ -2,14 +2,14 @@
 
 import bm
 
-from ddtrace.appsec.iast._taint_tracking import OriginType
-from ddtrace.appsec.iast._taint_tracking import Source
-from ddtrace.appsec.iast._taint_tracking import TaintRange
-from ddtrace.appsec.iast._taint_tracking import create_context
-from ddtrace.appsec.iast._taint_tracking import reset_context
-from ddtrace.appsec.iast._taint_tracking import set_ranges
-from ddtrace.appsec.iast._taint_tracking.aspects import add_aspect
-from ddtrace.appsec.iast._taint_tracking.aspects import join_aspect
+from ddtrace.appsec._iast._taint_tracking import OriginType
+from ddtrace.appsec._iast._taint_tracking import Source
+from ddtrace.appsec._iast._taint_tracking import TaintRange
+from ddtrace.appsec._iast._taint_tracking import create_context
+from ddtrace.appsec._iast._taint_tracking import reset_context
+from ddtrace.appsec._iast._taint_tracking import set_ranges
+from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect
+from ddtrace.appsec._iast._taint_tracking.aspects import join_aspect
 
 
 TAINT_ORIGIN = Source(name="sample_name", value="sample_value", origin=OriginType.PARAMETER)

ddtrace/_monkey.py

@@ -210,7 +210,7 @@ def patch_all(**patch_modules):
 
     patch(raise_errors=False, **modules)
     if config._iast_enabled:
-        from ddtrace.appsec.iast._patch_modules import patch_iast
+        from ddtrace.appsec._iast._patch_modules import patch_iast
 
         patch_iast()
 

ddtrace/appsec/_api_security/api_manager.py

@@ -6,7 +6,7 @@
 
 from ddtrace import config
 from ddtrace._tracing._limits import MAX_SPAN_META_VALUE_LEN
-from ddtrace.appsec import processor as appsec_processor
+from ddtrace.appsec import _processor as appsec_processor
 from ddtrace.appsec._asm_request_context import _WAF_RESULTS
 from ddtrace.appsec._asm_request_context import add_context_callback
 from ddtrace.appsec._asm_request_context import call_waf_callback

ddtrace/appsec/_asm_request_context.py

@@ -3,10 +3,10 @@
 from typing import TYPE_CHECKING
 
 from ddtrace import config
-from ddtrace.appsec import handlers
+from ddtrace.appsec import _handlers
 from ddtrace.appsec._constants import SPAN_DATA_NAMES
 from ddtrace.appsec._constants import WAF_CONTEXT_NAMES
-from ddtrace.appsec.iast._utils import _is_iast_enabled
+from ddtrace.appsec._iast._utils import _is_iast_enabled
 from ddtrace.internal import core
 from ddtrace.internal.compat import parse
 from ddtrace.internal.constants import REQUEST_PATH_PARAMS
@@ -153,7 +153,7 @@ def set_headers_response(headers):  # type: (Any) -> None
 
 def set_body_response(body_response):
     # local import to avoid circular import
-    from ddtrace.appsec.utils import parse_response_body
+    from ddtrace.appsec._utils import parse_response_body
 
     parsed_body = parse_response_body(body_response)
 
@@ -338,7 +338,7 @@ def _start_context(remote_ip, headers, headers_case_sensitive, block_request_cal
     if config._appsec_enabled:
         resources = _DataHandler()
         asm_request_context_set(remote_ip, headers, headers_case_sensitive, block_request_callable)
-        handlers.listen()
+        _handlers.listen()
         listen_context_handlers()
         return resources
 
@@ -382,8 +382,8 @@ def _on_wrapped_view(kwargs):
 
     # If IAST is enabled, taint the Flask function kwargs (path parameters)
     if _is_iast_enabled() and kwargs:
-        from ddtrace.appsec.iast._taint_tracking import OriginType
-        from ddtrace.appsec.iast._taint_tracking import taint_pyobject
+        from ddtrace.appsec._iast._taint_tracking import OriginType
+        from ddtrace.appsec._iast._taint_tracking import taint_pyobject
 
         _kwargs = {}
         for k, v in kwargs.items():
@@ -396,9 +396,9 @@ def _on_wrapped_view(kwargs):
 
 def _on_set_request_tags(request, span, flask_config):
     if _is_iast_enabled():
-        from ddtrace.appsec.iast._metrics import _set_metric_iast_instrumented_source
-        from ddtrace.appsec.iast._taint_tracking import OriginType
-        from ddtrace.appsec.iast._taint_utils import LazyTaintDict
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
+        from ddtrace.appsec._iast._taint_tracking import OriginType
+        from ddtrace.appsec._iast._taint_utils import LazyTaintDict
 
         _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME)
         _set_metric_iast_instrumented_source(OriginType.COOKIE)

ddtrace/appsec/_capabilities.py

@@ -5,7 +5,7 @@
 
 from ddtrace import Tracer
 from ddtrace import config as ddconfig
-from ddtrace.appsec.utils import _appsec_rc_features_is_enabled
+from ddtrace.appsec._utils import _appsec_rc_features_is_enabled
 from ddtrace.internal.compat import to_bytes_py2
 
 

ddtrace/appsec/ddwaf/__init__.py renamed to ddtrace/appsec/_ddwaf/__init__.py

@@ -5,9 +5,9 @@
 import xmltodict
 
 from ddtrace import config
-from ddtrace.appsec.iast._patch import if_iast_taint_returned_object_for
-from ddtrace.appsec.iast._patch import if_iast_taint_yield_tuple_for
-from ddtrace.appsec.iast._utils import _is_iast_enabled
+from ddtrace.appsec._iast._patch import if_iast_taint_returned_object_for
+from ddtrace.appsec._iast._patch import if_iast_taint_yield_tuple_for
+from ddtrace.appsec._iast._utils import _is_iast_enabled
 from ddtrace.contrib import trace_utils
 from ddtrace.internal import core
 from ddtrace.internal.constants import HTTP_REQUEST_BLOCKED
@@ -83,9 +83,9 @@ def _on_request_init(wrapped, instance, args, kwargs):
     wrapped(*args, **kwargs)
     if _is_iast_enabled():
         try:
-            from ddtrace.appsec.iast._metrics import _set_metric_iast_instrumented_source
-            from ddtrace.appsec.iast._taint_tracking import OriginType
-            from ddtrace.appsec.iast._taint_tracking import taint_pyobject
+            from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
+            from ddtrace.appsec._iast._taint_tracking import OriginType
+            from ddtrace.appsec._iast._taint_tracking import taint_pyobject
 
             # TODO: instance.query_string = ??
             instance.query_string = taint_pyobject(
@@ -109,8 +109,8 @@ def _on_request_init(wrapped, instance, args, kwargs):
 def _on_flask_patch(flask_version):
     if _is_iast_enabled():
         try:
-            from ddtrace.appsec.iast._metrics import _set_metric_iast_instrumented_source
-            from ddtrace.appsec.iast._taint_tracking import OriginType
+            from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
+            from ddtrace.appsec._iast._taint_tracking import OriginType
 
             _w(
                 "werkzeug.datastructures",
@@ -161,10 +161,10 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type):
     # If IAST is enabled and we're wrapping a Django view call, taint the kwargs (view's
     # path parameters)
     if _is_iast_enabled() and fn_args and isinstance(fn_args[0], first_arg_expected_type):
-        from ddtrace.appsec.iast._taint_tracking import OriginType  # noqa: F401
-        from ddtrace.appsec.iast._taint_tracking import is_pyobject_tainted
-        from ddtrace.appsec.iast._taint_tracking import taint_pyobject
-        from ddtrace.appsec.iast._taint_utils import LazyTaintDict
+        from ddtrace.appsec._iast._taint_tracking import OriginType  # noqa: F401
+        from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted
+        from ddtrace.appsec._iast._taint_tracking import taint_pyobject
+        from ddtrace.appsec._iast._taint_utils import LazyTaintDict
 
         http_req = fn_args[0]
 
@@ -216,9 +216,9 @@ def _on_wsgi_environ(wrapped, _instance, args, kwargs):
         if not args:
             return wrapped(*args, **kwargs)
 
-        from ddtrace.appsec.iast._metrics import _set_metric_iast_instrumented_source
-        from ddtrace.appsec.iast._taint_tracking import OriginType  # noqa: F401
-        from ddtrace.appsec.iast._taint_utils import LazyTaintDict
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
+        from ddtrace.appsec._iast._taint_tracking import OriginType  # noqa: F401
+        from ddtrace.appsec._iast._taint_utils import LazyTaintDict
 
         _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
         _set_metric_iast_instrumented_source(OriginType.HEADER)
@@ -240,7 +240,7 @@ def _on_wsgi_environ(wrapped, _instance, args, kwargs):
 
 def _on_django_patch():
     try:
-        from ddtrace.appsec.iast._taint_tracking import OriginType  # noqa: F401
+        from ddtrace.appsec._iast._taint_tracking import OriginType  # noqa: F401
 
         when_imported("django.http.request")(
             lambda m: trace_utils.wrap(

ddtrace/appsec/ddwaf/ddwaf_types.py renamed to ddtrace/appsec/_ddwaf/ddwaf_types.py

@@ -0,0 +1,73 @@
+"""IAST (interactive application security testing) analyzes code for security vulnerabilities.
+
+To add new vulnerabilities analyzers (Taint sink) we should update `IAST_PATCH` in
+`ddtrace/appsec/iast/_patch_modules.py`
+
+Create new file with the same name: `ddtrace/appsec/iast/taint_sinks/[my_new_vulnerability].py`
+
+Then, implement the `patch()` function and its wrappers.
+
+In order to have the better performance, the Overhead control engine (OCE) helps us to control the overhead of our
+wrapped functions. We should create a class that inherit from `ddtrace.appsec._iast.taint_sinks._base.VulnerabilityBase`
+and register with `ddtrace.appsec._iast.oce`.
+
+@oce.register
+class MyVulnerability(VulnerabilityBase):
+    vulnerability_type = "MyVulnerability"
+    evidence_type = "kind_of_Vulnerability"
+
+Before that, we should decorate our wrappers with `wrap` method and
+report the vulnerabilities with `report` method. OCE will manage the number of requests, number of vulnerabilities
+to reduce the overhead.
+
+@WeakHash.wrap
+def wrapped_function(wrapped, instance, args, kwargs):
+    # type: (Callable, str, Any, Any, Any) -> Any
+    WeakHash.report(
+        evidence_value=evidence,
+    )
+    return wrapped(*args, **kwargs)
+"""  # noqa: RST201, RST213, RST210
+import inspect
+import sys
+
+from ddtrace.internal.logger import get_logger
+
+from ._ast.ast_patching import astpatch_module
+from ._overhead_control_engine import OverheadControl
+from ._utils import _is_iast_enabled
+
+
+log = get_logger(__name__)
+
+oce = OverheadControl()
+
+
+def ddtrace_iast_flask_patch():
+    """
+    Patch the code inside the Flask main app source code file (typically "app.py") so
+    IAST/Custom Code propagation works also for the functions and methods defined inside it.
+    This must be called on the top level or inside the `if __name__ == "__main__"`
+    and must be before the `app.run()` call. It also requires `DD_IAST_ENABLED` to be
+    activated.
+    """
+    if not _is_iast_enabled():
+        return
+
+    module_name = inspect.currentframe().f_back.f_globals["__name__"]
+    module = sys.modules[module_name]
+    try:
+        module_path, patched_ast = astpatch_module(module, remove_flask_run=True)
+    except Exception:
+        log.debug("Unexpected exception while AST patching", exc_info=True)
+        return
+
+    compiled_code = compile(patched_ast, module_path, "exec")
+    exec(compiled_code, module.__dict__)  # nosec B102
+    sys.modules[module_name] = compiled_code
+
+
+__all__ = [
+    "oce",
+    "ddtrace_iast_flask_patch",
+]