feat(asm): add RC support for API sec (#7518)

New API Security features:

- Use last version of appsec event rules to provides static rules for
api security
- Remove temporary rule file previously used for api security
- Add remote config support for api security (ability to remotely update
scanners and processors from RC) (api security sample rate was already
added in a previous PR)
- Update api security unit tests accordingly

API Security is still a private feature, no release note for now.

## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [ ] Title is accurate.
- [ ] No unnecessary changes are introduced.
- [ ] Description motivates each change.
- [ ] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [ ] Testing strategy adequately addresses listed risk(s).
- [ ] Change is maintainable (easy to change, telemetry, documentation).
- [ ] Release note makes sense to a user of the library.
- [ ] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [ ] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [ ] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [ ] This PR doesn't touch any of that.

---------

Signed-off-by: Juanjo Alvarez <[email protected]>
Co-authored-by: Emmett Butler <[email protected]>
Co-authored-by: Yun Kim <[email protected]>
Co-authored-by: Tahir H. Butt <[email protected]>
Co-authored-by: Alberto Vara <[email protected]>
Co-authored-by: Kyle Verhoog <[email protected]>
Co-authored-by: Zachary Groves <[email protected]>
Co-authored-by: Federico Mon <[email protected]>
Co-authored-by: Romain Komorn <[email protected]>
Co-authored-by: Eric Navarro <[email protected]>
Co-authored-by: Emmett Butler <[email protected]>
Co-authored-by: Juanjo Alvarez Martinez <[email protected]>
Co-authored-by: Brett Langdon <[email protected]>
Co-authored-by: AJ Stuyvenberg <[email protected]>
Co-authored-by: Munir Abdinur <[email protected]>
Co-authored-by: Gabriele N. Tornetta <[email protected]>
Co-authored-by: HovhannesV <[email protected]>
Co-authored-by: Tahir H. Butt <[email protected]>

Author: 16 people

ddtrace/appsec/_api_security/processors.json

@@ -199,7 +199,6 @@ class LOGIN_EVENTS_MODE(metaclass=Constant_Class):
 class DEFAULT(metaclass=Constant_Class):
     ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
     RULES = os.path.join(ROOT_DIR, "rules.json")
-    API_SECURITY_PARAMETERS = os.path.join(ROOT_DIR, "_api_security/processors.json")
     TRACE_RATE_LIMIT = 100
     WAF_TIMEOUT = 5.0  # float (milliseconds)
     APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = (

ddtrace/appsec/_constants.py

@@ -137,11 +137,6 @@ def __post_init__(self) -> None:
         try:
             with open(self.rules, "r") as f:
                 rules = json.load(f)
-                if asm_config._api_security_enabled:
-                    with open(DEFAULT.API_SECURITY_PARAMETERS, "r") as f_apisec:
-                        processors = json.load(f_apisec)
-                        rules["processors"] = processors["processors"]
-                        rules["scanners"] = processors["scanners"]
                 self._update_actions(rules)
 
         except EnvironmentError as err:

ddtrace/appsec/_processor.py

@@ -121,6 +121,7 @@ def _appsec_rules_data(features: Mapping[str, Any], test_tracer: Optional[Tracer
         _add_rules_to_list(features, "exclusions", "exclusion filters", ruleset)
         _add_rules_to_list(features, "rules_override", "rules override", ruleset)
         _add_rules_to_list(features, "scanners", "scanners", ruleset)
+        _add_rules_to_list(features, "processors", "processors", ruleset)
         if ruleset:
             return tracer._appsec_processor._update_rules({k: v for k, v in ruleset.items() if v is not None})
 

ddtrace/appsec/_remoteconfiguration.py

@@ -4,9 +4,9 @@
 import json
 
 from ddtrace.appsec import _constants
+from ddtrace.appsec._constants import API_SECURITY
 from ddtrace.settings.asm import config as asm_config
 from tests.appsec.appsec.api_security.test_schema_fuzz import equal_with_meta
-from tests.appsec.appsec.test_processor import RULES_SRB
 from tests.utils import override_env
 from tests.utils import override_global_config
 
@@ -45,7 +45,9 @@ def _aux_appsec_get_root_span(
 def test_api_security(client, test_spans, tracer):
     import django
 
-    with override_global_config(dict(_asm_enabled=True, _api_security_enabled=True, _api_security_sample_rate=1.0)):
+    with override_global_config(
+        dict(_asm_enabled=True, _api_security_enabled=True, _api_security_sample_rate=1.0)
+    ), override_env({API_SECURITY.SAMPLE_RATE: "1.0"}):
         payload = {"key": "secret", "ids": [0, 1, 2, 3]}
         root_span, response = _aux_appsec_get_root_span(
             client,
@@ -120,7 +122,7 @@ def test_api_security_with_srb(client, test_spans, tracer):
 
     with override_global_config(
         dict(_asm_enabled=True, _api_security_enabled=True, _api_security_sample_rate=1.0)
-    ), override_env({"DD_APPSEC_RULES": RULES_SRB}):
+    ), override_env({API_SECURITY.SAMPLE_RATE: "1.0"}):
         payload = {"key": "secret", "ids": [0, 1, 2, 3]}
         root_span, response = _aux_appsec_get_root_span(
             client,
@@ -130,18 +132,19 @@ def test_api_security_with_srb(client, test_spans, tracer):
             payload=payload,
             cookies={"secret": "a1b2c3d4e5f6"},
             content_type="application/json",
+            headers={"HTTP_USER_AGENT": "dd-test-scanner-log-block"},
         )
         assert response.status_code == 403
         loaded = json.loads(root_span.get_tag(_constants.APPSEC.JSON))
-        assert [t["rule"]["id"] for t in loaded["triggers"]] == ["tst-037-001"]
+        assert [t["rule"]["id"] for t in loaded["triggers"]] == ["ua0-600-56x"]
 
         assert asm_config._api_security_enabled
 
         for name, expected_value in [
             ("_dd.appsec.s.req.body", [{"key": [8], "ids": [[[4]], {"len": 4}]}]),
             (
                 "_dd.appsec.s.req.headers",
-                [{"content-length": [8], "content-type": [8]}],
+                [{"user-agent": [8], "content-length": [8], "content-type": [8]}],
             ),
             ("_dd.appsec.s.req.cookies", [{"secret": [8]}]),
             ("_dd.appsec.s.req.query", [{"y": [8], "x": [8]}]),
@@ -152,14 +155,15 @@ def test_api_security_with_srb(client, test_spans, tracer):
             value = root_span.get_tag(name)
             assert value, name
             api = json.loads(gzip.decompress(base64.b64decode(value)).decode())
+            print(api)
             assert equal_with_meta(api, expected_value), name
 
 
 def test_api_security_deactivated(client, test_spans, tracer):
     """Test if blocking is still working as expected with api security deactivated"""
 
     with override_global_config(dict(_asm_enabled=True, _api_security_enabled=False)), override_env(
-        {_constants.API_SECURITY.SAMPLE_RATE: "1.0", "DD_APPSEC_RULES": RULES_SRB}
+        {_constants.API_SECURITY.SAMPLE_RATE: "1.0"}
     ):
         payload = {"key": "secret", "ids": [0, 1, 2, 3]}
         root_span, response = _aux_appsec_get_root_span(
@@ -170,10 +174,11 @@ def test_api_security_deactivated(client, test_spans, tracer):
             payload=payload,
             cookies={"secret": "a1b2c3d4e5f6"},
             content_type="application/json",
+            headers={"HTTP_USER_AGENT": "dd-test-scanner-log-block"},
         )
         assert response.status_code == 403
         loaded = json.loads(root_span.get_tag(_constants.APPSEC.JSON))
-        assert [t["rule"]["id"] for t in loaded["triggers"]] == ["tst-037-001"]
+        assert [t["rule"]["id"] for t in loaded["triggers"]] == ["ua0-600-56x"]
 
         assert not asm_config._api_security_enabled