fix(iast): fix memory leak error [backport #7788 to 2.3] (#7890)

This below backports list to 2.3.
- chore(iast): `format_aspect` refactor
#7884 and
#7772
- fix(iast): iast leak for python 3.11:
#7788
- chore(iast): add propagation for str aspect join:
#6940
- chore(iast): add collision tests for bytes and bytearray:
#7367
- chore(iast): store hash and id separately in tx_taint_map:
#7340

## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [x] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [x] This PR doesn't touch any of that.

Author: avara1986

ddtrace/appsec/_iast/_ast/ast_patching.py

@@ -5,13 +5,13 @@
 import os
 import re
 from sys import builtin_module_names
-from typing import TYPE_CHECKING
+from types import ModuleType
+from typing import TYPE_CHECKING  # noqa:F401
+from typing import Tuple
 
 
 if TYPE_CHECKING:
-    from types import ModuleType
-    from typing import Optional
-    from typing import Tuple
+    from typing import Optional  # noqa:F401
 
 from ddtrace.appsec._constants import IAST
 from ddtrace.appsec._python_info.stdlib import _stdlib_for_python_version
@@ -123,8 +123,7 @@ def _remove_flask_run(text):  # type (str) -> str
     return new_text
 
 
-def astpatch_module(module, remove_flask_run=False):
-    # type: (ModuleType, bool) -> Tuple[str, str]
+def astpatch_module(module: ModuleType, remove_flask_run: bool = False) -> Tuple[str, str]:
     module_name = module.__name__
     module_path = str(origin(module))
     try:

ddtrace/appsec/_iast/_ast/visitor.py

@@ -4,9 +4,9 @@
 import ast
 import copy
 import sys
-from typing import Any
-from typing import List
-from typing import Set
+from typing import Any  # noqa:F401
+from typing import List  # noqa:F401
+from typing import Set  # noqa:F401
 
 from six import iteritems
 

ddtrace/appsec/_iast/_overhead_control_engine.py

@@ -5,18 +5,18 @@
 """
 import os
 import threading
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING  # noqa:F401
 
 from ddtrace.internal.logger import get_logger
 from ddtrace.sampler import RateSampler
 
 
 if TYPE_CHECKING:  # pragma: no cover
-    from typing import Set
-    from typing import Tuple
-    from typing import Type
+    from typing import Set  # noqa:F401
+    from typing import Tuple  # noqa:F401
+    from typing import Type  # noqa:F401
 
-    from ddtrace.span import Span
+    from ddtrace.span import Span  # noqa:F401
 
 log = get_logger(__name__)
 
@@ -98,7 +98,8 @@ class OverheadControl(object):
     def reconfigure(self):
         self._sampler = RateSampler(sample_rate=get_request_sampling_value() / 100.0)
 
-    def acquire_request(self, span):  # type: (Span) -> None
+    def acquire_request(self, span):
+        # type: (Span) -> None
         """Decide whether if IAST analysis will be done for this request.
         - Block a request's quota at start of the request to limit simultaneous requests analyzed.
         - Use sample rating to analyze only a percentage of the total requests (30% by default).

ddtrace/appsec/_iast/_patch.py

@@ -1,7 +1,7 @@
 import ctypes
 import gc
 import sys
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING  # noqa:F401
 
 from ddtrace.internal.logger import get_logger
 from ddtrace.vendor.wrapt import FunctionWrapper
@@ -11,10 +11,10 @@
 
 
 if TYPE_CHECKING:  # pragma: no cover
-    from typing import Any
-    from typing import Callable
-    from typing import Dict
-    from typing import Optional
+    from typing import Any  # noqa:F401
+    from typing import Callable  # noqa:F401
+    from typing import Dict  # noqa:F401
+    from typing import Optional  # noqa:F401
 
 
 _DD_ORIGINAL_ATTRIBUTES = {}  # type: Dict[Any, Any]

ddtrace/appsec/_iast/_stacktrace.c

@@ -17,19 +17,22 @@
 #define GET_LINENO(frame) PyFrame_GetLineNumber((PyFrameObject*)frame)
 #define GET_FRAME(tstate) PyThreadState_GetFrame(tstate)
 #define GET_PREVIOUS(frame) PyFrame_GetBack(frame)
-#define FRAME_DECREF(frame) Py_DECREF(frame)
+#define FRAME_DECREF(frame) Py_DecRef(frame)
 #define FRAME_XDECREF(frame) Py_XDECREF(frame)
-#define FILENAME_DECREF(filename) Py_DECREF(filename)
-#define FILENAME_XDECREF(filename) Py_XDECREF(filename)
+#define FILENAME_DECREF(filename) Py_DecRef(filename)
+#define FILENAME_XDECREF(filename)                                                                                     \
+    if (filename)                                                                                                      \
+    Py_DecRef(filename)
 static inline PyObject*
 GET_FILENAME(PyFrameObject* frame)
 {
     PyCodeObject* code = PyFrame_GetCode(frame);
     if (!code) {
-        Py_DECREF(code);
         return NULL;
     }
-    return PyObject_GetAttrString((PyObject*)code, "co_filename");
+    PyObject* filename = PyObject_GetAttrString((PyObject*)code, "co_filename");
+    Py_DecRef(code);
+    return filename;
 }
 #else
 #define GET_FRAME(tstate) tstate->frame
@@ -113,7 +116,7 @@ get_file_and_line(PyObject* Py_UNUSED(module), PyObject* cwd_obj)
     }
 
 exit:
-    Py_DECREF(cwd_bytes);
+    Py_DecRef(cwd_bytes);
     FRAME_XDECREF(frame);
     FILENAME_XDECREF(filename_o);
     return result;
@@ -123,10 +126,10 @@ exit_0:; // fix: "a label can only be part of a statement and a declaration is n
     PyObject* line_obj = Py_BuildValue("i", 0);
     filename_o = PyUnicode_FromString("");
     result = PyTuple_Pack(2, filename_o, line_obj);
-    Py_DECREF(cwd_bytes);
+    Py_DecRef(cwd_bytes);
     FRAME_XDECREF(frame);
     FILENAME_XDECREF(filename_o);
-    Py_DECREF(line_obj);
+    Py_DecRef(line_obj);
     return result;
 }
 

ddtrace/appsec/_iast/_taint_dict.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 #
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING  # noqa:F401
 
 
 if TYPE_CHECKING:
-    from typing import Dict
-    from typing import Tuple
+    from typing import Dict  # noqa:F401
+    from typing import Tuple  # noqa:F401
 
-    from ._taint_tracking import Source
+    from ._taint_tracking import Source  # noqa:F401
 
 _IAST_TAINT_DICT = {}  # type: Dict[int, Tuple[Tuple[Source, int, int],...]]
 

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectExtend.cpp

@@ -1,5 +1,4 @@
 #include "AspectExtend.h"
-#include "Initializer/Initializer.h"
 
 PyObject*
 api_extend_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs)
@@ -28,7 +27,7 @@ api_extend_aspect(PyObject* self, PyObject* const* args, Py_ssize_t nargs)
     // Ensure no returns are done before this method call
     auto method_name = PyUnicode_FromString("extend");
     PyObject_CallMethodObjArgs(candidate_text, method_name, to_add, nullptr);
-    Py_DECREF(method_name);
+    Py_DecRef(method_name);
 
     if (not ctx_map or ctx_map->empty() or to_result == nullptr) {
         Py_RETURN_NONE;

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectExtend.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "Initializer/Initializer.h"
 #include <Python.h>
 
 PyObject*

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectFormat.cpp

@@ -0,0 +1,53 @@
+#pragma once
+#include "Aspects/AspectFormat.h"
+
+template<class StrType>
+StrType
+api_format_aspect(StrType& candidate_text,
+                  const py::tuple& parameter_list,
+                  const py::args& args,
+                  const py::kwargs& kwargs)
+{
+    auto [ranges_orig, candidate_text_ranges] = are_all_text_all_ranges(candidate_text.ptr(), parameter_list);
+
+    if (!ranges_orig.empty() or !candidate_text_ranges.empty()) {
+        auto new_template =
+          _int_as_formatted_evidence<StrType>(candidate_text, candidate_text_ranges, TagMappingMode::Mapper);
+
+        py::list new_args;
+        py::dict new_kwargs;
+        for (const auto arg : args) {
+            if (is_text(arg.ptr())) {
+                auto str_arg = py::cast<py::str>(arg);
+                auto n_arg = _all_as_formatted_evidence<py::str>(str_arg, TagMappingMode::Mapper);
+                new_args.append(n_arg);
+            } else {
+                new_args.append(arg);
+            }
+        }
+        for (auto [key, value] : kwargs) {
+            if (is_text(value.ptr())) {
+                auto str_value = py::cast<py::str>(value);
+                auto n_value = _all_as_formatted_evidence<py::str>(str_value, TagMappingMode::Mapper);
+                new_kwargs[key] = n_value;
+            } else {
+                new_kwargs[key] = value;
+            }
+        }
+        StrType new_template_format =
+          py::getattr(new_template, "format")(*(py::cast<py::tuple>(new_args)), **new_kwargs);
+        std::tuple result = _convert_escaped_text_to_taint_text<StrType>(new_template_format, ranges_orig);
+        StrType result_text = get<0>(result);
+        TaintRangeRefs result_ranges = get<1>(result);
+        PyObject* new_result = new_pyobject_id(result_text.ptr());
+        set_ranges(new_result, result_ranges);
+        return py::reinterpret_steal<StrType>(new_result);
+    }
+    return py::getattr(candidate_text, "format")(*args, **kwargs);
+}
+
+void
+pyexport_format_aspect(py::module& m)
+{
+    m.def("_format_aspect", &api_format_aspect<py::str>, "candidate_text"_a, "parameter_list"_a);
+}

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectFormat.h

@@ -0,0 +1,16 @@
+#pragma once
+#include "Aspects/Helpers.h"
+#include "Initializer/Initializer.h"
+#include "TaintTracking/Source.h"
+#include "TaintTracking/TaintRange.h"
+#include "TaintTracking/TaintedObject.h"
+
+template<class StrType>
+StrType
+api_format_aspect(StrType& candidate_text,
+                  const py::tuple& parameter_list,
+                  const py::args& args,
+                  const py::kwargs& kwargs);
+
+void
+pyexport_format_aspect(py::module& m);