chore: fast build environment variables [backport 2.21] (#12736)

Backport #12509 to 2.21.

## Description

- Adds `DD_FAST_BUILD` which, if present, should halve the time to build
the project (in my machine, 146 seconds without to 69 seconds without
sccache) or be 4x faster if previously catched with sccache (44 seconds
to 10) by exchanging some optimizations (disable abseil lib in IAST,
-O0) for better build time. To be used for development and CI (CI config
is also modified to use this env var, except for the `main` branch to
catch corner cases that could happen only when optimized).

- Adds `DD_COMPILE_ABSEIL` which, is present and "0" will disable the
compilation of abseil in the native module independently of the release
mode (currently is compiled by default if in `Release` mode only; this
behavior will continue if `DD_COMPILE_ABSEIL` is not present).

- Add a new "build" section in `docs/configuration.rst` with these two
environment vars plus two that were not
 documented.

- Fix an error in the documentation of `DD_COMPILE_DEBUG` (it enables
`Debug` mode, not `RelWithDebInfo` as stated, and does it for all native
extensions not only pyx ones). I will add an option to compile with
`RelWithDebInfo` in a further PR.



Signed-off-by: Juanjo Alvarez <[email protected]>##
Checklist
- [X] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

---------

Co-authored-by: Juanjo Alvarez Martinez <[email protected]>

Author: avara1986

.circleci/config.templ.yml

@@ -394,7 +394,14 @@ jobs:
       - run:
           name: "Generate base virtual environments."
           # DEV: riot list -i tracer lists all supported Python versions
-          command: "riot list -i tracer | circleci tests split | xargs -I PY riot -P -v generate --python=PY"
+          command: |
+            if [ "$CIRCLE_BRANCH" = "main" ]; then
+              export DD_FAST_BUILD=0
+            else
+              export DD_FAST_BUILD=1
+            fi
+            echo "DD_FAST_BUILD is set to $DD_FAST_BUILD"
+            riot list -i tracer | circleci tests split | xargs -I PY riot -P -v generate --python=PY
           environment:
             CMAKE_BUILD_PARALLEL_LEVEL: 12
             PIP_VERBOSE: 1

.gitlab/tests.yml

@@ -39,6 +39,11 @@ include:
         echo "Running hatch env: ${env}:test"
         hatch run ${env}:test
       done
+  variables:
+    CMAKE_BUILD_PARALLEL_LEVEL = "12"
+    CARGO_BUILD_JOBS = "12"
+    DD_FAST_BUILD = "1"
+
 
 
 .test_base_hatch_snapshot:

.riot/requirements/1fe8dd2.txt

@@ -30,12 +30,23 @@ else()
 endif(BUILD_MACOS)
 unset(BUILD_MACOS CACHE)
 
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
-    message("Release mode: using abseil")
+# Check the DD_COMPILE_ABSEIL environment variable and build type
+if(DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}" STREQUAL
+                                                                                 "false"))
+
+    message("==============================================================")
+    message("WARNING: DD_COMPILE_ABSEIL set to 0 or false: not using abseil")
+    message("==============================================================")
+    add_definitions(-DDONT_COMPILE_ABSEIL) # Define DONT_COMPILE_ABSEIL preprocessor variable
+elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message("=====================================")
+    message("WARNING: Debug mode: not using abseil")
+    message("=====================================")
+    add_definitions(-DDONT_COMPILE_ABSEIL) # Define DONT_COMPILE_ABSEIL preprocessor variable
+else()
+    message("Release mode: using abseil (DD_COMPILE_ABSEIL unset or not 0/false)")
     FetchContent_Declare(absl URL "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip")
     FetchContent_MakeAvailable(absl)
-else()
-    message("Debug mode: not using abseil")
 endif()
 
 include_directories(".")
@@ -81,7 +92,9 @@ set_target_properties(_native PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRE
 
 # target_link_libraries(_native PRIVATE ${ICU_LIBS})
 
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+if(CMAKE_BUILD_TYPE STREQUAL "Release"
+   AND NOT (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}"
+                                                                                          STREQUAL "false")))
     target_link_libraries(${APP_NAME} PRIVATE absl::node_hash_map)
 endif()
 

.riot/requirements/6bec1ec.txt

@@ -19,17 +19,14 @@ class TaintedObject;
 // Alias
 using TaintedObjectPtr = shared_ptr<TaintedObject>;
 
-#ifdef NDEBUG // Decide wether to use abseil
-
+// Use Abseil only if NDEBUG is set and DONT_COMPILE_ABSEIL is not set
+#if defined(NDEBUG) && !defined(DONT_COMPILE_ABSEIL)
 #include "absl/container/node_hash_map.h"
 using TaintRangeMapType = absl::node_hash_map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
-
 #else
-
 #include <unordered_map>
-using TaintRangeMapType = std::map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
-
-#endif // NDEBUG
+using TaintRangeMapType = std::unordered_map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
+#endif
 
 using TaintRangeMapTypePtr = shared_ptr<TaintRangeMapType>;
 // using TaintRangeMapTypePtr = TaintRangeMapType*;

ddtrace/appsec/_iast/_taint_tracking/CMakeLists.txt

@@ -11,7 +11,10 @@ file(GLOB TEST_SOURCES "*.cpp" "*.hpp")
 add_executable(native_tests ${TEST_SOURCES} ${SOURCE_FILES} ${HEADER_FILES})
 
 set(NATIVE_TEST_LIBRARIES gtest gtest_main ${PYTHON_LIBRARIES} pybind11::module)
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release"
+   AND NOT (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}"
+                                                                                          STREQUAL "false")))
     list(APPEND NATIVE_TEST_LIBRARIES absl::node_hash_map)
 endif()
 

ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h

@@ -185,8 +185,6 @@ DD_CIVISIBILITY_AGENTLESS_ENABLED = "1"
 DD_CIVISIBILITY_CODE_COVERAGE_ENABLED = "1"
 DD_CIVISIBILITY_ITR_ENABLED = "1"
 DD_PATCH_MODULES = "unittest:false"
-CMAKE_BUILD_PARALLEL_LEVEL = "12"
-
 
 ## ASM Django
 
@@ -323,6 +321,35 @@ test = [
 [[envs.appsec_iast_packages.matrix]]
 python = ["3.9", "3.10", "3.11", "3.12"]
 
+## ASM iast_tdd_propagation
+
+[envs.iast_tdd_propagation]
+template = "iast_tdd_propagation"
+dependencies = [
+    "pytest",
+    "pytest-cov",
+    "requests",
+    "hypothesis",
+    "flask",
+    "pycryptodome",
+    "sqlalchemy~=2.0.23",
+    "pony",
+    "aiosqlite",
+    "tortoise-orm",
+    "peewee",
+    "aiosqlite",
+]
+
+[envs.iast_tdd_propagation.scripts]
+test = [
+    "uname -a",
+    "pip freeze",
+    "DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 DD_IAST_DEDUPLICATION_ENABLED=false python -m pytest tests/appsec/iast_tdd_propagation",
+]
+
+[[envs.iast_tdd_propagation.matrix]]
+python = ["3.9", "3.10", "3.11", "3.12", "3.13"]
+
 ## ASM appsec_integrations_django
 
 [envs.appsec_integrations_django]
@@ -399,6 +426,50 @@ flask = ["~=2.2"]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 flask = ["~=3.0"]
 
+[[envs.appsec_integrations_flask.matrix]]
+# werkzeug 3.1 drops support for py3.8
+python = ["3.11", "3.12", "3.13"]
+flask = ["~=3.1"]
+werkzeug = ["~=3.1"]
+
+## ASM appsec_integrations_fastapi
+
+[envs.appsec_integrations_fastapi]
+template = "appsec_integrations_fastapi"
+dependencies = [
+    "pytest",
+    "pytest-cov",
+    "requests",
+    "hypothesis",
+    "python-multipart",
+    "jinja2",
+    "httpx<0.28.0",
+    "anyio{matrix:anyio:}",
+    "fastapi{matrix:fastapi}"
+]
+
+[envs.appsec_integrations_fastapi.scripts]
+test = [
+    "uname -a",
+    "pip freeze",
+    "DD_TRACE_AGENT_URL=\"http://testagent:9126\" DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 DD_IAST_DEDUPLICATION_ENABLED=false python -m pytest -vvv {args:tests/appsec/integrations/fastapi_tests/}",
+]
+
+
+# if you add or remove a version here, please also update the parallelism parameter
+# in .circleci/config.templ.yml
+[[envs.appsec_integrations_fastapi.matrix]]
+python = ["3.8", "3.10", "3.13"]
+fastapi = ["==0.86.0"]
+anyio = ["==3.7.1"]
+
+[[envs.appsec_integrations_fastapi.matrix]]
+python = ["3.8", "3.10", "3.13"]
+fastapi = ["==0.94.1"]
+
+[[envs.appsec_integrations_fastapi.matrix]]
+python = ["3.8", "3.10", "3.13"]
+fastapi = ["~=0.114.2"]
 
 
 ## ASM FastAPI
@@ -441,7 +512,6 @@ fastapi = ["==0.94.1"]
 python = ["3.8", "3.10", "3.13"]
 fastapi = ["~=0.114.2"]
 
-
 ## ASM Appsec Aggregated Leak Testing
 
 [envs.appsec_aggregated_leak_testing]
@@ -457,9 +527,9 @@ dependencies = [
     "pydantic-settings",
 ]
 
-[envs.appsec_aggregated_leak_testing.env-vars]
-CMAKE_BUILD_PARALLEL_LEVEL = "12"
+[envs.iast_aggregated_leak_testing.env-vars]
 DD_IAST_ENABLED = "true"
+_DD_IAST_PATCH_MODULES = "scripts.iast"
 
 [envs.appsec_aggregated_leak_testing.scripts]
 test = [
@@ -622,7 +692,6 @@ dependencies = [
 
 [evs.selenium.env-vars]
 DD_AGENT_TRACER_URL = "9126"
-CMAKE_BUILD_PARALLEL_LEVEL = "12"
 
 [envs.selenium.scripts]
 test = [

ddtrace/appsec/_iast/_taint_tracking/tests/CMakeLists.txt

@@ -2937,10 +2937,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             },
             venvs=[
                 # Python 3.8
-                Venv(
-                    pys=["3.8"],
-                    pkgs={"greenlet": "==3.1.0"},
-                ),
+                # Venv(
+                #     pys=["3.8"],
+                #     pkgs={"greenlet": "==3.1.0"},
+                # ),
                 # Python 3.9+
                 Venv(
                     pys=select_pys(min_version="3.9"),
@@ -2961,7 +2961,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
             pkgs={"vcrpy": latest, "pytest-asyncio": "==0.21.1"},
             venvs=[
                 Venv(pys="3.7"),
-                Venv(pys=select_pys(min_version="3.8"), pkgs={"ragas": "==0.1.21", "langchain": latest}),
+                Venv(
+                    pys=select_pys(min_version="3.8", max_version="3.12"),
+                    pkgs={"ragas": "==0.1.21", "langchain": latest},
+                ),
             ],
         ),
         Venv(