chore(iast): fix wheel build error on macOS [backport 2.21] (#12542)

Backport 2ef9f1e from #12536 to 2.21.

With this PR, all macOS wheels should include the IAST `.so` file. If
not, it will trigger an error **in the CI**
- Fix IAST C++ code to support macOS 10.13 or later  
- Enable smoke tests for all macOS builds in `build_python_3.yml`  
- Remove the "optional" flag for the IAST build in `setup.py` to ensure
a fast failure if any issues arise

APPSEC-56890

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: avara1986

ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.cpp

@@ -60,7 +60,8 @@ as_formatted_evidence(const string& text,
 
     for (const auto& taint_range : text_ranges) {
         string content;
-        if (!tag_mapping_mode or tag_mapping_mode.value() == TagMappingMode::Normal) {
+        // tag_mapping_mode.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+        if (!tag_mapping_mode or !tag_mapping_mode.has_value() or *tag_mapping_mode == TagMappingMode::Normal) {
             content = get_default_content(taint_range);
         } else
             switch (*tag_mapping_mode) {
@@ -111,10 +112,11 @@ api_as_formatted_evidence(const StrType& text,
     }
 
     TaintRangeRefs _ranges;
-    if (!text_ranges) {
+    if (!text_ranges or !text_ranges.has_value()) {
         _ranges = api_get_ranges(text);
     } else {
-        _ranges = text_ranges.value();
+        // text_ranges.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+        _ranges = *text_ranges;
     }
     return StrType(as_formatted_evidence(AnyTextObjectToString(text), _ranges, tag_mapping_mode, new_ranges));
 }
@@ -161,19 +163,22 @@ api_convert_escaped_text_to_taint_text(PyObject* taint_escaped_text,
 
     switch (py_str_type) {
         case PyTextType::UNICODE: {
-            const auto text_str = py::reinterpret_borrow<py::str>(text_pyobj_opt.value());
+            // text_pyobj_opt.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+            const auto text_str = py::reinterpret_borrow<py::str>(*text_pyobj_opt);
             auto obj = api_convert_escaped_text_to_taint_text<py::str>(text_str, ranges_orig);
             Py_INCREF(obj.ptr());
             return obj.ptr();
         }
         case PyTextType::BYTES: {
-            const auto text_bytes = py::reinterpret_borrow<py::bytes>(text_pyobj_opt.value());
+            // text_pyobj_opt.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+            const auto text_bytes = py::reinterpret_borrow<py::bytes>(*text_pyobj_opt);
             auto obj = api_convert_escaped_text_to_taint_text<py::bytes>(text_bytes, ranges_orig);
             Py_INCREF(obj.ptr());
             return obj.ptr();
         }
         case PyTextType::BYTEARRAY: {
-            const auto text_bytearray = py::reinterpret_borrow<py::bytearray>(text_pyobj_opt.value());
+            // text_pyobj_opt.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+            const auto text_bytearray = py::reinterpret_borrow<py::bytearray>(*text_pyobj_opt);
             auto obj = api_convert_escaped_text_to_taint_text<py::bytearray>(text_bytearray, ranges_orig);
             Py_INCREF(obj.ptr());
             return obj.ptr();

ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.h

@@ -144,16 +144,18 @@ inline string
 mapper_replace(const TaintRangePtr& taint_range, const optional<const py::dict>& new_ranges)
 {
 
-    if (!taint_range or !new_ranges.has_value() or py::len(new_ranges.value()) == 0) {
+    if (!taint_range or !new_ranges.has_value() or py::len(*new_ranges) == 0) {
         return {};
     }
 
-    const py::dict& new_ranges_value = new_ranges.value();
+    // new_ranges.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+    const py::dict& new_ranges_value = *new_ranges;
     py::object o = py::cast(taint_range);
 
     if (!new_ranges_value.contains(o)) {
         return {};
     }
+    // new_ranges.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
     const TaintRange new_range = py::cast<TaintRange>((*new_ranges)[o]);
     return to_string(new_range.get_hash());
 }

ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.cpp

@@ -223,13 +223,13 @@ are_all_text_all_ranges(PyObject* candidate_text, const py::tuple& parameter_lis
 TaintRangePtr
 get_range_by_hash(const size_t range_hash, optional<TaintRangeRefs>& taint_ranges)
 {
-    if (not taint_ranges or taint_ranges->empty()) {
+    if (not taint_ranges or !taint_ranges.has_value() or taint_ranges->empty()) {
         return nullptr;
     }
-    // TODO: Replace this loop with a efficient function, vector.find() is O(n)
-    // too.
+    // TODO: Replace this loop with a efficient function, vector.find() is O(n) too.
     TaintRangePtr null_range = nullptr;
-    for (const auto& range : taint_ranges.value()) {
+    // taint_ranges.value throws a compile error: 'value' is unavailable: introduced in macOS 10.13
+    for (const auto& range : *taint_ranges) {
         if (range_hash == range->get_hash()) {
             return range;
         }

setup.py

@@ -534,7 +534,9 @@ def get_exts_for(name):
             )
         )
 
-        ext_modules.append(CMakeExtension("ddtrace.appsec._iast._taint_tracking._native", source_dir=IAST_DIR))
+        ext_modules.append(
+            CMakeExtension("ddtrace.appsec._iast._taint_tracking._native", source_dir=IAST_DIR, optional=False)
+        )
 
     if (
         platform.system() == "Linux" or (platform.system() == "Darwin" and platform.machine() == "arm64")

tests/smoke_test.py

@@ -1,20 +1,11 @@
 import copy
 import os
-from platform import system
+import platform
 import subprocess
 import sys
 import textwrap
 
 
-def mac_supported_iast_version():
-    if system() == "Darwin":
-        # TODO: MacOS 10.9 or lower has a old GCC version but cibuildwheel has a GCC old version in newest mac versions
-        # mac_version = [int(i) for i in mac_ver()[0].split(".")]
-        # mac_version > [10, 9]
-        return False
-    return True
-
-
 # Code need to be run in a separate subprocess to reload since reloading .so files doesn't
 # work like normal Python ones
 test_native_load_code = """
@@ -45,7 +36,7 @@ def emit(self, record):
 
 if __name__ == "__main__":
     # ASM IAST smoke test
-    if sys.version_info >= (3, 6, 0) and system() != "Windows" and mac_supported_iast_version():
+    if sys.version_info >= (3, 6, 0) and platform.system() != "Windows":
         print("Running native IAST module load test...")
         test_code = textwrap.dedent(test_native_load_code)
         cmd = [sys.executable, "-c", test_code]
@@ -67,7 +58,7 @@ def emit(self, record):
             os.environ = orig_env
 
     # ASM WAF smoke test
-    if system() != "Linux" or sys.maxsize > 2**32:
+    if platform.system() != "Linux" or sys.maxsize > 2**32:
         import ddtrace.appsec._ddwaf
         import ddtrace.bootstrap.sitecustomize as module
 
@@ -80,3 +71,33 @@ def emit(self, record):
     else:
         # Skip the test for 32-bit Linux systems
         print("Skipping test, 32-bit DDWAF not ready yet")
+
+    # Profiling smoke test
+    print("Running profiling smoke test...")
+    profiling_cmd = [sys.executable, "-c", "import ddtrace.profiling.auto"]
+    if sys.version_info >= (3, 13, 0):
+        print("Skipping profiling smoke test for Python 3.13+ as it's not supported yet")
+    elif (
+        # echion doesn't work on Windows
+        platform.system() == "Windows"
+        # libdatadog x86_64-apple-darwin has not yet been integrated to dd-trace-py
+        or (platform.system() == "Darwin" and platform.machine() == "x86_64")
+        # echion crashes on musl linux with Python 3.12 for both x86_64 and
+        # aarch64
+        or (platform.system() == "Linux" and sys.version_info[:2] == (3, 12) and platform.libc_ver()[0] != "glibc")
+    ):
+        orig_env = os.environ.copy()
+        copied_env = copy.deepcopy(orig_env)
+        copied_env["DD_PROFILING_STACK_V2_ENABLED"] = "False"
+        if platform.system() == "Windows":
+            # Memory profiler crashes on Windows
+            copied_env["DD_PROFILING_MEMORY_ENABLED"] = "False"
+        result = subprocess.run(profiling_cmd, env=copied_env, capture_output=True, text=True)
+        assert result.returncode == 0, "Failed with DD_PROFILING_STACK_V2_ENABLED=0: %s, %s" % (
+            result.stdout,
+            result.stderr,
+        )
+    else:
+        result = subprocess.run(profiling_cmd, capture_output=True, text=True)
+        assert result.returncode == 0, "Failed: %s, %s" % (result.stdout, result.stderr)
+    print("Profiling smoke test completed successfully")