fix(botocore): [SVLS-5973] less noisy span pointers [backport 2.16] (#11422)

Backport ecd0ab1 from #11353 to 2.16.

Span pointer issues should be debug messages rather than warnings since
they are not really actionable from the perspective of our users.

We'd also like to have some instrumentation telemetry to see how the
logic is doing.

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: Aleksandr Pasechnik <[email protected]>

Author: github-actions[bot]

ddtrace/_trace/_span_pointer.py

@@ -8,6 +8,7 @@
 
 from ddtrace._trace._span_link import SpanLink
 from ddtrace._trace._span_link import SpanLinkKind
+from ddtrace._trace.telemetry import record_span_pointer_calculation_issue
 from ddtrace.internal.logger import get_logger
 
 
@@ -67,20 +68,28 @@ def __post_init__(self):
 def _standard_hashing_function(*elements: bytes) -> str:
     try:
         if not elements:
-            raise ValueError("elements must not be empty")
+            return _standard_hashing_function_failure("elements must not be empty")
 
         # Please see the tests for more details about this logic.
         return sha256(b"|".join(elements)).hexdigest()[:32]
 
     except Exception as e:
-        log.warning(
-            "failed to generate standard hash for span pointer: %s",
-            str(e),
-        )
-        return _add_random_suffix(
-            prefix=_STANDARD_HASHING_FUNCTION_FAILURE_PREFIX,
-            minimum_length=32,
-        )
+        return _standard_hashing_function_failure(str(e))
+
+
+def _standard_hashing_function_failure(reason: str) -> str:
+    log.debug(
+        "failed to generate standard hash for span pointer: %s",
+        reason,
+    )
+    record_span_pointer_calculation_issue(
+        context="standard_hashing_function",
+    )
+
+    return _add_random_suffix(
+        prefix=_STANDARD_HASHING_FUNCTION_FAILURE_PREFIX,
+        minimum_length=32,
+    )
 
 
 def _add_random_suffix(*, prefix: str, minimum_length: int) -> str:

ddtrace/_trace/telemetry.py

@@ -0,0 +1,52 @@
+from typing import Optional
+from typing import Tuple
+
+from ddtrace.internal.telemetry import telemetry_writer
+
+
+def record_span_pointer_calculation(context: str, span_pointer_count: int) -> None:
+    telemetry_writer.add_count_metric(
+        namespace="tracer",
+        name="span_pointer_calculation",
+        value=1,
+        tags=(("context", context), ("count", _span_pointer_count_to_tag(span_pointer_count))),
+    )
+
+
+def _span_pointer_count_to_tag(span_pointer_count: int) -> str:
+    if span_pointer_count < 0:
+        # this shouldn't be possible, but let's make sure
+        return "negative"
+
+    elif span_pointer_count <= 5:
+        return str(span_pointer_count)
+
+    elif span_pointer_count <= 10:
+        return "6-10"
+
+    elif span_pointer_count <= 20:
+        return "11-20"
+
+    elif span_pointer_count <= 50:
+        return "21-50"
+
+    elif span_pointer_count <= 100:
+        return "51-100"
+
+    else:
+        return "101+"
+
+
+def record_span_pointer_calculation_issue(
+    context: str, additional_tags: Optional[Tuple[Tuple[str, str], ...]] = None
+) -> None:
+    tags: Tuple[Tuple[str, str], ...] = (("context", context),)
+    if additional_tags:
+        tags += additional_tags
+
+    telemetry_writer.add_count_metric(
+        namespace="tracer",
+        name="span_pointer_calculation.issue",
+        value=1,
+        tags=tags,
+    )

ddtrace/_trace/utils_botocore/span_pointers/__init__.py

@@ -14,6 +14,12 @@
 # import from here as well.
 from ddtrace._trace.utils_botocore.span_pointers.s3 import _aws_s3_object_span_pointer_description  # noqa: F401
 from ddtrace._trace.utils_botocore.span_pointers.s3 import _extract_span_pointers_for_s3_response
+from ddtrace._trace.utils_botocore.span_pointers.telemetry import record_span_pointer_calculation
+from ddtrace._trace.utils_botocore.span_pointers.telemetry import record_span_pointer_calculation_issue
+from ddtrace.internal.logger import get_logger
+
+
+log = get_logger(__name__)
 
 
 def extract_span_pointers_from_successful_botocore_response(
@@ -23,12 +29,22 @@ def extract_span_pointers_from_successful_botocore_response(
     request_parameters: Dict[str, Any],
     response: Dict[str, Any],
 ) -> List[_SpanPointerDescription]:
-    if endpoint_name == "s3":
-        return _extract_span_pointers_for_s3_response(operation_name, request_parameters, response)
+    result = []
+
+    try:
+        if endpoint_name == "s3":
+            result = _extract_span_pointers_for_s3_response(operation_name, request_parameters, response)
+
+        elif endpoint_name == "dynamodb":
+            result = _extract_span_pointers_for_dynamodb_response(
+                dynamodb_primary_key_names_for_tables, operation_name, request_parameters, response
+            )
+
+    except Exception as e:
+        # Catch-all in case we miss something in the helpers
+        log.debug("Error extracting span pointers from botocore response: %s", e)
+        record_span_pointer_calculation_issue("extractor_root", "unexpected_error")
 
-    if endpoint_name == "dynamodb":
-        return _extract_span_pointers_for_dynamodb_response(
-            dynamodb_primary_key_names_for_tables, operation_name, request_parameters, response
-        )
+    record_span_pointer_calculation(span_pointer_count=len(result))
 
-    return []
+    return result