pythonGH-140643: Add <native> and <GC> frames to the sampling profiler (python#141108)

- Introduce a new field in the GC state to store the frame that initiated garbage collection.
- Update RemoteUnwinder to include options for including "<native>" and "<GC>" frames in the stack trace.
- Modify the sampling profiler to accept parameters for controlling the inclusion of native and GC frames.
- Enhance the stack collector to properly format and append these frames during profiling.
- Add tests to verify the correct behavior of the profiler with respect to native and GC frames, including options to exclude them.

Co-authored-by: Pablo Galindo Salgado <[email protected]>

Author: brandtbucher

Doc/library/profile.rst

@@ -265,6 +265,14 @@ Profile with real-time sampling statistics::
 
    Sample all threads in the process instead of just the main thread
 
+.. option:: --native
+
+   Include artificial ``<native>`` frames to denote calls to non-Python code.
+
+.. option:: --no-gc
+
+   Don't include artificial ``<GC>`` frames to denote active garbage collection.
+
 .. option:: --realtime-stats
 
    Print real-time sampling statistics during profiling
@@ -349,7 +357,7 @@ This section documents the programmatic interface for the :mod:`!profiling.sampl
 For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information
 about statistical profiling, see :ref:`statistical-profiling`
 
-.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False)
+.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False, native=False, gc=True)
 
    Sample a Python process and generate profiling data.
 
@@ -367,6 +375,8 @@ about statistical profiling, see :ref:`statistical-profiling`
    :param bool show_summary: Whether to show summary statistics (default: True)
    :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats')
    :param bool realtime_stats: Whether to display real-time statistics (default: False)
+   :param bool native: Whether to include ``<native>`` frames (default: False)
+   :param bool gc: Whether to include ``<GC>`` frames (default: True)
 
    :raises ValueError: If output_format is not 'pstats' or 'collapsed'
 

Include/internal/pycore_debug_offsets.h

@@ -212,6 +212,7 @@ typedef struct _Py_DebugOffsets {
     struct _gc {
         uint64_t size;
         uint64_t collecting;
+        uint64_t frame;
     } gc;
 
     // Generator object offset;
@@ -355,6 +356,7 @@ typedef struct _Py_DebugOffsets {
     .gc = { \
         .size = sizeof(struct _gc_runtime_state), \
         .collecting = offsetof(struct _gc_runtime_state, collecting), \
+        .frame = offsetof(struct _gc_runtime_state, frame), \
     }, \
     .gen_object = { \
         .size = sizeof(PyGenObject), \

Include/internal/pycore_global_objects_fini_generated.h

@@ -46,10 +46,12 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(dot_locals, ".<locals>")
         STRUCT_FOR_STR(empty, "")
         STRUCT_FOR_STR(format, ".format")
+        STRUCT_FOR_STR(gc, "<GC>")
         STRUCT_FOR_STR(generic_base, ".generic_base")
         STRUCT_FOR_STR(json_decoder, "json.decoder")
         STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
         STRUCT_FOR_STR(list_err, "list index out of range")
+        STRUCT_FOR_STR(native, "<native>")
         STRUCT_FOR_STR(str_replace_inf, "1e309")
         STRUCT_FOR_STR(type_params, ".type_params")
         STRUCT_FOR_STR(utf_8, "utf-8")
@@ -486,6 +488,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(fullerror)
         STRUCT_FOR_ID(func)
         STRUCT_FOR_ID(future)
+        STRUCT_FOR_ID(gc)
         STRUCT_FOR_ID(generation)
         STRUCT_FOR_ID(get)
         STRUCT_FOR_ID(get_debug)
@@ -629,6 +632,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(name_from)
         STRUCT_FOR_ID(namespace_separator)
         STRUCT_FOR_ID(namespaces)
+        STRUCT_FOR_ID(native)
         STRUCT_FOR_ID(ndigits)
         STRUCT_FOR_ID(nested)
         STRUCT_FOR_ID(new_file_name)

Include/internal/pycore_global_strings.h

@@ -212,6 +212,9 @@ struct _gc_runtime_state {
     struct gc_generation_stats generation_stats[NUM_GENERATIONS];
     /* true if we are currently running the collector */
     int collecting;
+    // The frame that started the current collection. It might be NULL even when
+    // collecting (if no Python frame is running):
+    _PyInterpreterFrame *frame;
     /* list of uncollectable objects */
     PyObject *garbage;
     /* a list of callbacks to be invoked when collection is performed */

Include/internal/pycore_interp_structs.h

@@ -24,7 +24,6 @@ enum _frameowner {
     FRAME_OWNED_BY_GENERATOR = 1,
     FRAME_OWNED_BY_FRAME_OBJECT = 2,
     FRAME_OWNED_BY_INTERPRETER = 3,
-    FRAME_OWNED_BY_CSTACK = 4,
 };
 
 struct _PyInterpreterFrame {

Include/internal/pycore_interpframe_structs.h

@@ -151,17 +151,22 @@ function createPythonTooltip(data) {
     const funcname = resolveString(d.data.funcname) || resolveString(d.data.name);
     const filename = resolveString(d.data.filename) || "";
 
+    // Don't show file location for special frames like <GC> and <native>
+    const isSpecialFrame = filename === "~";
+    const fileLocationHTML = isSpecialFrame ? "" : `
+        <div style="color: #5a6c7d; font-size: 13px; margin-bottom: 12px;
+                    font-family: monospace; background: #f8f9fa;
+                    padding: 4px 8px; border-radius: 4px; word-break: break-all; overflow-wrap: break-word;">
+          ${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
+        </div>`;
+
     const tooltipHTML = `
       <div>
         <div style="color: #3776ab; font-weight: 600; font-size: 16px;
                     margin-bottom: 8px; line-height: 1.3; word-break: break-word; overflow-wrap: break-word;">
           ${funcname}
         </div>
-        <div style="color: #5a6c7d; font-size: 13px; margin-bottom: 12px;
-                    font-family: monospace; background: #f8f9fa;
-                    padding: 4px 8px; border-radius: 4px; word-break: break-all; overflow-wrap: break-word;">
-          ${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
-        </div>
+        ${fileLocationHTML}
         <div style="display: grid; grid-template-columns: auto 1fr;
                     gap: 8px 16px; font-size: 14px;">
           <span style="color: #5a6c7d; font-weight: 500;">Execution Time:</span>
@@ -474,14 +479,23 @@ function populateStats(data) {
     if (i < hotSpots.length && hotSpots[i]) {
       const hotspot = hotSpots[i];
       const filename = hotspot.filename || 'unknown';
-      const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown';
       const lineno = hotspot.lineno ?? '?';
       let funcDisplay = hotspot.funcname || 'unknown';
       if (funcDisplay.length > 35) {
         funcDisplay = funcDisplay.substring(0, 32) + '...';
       }
 
-      document.getElementById(`hotspot-file-${num}`).textContent = `${basename}:${lineno}`;
+      // Don't show file:line for special frames like <GC> and <native>
+      const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?');
+      let fileDisplay;
+      if (isSpecialFrame) {
+        fileDisplay = '--';
+      } else {
+        const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown';
+        fileDisplay = `${basename}:${lineno}`;
+      }
+
+      document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay;
       document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay;
       document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`;
     } else {

Include/internal/pycore_runtime_init_generated.h

@@ -137,19 +137,19 @@ def _run_with_sync(original_cmd):
 
 
 class SampleProfiler:
-    def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True):
+    def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True):
         self.pid = pid
         self.sample_interval_usec = sample_interval_usec
         self.all_threads = all_threads
         if _FREE_THREADED_BUILD:
             self.unwinder = _remote_debugging.RemoteUnwinder(
-                self.pid, all_threads=self.all_threads, mode=mode,
+                self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc,
                 skip_non_matching_threads=skip_non_matching_threads
             )
         else:
             only_active_threads = bool(self.all_threads)
             self.unwinder = _remote_debugging.RemoteUnwinder(
-                self.pid, only_active_thread=only_active_threads, mode=mode,
+                self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc,
                 skip_non_matching_threads=skip_non_matching_threads
             )
         # Track sample intervals and total sample count
@@ -616,6 +616,8 @@ def sample(
     output_format="pstats",
     realtime_stats=False,
     mode=PROFILING_MODE_WALL,
+    native=False,
+    gc=True,
 ):
     # PROFILING_MODE_ALL implies no skipping at all
     if mode == PROFILING_MODE_ALL:
@@ -627,7 +629,7 @@ def sample(
         skip_idle = mode != PROFILING_MODE_WALL
 
     profiler = SampleProfiler(
-        pid, sample_interval_usec, all_threads=all_threads, mode=mode,
+        pid, sample_interval_usec, all_threads=all_threads, mode=mode, native=native, gc=gc,
         skip_non_matching_threads=skip_non_matching_threads
     )
     profiler.realtime_stats = realtime_stats
@@ -717,6 +719,8 @@ def wait_for_process_and_sample(pid, sort_value, args):
         output_format=args.format,
         realtime_stats=args.realtime_stats,
         mode=mode,
+        native=args.native,
+        gc=args.gc,
     )
 
 
@@ -767,9 +771,19 @@ def main():
     sampling_group.add_argument(
         "--realtime-stats",
         action="store_true",
-        default=False,
         help="Print real-time sampling statistics (Hz, mean, min, max, stdev) during profiling",
     )
+    sampling_group.add_argument(
+        "--native",
+        action="store_true",
+        help="Include artificial \"<native>\" frames to denote calls to non-Python code.",
+    )
+    sampling_group.add_argument(
+        "--no-gc",
+        action="store_false",
+        dest="gc",
+        help="Don't include artificial \"<GC>\" frames to denote active garbage collection.",
+    )
 
     # Mode options
     mode_group = parser.add_argument_group("Mode options")
@@ -934,6 +948,8 @@ def main():
             output_format=args.format,
             realtime_stats=args.realtime_stats,
             mode=mode,
+            native=args.native,
+            gc=args.gc,
         )
     elif args.module or args.args:
         if args.module: