Default cProfile command line to cumulative time

Display one folder level for pstats.stripdirs() when filename starts with __ (__init__.py, __main__.py primarily)

Author: boxed

Lib/cProfile.py

@@ -39,7 +39,7 @@ class Profile(_lsprof.Profiler):
 
     def print_stats(self, sort=-1):
         import pstats
-        pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats()
+        pstats.Stats(self).strip_non_unique_dirs().sort_stats(sort).print_stats()
 
     def dump_stats(self, file):
         import marshal
@@ -140,7 +140,7 @@ def main():
         help="Save stats to <outfile>", default=None)
     parser.add_option('-s', '--sort', dest="sort",
         help="Sort order when printing to stdout, based on pstats.Stats class",
-        default=-1,
+        default=2,
         choices=sorted(pstats.Stats.sort_arg_dict_default))
     parser.add_option('-m', dest="module", action="store_true",
         help="Profile a library module", default=False)

Lib/pstats.py

@@ -26,10 +26,13 @@
 import marshal
 import re
 
+from collections import Counter
 from enum import Enum
 from functools import cmp_to_key
 from dataclasses import dataclass
 from typing import Dict
+from os.path import join
+from pathlib import Path
 
 __all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"]
 
@@ -278,16 +281,19 @@ def reverse_order(self):
         return self
 
     def strip_dirs(self):
+        return self._strip_directory_data(lambda fullpath_linenum_funcname: func_strip_path(fullpath_linenum_funcname))
+
+    def _strip_directory_data(self, strip_function):
         oldstats = self.stats
         self.stats = newstats = {}
         max_name_len = 0
         for func, (cc, nc, tt, ct, callers) in oldstats.items():
-            newfunc = func_strip_path(func)
+            newfunc = strip_function(func)
             if len(func_std_string(newfunc)) > max_name_len:
                 max_name_len = len(func_std_string(newfunc))
             newcallers = {}
             for func2, caller in callers.items():
-                newcallers[func_strip_path(func2)] = caller
+                newcallers[strip_function(func2)] = caller
 
             if newfunc in newstats:
                 newstats[newfunc] = add_func_stats(
@@ -298,14 +304,30 @@ def strip_dirs(self):
         old_top = self.top_level
         self.top_level = new_top = set()
         for func in old_top:
-            new_top.add(func_strip_path(func))
+            new_top.add(strip_function(func))
 
         self.max_name_len = max_name_len
 
         self.fcn_list = None
         self.all_callees = None
         return self
 
+    def strip_non_unique_dirs(self):
+        full_paths = set()
+
+        for (full_path, _, _), (_, _, _, _, callers) in self.stats.items():
+            full_paths.add(full_path)
+            for (full_path_caller, _, _), _ in callers.items():
+                full_paths.add(full_path_caller)
+
+        minimal_path_by_full_path = _build_minimal_path_by_full_path(full_paths)
+
+        def strip_function(fullpath_linenum_funcname):
+            fullpath, linenum, funcname = fullpath_linenum_funcname
+            return minimal_path_by_full_path[fullpath], linenum, funcname
+
+        return self._strip_directory_data(strip_function)
+
     def calc_callees(self):
         if self.all_callees:
             return
@@ -776,4 +798,41 @@ def postcmd(self, stop, line):
     except KeyboardInterrupt:
         pass
 
+
+def _build_minimal_path_by_full_path(paths):
+    if not paths:
+        return paths
+
+    completed = {
+        full_path: None
+        for full_path in paths
+    }
+    split_path_by_full = {full_path: Path(full_path).parts for full_path in paths}
+    max_step = max(len(x) for x in split_path_by_full.values())
+    step = 1
+    while step <= max_step:
+        short_path_by_full = {
+            full_path: split_path_by_full[full_path][-step:]
+            for full_path, value in completed.items()
+            if value is None
+        }
+        full_path_by_short = {v: k for k, v in short_path_by_full.items()}
+
+        for short_path, count in Counter(short_path_by_full.values()).most_common():
+            if count == 1:
+                joined_short_path = join(*short_path)
+                # __init__.py is handled specially because it's a very common
+                # file name which gives no clue what file is meant
+                if joined_short_path == '__init__.py':
+                    continue
+                completed[full_path_by_short[short_path]] = joined_short_path
+
+        step += 1
+
+    return {
+        full_path: short_path if short_path is not None else full_path
+        for full_path, short_path in completed.items()
+    }
+
+
 # That's all, folks.

Lib/test/test_pstats.py

@@ -3,7 +3,7 @@
 from test import support
 from io import StringIO
 from pstats import SortKey
-
+from pstats import _build_minimal_path_by_full_path
 import pstats
 import cProfile
 
@@ -99,5 +99,66 @@ def test_SortKey_enum(self):
         self.assertEqual(SortKey.FILENAME, 'filename')
         self.assertNotEqual(SortKey.FILENAME, SortKey.CALLS)
 
+class BuildMinimalPathTests(unittest.TestCase):
+    def test_non_unique(self):
+        self.assertEqual(
+            _build_minimal_path_by_full_path([
+                'foo/bar',
+                'foo/bar',
+            ]),
+            {
+                'foo/bar': 'bar',
+            },
+        )
+
+    def test_needs_no_minimizing(self):
+        self.assertEqual(
+            _build_minimal_path_by_full_path([
+                'foo',
+                'bar',
+            ]),
+            {
+                'foo': 'foo',
+                'bar': 'bar',
+            },
+        )
+
+    def test_normal_case(self):
+        self.assertEqual(
+            _build_minimal_path_by_full_path([
+                'foo/bar',
+                'baz/bar',
+                'apple/orange',
+            ]),
+            {
+                'foo/bar': 'foo/bar',
+                'baz/bar': 'baz/bar',
+                'apple/orange': 'orange',
+            }
+        )
+
+    def test_intermediate(self):
+        self.assertEqual(
+            _build_minimal_path_by_full_path([
+                'apple/mango/orange/grape/melon',
+                'apple/mango/lemon/grape/melon',
+            ]),
+            {
+                'apple/mango/orange/grape/melon': 'orange/grape/melon',
+                'apple/mango/lemon/grape/melon': 'lemon/grape/melon',
+            }
+        )
+
+    def test_dunder_init_special_case(self):
+        self.assertEqual(
+            _build_minimal_path_by_full_path([
+                'apple/mango/orange/grape/__init__.py',
+            ]),
+            {
+                'apple/mango/orange/grape/__init__.py': 'grape/__init__.py',
+            }
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Library/2020-10-18-13-20-00.bpo-34861.8JanqQ.rst

@@ -0,0 +1,2 @@
+cProfile command line output now defaults to cumulative time
+cProfile command line gives you unique filenames in the output (plus an extra path level of path if the filename is __init__.py)