Skip to content

Percentile Support #93

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jdhardy wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Percentile Support #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dcc0562
Add percentile calculations to Stats.
jdhardy Nov 6, 2017
7e148ac
Allow percentile (pxx.xx) columns in output.
jdhardy Nov 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/usage.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ Commandline options
@pytest.parametrize. Default: 'group'
--benchmark-columns=LABELS
Comma-separated list of columns to show in the result
table. Default: 'min, max, mean, stddev, median, iqr,
table. Use 'pXX.XX' (e.g. 'p99.9') to show percentiles.
Default: 'min, max, mean, stddev, median, iqr,
outliers, rounds, iterations'
--benchmark-name=FORMAT
How to format names in results. Can be one of 'short',
Expand Down
7 changes: 4 additions & 3 deletions src/pytest_benchmark/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ def add_display_options(addoption, prefix="benchmark-"):
"--{0}columns".format(prefix),
metavar="LABELS", type=parse_columns,
default=["min", "max", "mean", "stddev", "median", "iqr", "outliers", "ops", "rounds", "iterations"],
help="Comma-separated list of columns to show in the result table. Default: "
"'min, max, mean, stddev, median, iqr, outliers, rounds, iterations'"
help="Comma-separated list of columns to show in the result table. Use 'pXX.XX' (e.g. 'p99.9') to show "
"percentiles. Default: 'min, max, mean, stddev, median, iqr, outliers, rounds, iterations'"
)
addoption(
"--{0}name".format(prefix),
Expand Down Expand Up @@ -374,9 +374,10 @@ def pytest_benchmark_generate_json(config, benchmarks, include_data, machine_inf
"datetime": datetime.utcnow().isoformat(),
"version": __version__,
}
columns = config.getoption("benchmark_columns")
for bench in benchmarks:
if not bench.has_error:
benchmarks_json.append(bench.as_dict(include_data=include_data))
benchmarks_json.append(bench.as_dict(include_data=include_data, columns=columns))
return output_json


Expand Down
7 changes: 6 additions & 1 deletion src/pytest_benchmark/session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,12 @@ def prepare_benchmarks(self):
if fail:
self.performance_regressions.append((self.name_format(flat_bench), fail))
yield flat_bench
flat_bench = bench.as_dict(include_data=False, flat=True, cprofile=self.cprofile_sort_by)
flat_bench = bench.as_dict(
include_data=False,
flat=True,
cprofile=self.cprofile_sort_by,
columns=self.columns
)
flat_bench["path"] = None
flat_bench["source"] = compared and "NOW"
yield flat_bench
Expand Down
63 changes: 58 additions & 5 deletions src/pytest_benchmark/stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from bisect import bisect_left
from bisect import bisect_right

from .utils import PERCENTILE_COL_RX
from .utils import cached_property
from .utils import funcname
from .utils import get_cprofile_functions
Expand All @@ -26,10 +27,11 @@ def __bool__(self):
def __nonzero__(self):
return bool(self.data)

def as_dict(self):
def as_dict(self, extra_fields=None):
fields = Stats.fields + tuple(extra_fields) if extra_fields else Stats.fields
return dict(
(field, getattr(self, field))
for field in self.fields
for field in fields
)

def update(self, duration):
Expand Down Expand Up @@ -168,6 +170,52 @@ def ops(self):
return self.rounds / self.total
return 0

def __getattr__(self, name):
m = PERCENTILE_COL_RX.match(name)
if not m:
raise AttributeError(name)

p = float(m.group(1)) / 100.0
return self.percentile(p)

def percentile(self, percent):
''' Compute the interpolated percentile.

This is the method recommmended by NIST:
http://www.itl.nist.gov/div898/handbook/prc/section2/prc262.htm

percent must be in the range [0.0, 1.0].
'''
if not (0.0 <= percent <= 1.0):
raise ValueError('percent must be in the range [0.0, 1.0]')

if not hasattr(self, '_percentile_cache'):
self._percentile_cache = {}

# Check the cache first
# This isn't perfect with floats for the usual reasons, but is good enough
cached = self._percentile_cache.get(percent)
if cached is not None:
return cached

# percentiles require sorted data
data = self.sorted_data
N = len(data)
if percent <= 1/(N+1):
# Too small, return min
return self._percentile_cache.setdefault(percent, data[0])
elif percent >= N/(N+1):
# too big, return max
return self._percentile_cache.setdefault(percent, data[-1])
else:
r = percent * (N + 1)
k = r // 1
d = r % 1

n = int(k - 1) # zero-indexed lists
result = data[n] + d * (data[n+1] - data[n])
return self._percentile_cache.setdefault(percent, result)


class Metadata(object):
def __init__(self, fixture, iterations, options):
Expand All @@ -180,9 +228,9 @@ def __init__(self, fixture, iterations, options):
self.cprofile_stats = fixture.cprofile_stats

self.iterations = iterations
self.stats = Stats()
self.options = options
self.fixture = fixture
self.stats = Stats()

def __bool__(self):
return bool(self.stats)
Expand All @@ -206,7 +254,7 @@ def __getitem__(self, key):
def has_error(self):
return self.fixture.has_error

def as_dict(self, include_data=True, flat=False, stats=True, cprofile=None):
def as_dict(self, include_data=True, flat=False, stats=True, cprofile=None, columns=None):
result = {
"group": self.group,
"name": self.name,
Expand Down Expand Up @@ -236,7 +284,12 @@ def as_dict(self, include_data=True, flat=False, stats=True, cprofile=None):
if cprofile is None or len(cprofile_functions) == len(cprofile_list):
break
if stats:
stats = self.stats.as_dict()
if columns is not None:
extra_fields = tuple(c for c in columns if c not in Stats.fields and PERCENTILE_COL_RX.match(c))
else:
extra_fields = None

stats = self.stats.as_dict(extra_fields=extra_fields)
if include_data:
stats["data"] = self.stats.data
stats["iterations"] = self.iterations
Expand Down
17 changes: 12 additions & 5 deletions src/pytest_benchmark/table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import sys
from math import isinf

from .utils import PERCENTILE_COL_RX
from .utils import operations_unit
from .utils import report_progress
from .utils import time_unit
Expand All @@ -14,6 +15,8 @@


class TableResults(object):
standard_columns = ("min", "max", "mean", "stddev", "median", "iqr")

def __init__(self, columns, sort, histogram, name_format, logger):
self.columns = columns
self.sort = sort
Expand All @@ -22,6 +25,9 @@ def __init__(self, columns, sort, histogram, name_format, logger):
self.logger = logger

def display(self, tr, groups, progress_reporter=report_progress):
percentile_columns = tuple(c for c in self.columns if PERCENTILE_COL_RX.match(c))
numeric_columns = self.standard_columns + percentile_columns

tr.write_line("")
tr.rewrite("Computing stats ...", black=True, bold=True)
for line, (group, benchmarks) in progress_reporter(groups, tr, "Computing stats ... group {pos}/{total}"):
Expand All @@ -32,8 +38,7 @@ def display(self, tr, groups, progress_reporter=report_progress):
worst = {}
best = {}
solo = len(benchmarks) == 1
for line, prop in progress_reporter(("min", "max", "mean", "median", "iqr", "stddev", "ops"),
tr, "{line}: {value}", line=line):
for line, prop in progress_reporter(numeric_columns + ("ops",), tr, "{line}: {value}", line=line):
if prop == "ops":
worst[prop] = min(bench[prop] for _, bench in progress_reporter(
benchmarks, tr, "{line} ({pos}/{total})", line=line))
Expand Down Expand Up @@ -66,14 +71,16 @@ def display(self, tr, groups, progress_reporter=report_progress):
"outliers": "Outliers",
"ops": "OPS ({0}ops/s)".format(ops_unit) if ops_unit else "OPS",
}
labels.update(dict((c, c.upper()) for c in percentile_columns))

widths = {
"name": 3 + max(len(labels["name"]), max(len(benchmark["name"]) for benchmark in benchmarks)),
"rounds": 2 + max(len(labels["rounds"]), len(str(worst["rounds"]))),
"iterations": 2 + max(len(labels["iterations"]), len(str(worst["iterations"]))),
"outliers": 2 + max(len(labels["outliers"]), len(str(worst["outliers"]))),
"ops": 2 + max(len(labels["ops"]), len(NUMBER_FMT.format(best["ops"] * ops_adjustment))),
}
for prop in "min", "max", "mean", "stddev", "median", "iqr":
for prop in numeric_columns:
widths[prop] = 2 + max(len(labels[prop]), max(
len(NUMBER_FMT.format(bench[prop] * adjustment))
for bench in benchmarks
Expand All @@ -83,7 +90,7 @@ def display(self, tr, groups, progress_reporter=report_progress):
labels_line = labels["name"].ljust(widths["name"]) + "".join(
labels[prop].rjust(widths[prop]) + (
" " * rpadding
if prop not in ["outliers", "rounds", "iterations"]
if prop not in ("outliers", "rounds", "iterations")
else ""
)
for prop in self.columns
Expand All @@ -103,7 +110,7 @@ def display(self, tr, groups, progress_reporter=report_progress):
has_error = bench.get("has_error")
tr.write(bench["name"].ljust(widths["name"]), red=has_error, invert=has_error)
for prop in self.columns:
if prop in ("min", "max", "mean", "stddev", "median", "iqr"):
if prop in numeric_columns:
tr.write(
ALIGNED_NUMBER_FMT.format(
bench[prop] * adjustment,
Expand Down
5 changes: 3 additions & 2 deletions src/pytest_benchmark/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ def check_output(*popenargs, **kwargs):
"n": "Nanoseconds (ns)"
}
ALLOWED_COLUMNS = ["min", "max", "mean", "stddev", "median", "iqr", "ops", "outliers", "rounds", "iterations"]
PERCENTILE_COL_RX = re.compile(r'p(\d+(?:.\d+)?)')


class SecondsDecimal(Decimal):
Expand Down Expand Up @@ -360,11 +361,11 @@ def parse_sort(string):

def parse_columns(string):
columns = [str.strip(s) for s in string.lower().split(',')]
invalid = set(columns) - set(ALLOWED_COLUMNS)
invalid = set(columns) - set(ALLOWED_COLUMNS) - set(c for c in columns if PERCENTILE_COL_RX.match(c))
if invalid:
# there are extra items in columns!
msg = "Invalid column name(s): %s. " % ', '.join(invalid)
msg += "The only valid column names are: %s" % ', '.join(ALLOWED_COLUMNS)
msg += "The only valid column names are: %s, pXX.XX" % ', '.join(ALLOWED_COLUMNS)
raise argparse.ArgumentTypeError(msg)
return columns

Expand Down
28 changes: 26 additions & 2 deletions tests/test_benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,9 @@ def test_help(testdir):
" @pytest.parametrize. Default: 'group'",
" --benchmark-columns=LABELS",
" Comma-separated list of columns to show in the result",
" table. Default: 'min, max, mean, stddev, median, iqr,",
" outliers, rounds, iterations'",
" table. Use 'pXX.XX' (e.g. 'p99.9') to show",
" percentiles. Default: 'min, max, mean, stddev, median,",
" iqr, outliers, rounds, iterations'",
" --benchmark-histogram=[FILENAME-PREFIX]",
" Plot graphs of min/max/avg/stddev over time in",
" FILENAME-PREFIX-test_name.svg. If FILENAME-PREFIX",
Expand Down Expand Up @@ -679,6 +680,18 @@ def test_extra(benchmark):
assert bench_info['extra_info'] == {'foo': 'bar'}


def test_save_percentiles(testdir):
test = testdir.makepyfile(SIMPLE_TEST)
result = testdir.runpytest('--doctest-modules', '--benchmark-save=foobar',
'--benchmark-max-time=0.0000001', '--benchmark-columns=min,p99,max', test)
result.stderr.fnmatch_lines([
"Saved benchmark data in: *",
])
info = json.loads(testdir.tmpdir.join('.benchmarks').listdir()[0].join('0001_foobar.json').read())
bench_info = info['benchmarks'][0]
assert 'p99' in bench_info['stats']


def test_histogram(testdir):
test = testdir.makepyfile(SIMPLE_TEST)
result = testdir.runpytest('--doctest-modules', '--benchmark-histogram=foobar',
Expand Down Expand Up @@ -1072,3 +1085,14 @@ def test_columns(testdir):
"Name (time in ?s) * Max * Iterations * Min *",
"------*",
])

def test_columns_percentiles(testdir):
test = testdir.makepyfile(SIMPLE_TEST)
result = testdir.runpytest('--doctest-modules', '--benchmark-columns=max,p99,iterations,min', test)
result.stdout.fnmatch_lines([
"*collected 3 items",
"test_columns_percentiles.py ...",
"* benchmark: 2 tests *",
"Name (time in ?s) * Max * P99 * Iterations * Min *",
"------*",
])
5 changes: 3 additions & 2 deletions tests/test_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ def test_help_compare(testdir, args):
" 'param:NAME', where NAME is the name passed to",
" @pytest.parametrize. Default: 'group'",
" --columns LABELS Comma-separated list of columns to show in the result",
" table. Default: 'min, max, mean, stddev, median, iqr,",
" outliers, rounds, iterations'",
" table. Use 'pXX.XX' (e.g. 'p99.9') to show",
" percentiles. Default: 'min, max, mean, stddev, median,",
" iqr, outliers, rounds, iterations'",
" --name FORMAT How to format names in results. Can be one of 'short',",
" 'normal', 'long'. Default: 'normal'",
" --histogram [FILENAME-PREFIX]",
Expand Down
8 changes: 4 additions & 4 deletions tests/test_elasticsearch_storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ def __init__(self):
'max_time': 345,
}
self.compare_fail = []
self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr',
'outliers', 'rounds', 'iterations']
self.config = Namespace(hook=Namespace(
pytest_benchmark_group_stats=pytest_benchmark_group_stats,
pytest_benchmark_generate_machine_info=lambda **kwargs: {'foo': 'bar'},
Expand All @@ -90,20 +92,18 @@ def __init__(self):
pytest_benchmark_update_json=lambda **kwargs: None,
pytest_benchmark_generate_commit_info=lambda **kwargs: {'foo': 'bar'},
pytest_benchmark_update_commit_info=lambda **kwargs: None,
))
), getoption=lambda name: {'benchmark_columns': self.columns}[name])
self.elasticsearch_host = "localhost:9200"
self.elasticsearch_index = "benchmark"
self.elasticsearch_doctype = "benchmark"
self.storage = MockStorage()
self.group_by = 'group'
self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr',
'outliers', 'rounds', 'iterations']
self.benchmarks = []
with BENCHFILE.open('rU') as fh:
data = json.load(fh)
self.benchmarks.extend(
Namespace(
as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench:
as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench, columns=None:
dict(_bench, **_bench["stats"]) if flat else dict(_bench),
name=bench['name'],
fullname=bench['fullname'],
Expand Down
Loading