Employ pyperf and separate environments to compare performance against the main branch. Ref #292.

Author: jaraco

.github/workflows/main.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Run benchmarks
         run: tox
         env:
-          TOXENV: perf
+          TOXENV: perf{,-ref}
 
   diffcov:
     runs-on: ubuntu-latest

tox.ini

@@ -33,19 +33,44 @@ commands =
 	diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html
 	diff-cover coverage.xml --compare-branch=origin/main --fail-under=100
 
+[perf]
+deps =
+	ipython
+	pyperf
+	path
+commands =
+	python -c "import path; path.Path('{env:SCOPE}.json').remove_p()"
+
+	python -m pyperf timeit --name discovery --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.distribution("ipython")'
+
+	python -m pyperf timeit --name 'entry_points()' --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.entry_points()'
+
+	python -m pyperf timeit --name 'cached distribution' --append {env:SCOPE}.json -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' 'importlib_metadata.distribution("ipython")'
+
+	python -m pyperf timeit --name 'uncached distribution' --append {env:SCOPE}.json -s 'import importlib, importlib_metadata' 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")'
+
 [testenv:perf]
 use_develop = False
+# change dir to avoid picking up local package.
+changedir = {toxworkdir}
+setenv =
+	SCOPE = local
+deps = {[perf]deps}
+commands = {[perf]commands}
+
+[testenv:perf-ref]
+# compare perf results to the main branch
+skip_install = True
+# change dir to avoid picking up local package.
+changedir = {toxworkdir}
+setenv =
+	SCOPE = main
 deps =
-	ipython
+	{[perf]deps}
+	git+https://github.com/python/importlib_metadata
 commands =
-	python -c 'print("Simple discovery performance")'
-	python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")'
-	python -c 'print("Entry point discovery performance")'
-	python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.entry_points()'
-	python -c 'print("Cached lookup performance")'
-	python -m timeit -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' -- 'importlib_metadata.distribution("ipython")'
-	python -c 'print("Uncached lookup performance")'
-	python -m timeit -s 'import importlib, importlib_metadata' -- 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")'
+	{[perf]commands}
+	python -m pyperf compare_to --verbose main.json local.json --table
 
 [testenv:release]
 skip_install = True