Run corpus tests after regression test

Author: dflook

.github/workflows/test_corpus.yaml

@@ -18,16 +18,31 @@ on:
         description: 'Regenerate results'
         required: true
         default: true
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref to test'
+        required: true
+        type: string
+      base-ref:
+        type: string
+        description: 'Git ref to compare against'
+        required: true
+      regenerate-results:
+        type: boolean
+        description: 'Regenerate results'
+        required: false
+        default: false
 
 jobs:
   generate_results:
     name: Minify Corpus
     runs-on: self-hosted
-    if: ${{ inputs.regenerate-results }}
     strategy:
       fail-fast: false
       matrix:
         python: ["python2.7", "python3.3", "python3.4", "python3.5", "python3.6", "python3.7", "python3.8", "python3.9", "python3.10", "python3.11"]
+        ref: ["${{ inputs.ref }}", "${{ inputs.base-ref }}"]
     container:
       image: danielflook/python-minifier-build:${{ matrix.python }}-2022-10-25
       volumes:
@@ -45,7 +60,7 @@ jobs:
       - name: Checkout ref
         uses: actions/checkout@v3
         with:
-          ref: ${{ inputs.ref }}
+          ref: ${{ matrix.ref }}
           path: python-minifier
 
       - name: Install ref
@@ -59,7 +74,7 @@ jobs:
 
       - name: Run tests
         run: |
-          ${{matrix.python}} workflow/corpus_test/generate_results.py /corpus /corpus-results $(<sha.txt)
+          ${{matrix.python}} workflow/corpus_test/generate_results.py /corpus /corpus-results $(<sha.txt) ${{ inputs.regenerate-results }}
 
   generate_report:
     name: Generate Report

.github/workflows/xtest.yaml

@@ -36,3 +36,11 @@ jobs:
           fi
           
           tox -r -e $(echo "${{ matrix.python }}" | tr -d .) xtest
+
+  test-corpus:
+    needs: test
+    name: Minify Corpus
+    uses: ./.github/workflows/test_corpus.yaml
+    with:
+      ref: ${{ github.ref }}
+      base-ref: ${{ github.base_ref }}

corpus_test/generate_report.py

@@ -6,10 +6,15 @@
 
 from result import Result, ResultReader
 
-ENHANCED_REPORT = False
+ENHANCED_REPORT = True
+
 
 @dataclass
 class ResultSet:
+    """
+    A set of results from minifying a corpus using a specific version of Python and a specific version of python-minifier
+    """
+
     python_version: str
     sha: str
 
@@ -106,7 +111,16 @@ def compare_size_decrease(self, base: 'ResultSet') -> Iterable[Result]:
             if result.minified_size < base_result.minified_size:
                 yield result
 
+
 def result_summary(results_dir: str, python_version: str, sha: str) -> ResultSet:
+    """
+    Return a summary of the results for a specific version of Python and a specific version of python-minifier
+
+    :param results_dir: The directory containing the results
+    :param python_version: The version of Python
+    :param sha: The git sha of the version of python-minifier
+    """
+
     summary = ResultSet(python_version, sha)
 
     results_file_path = os.path.join(results_dir, 'results_' + python_version + '_' + sha + '.csv')
@@ -118,7 +132,20 @@ def result_summary(results_dir: str, python_version: str, sha: str) -> ResultSet
 
     return summary
 
+
 def format_difference(compare: Iterable[Result], base: Iterable[Result]) -> str:
+    """
+    Return a string representing the difference between two sets of results
+
+    The returned string will include:
+     - the size of the compare set
+     - the number of new entries in the compare set that are not in the base set
+     - and the number of entries that are in the base set but not in the compare set.
+
+    :param compare: The results we are interested in
+    :param base: The results to compare against
+    """
+
     compare_set = set(result.corpus_entry for result in compare)
     base_set = set(result.corpus_entry for result in base)
 
@@ -139,6 +166,18 @@ def format_difference(compare: Iterable[Result], base: Iterable[Result]) -> str:
 
 
 def report(results_dir: str, minifier_ref: str, minifier_sha: str, base_ref: str, base_sha: str) -> Iterable[str]:
+    """
+    Generate a report comparing the results of two versions of python-minifier
+
+    The report is generated as a markdown string.
+
+    :param results_dir: The directory containing the results
+    :param minifier_ref: The git ref of the version of python-minifier
+    :param minifier_sha: The git sha of the version of python-minifier
+    :param base_ref: The git ref of the base version of python-minifier we are comparing against
+    :param base_sha: The git sha of the base version of python-minifier we are comparing against
+    """
+
     yield f'''
 # Python Minifier Test Report
 
@@ -186,14 +225,14 @@ def format_size_change_detail() -> str:
             return s
 
         yield (
-            f'| {python_version} ' +
-            f'| {summary.valid_count} ' +
-            f'| {summary.mean_time:.3f} ({mean_time_change:+.3f}) ' +
-            f'| {format_size_change_detail()} ' +
-            f'| {format_difference(summary.larger_than_original(), base_summary.larger_than_original())} ' +
-            f'| {format_difference(summary.recursion_error(), base_summary.recursion_error())} ' +
-            f'| {format_difference(summary.unstable_minification(), base_summary.unstable_minification())} ' +
-            f'| {format_difference(summary.exception(), base_summary.exception())} '
+                f'| {python_version} ' +
+                f'| {summary.valid_count} ' +
+                f'| {summary.mean_time:.3f} ({mean_time_change:+.3f}) ' +
+                f'| {format_size_change_detail()} ' +
+                f'| {format_difference(summary.larger_than_original(), base_summary.larger_than_original())} ' +
+                f'| {format_difference(summary.recursion_error(), base_summary.recursion_error())} ' +
+                f'| {format_difference(summary.unstable_minification(), base_summary.unstable_minification())} ' +
+                f'| {format_difference(summary.exception(), base_summary.exception())} '
         )
 
     if ENHANCED_REPORT:
@@ -242,6 +281,7 @@ def format_size_change_detail() -> str:
             for entry in sorted(summary.entries.values(), key=lambda entry: entry.time, reverse=True)[:10]:
                 yield f'| {entry.corpus_entry} | {entry.original_size} | {entry.minified_size} | {entry.time:.3f} |'
 
+
 def main():
     parser = argparse.ArgumentParser(description='Generate a test report for a given python-minifier ref')
     parser.add_argument('results_dir', type=str, help='Path to results directory', default='results')

corpus_test/generate_results.py

@@ -1,26 +1,28 @@
 import argparse
-import logging
 import os
 import sys
 import time
 
 import python_minifier
 from result import Result, ResultWriter
 
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
-
 try:
     RE = RecursionError
 except NameError:
     # Python 2
-    class RE(Exception): pass
+    class RE(Exception):
+        pass
+
 
-def minify_test(corpus_path, corpus_entry):
+def minify_corpus_entry(corpus_path, corpus_entry):
     """
+    Minify a single entry in the corpus and return the result
+
     :param str corpus_path: Path to the corpus
     :param str corpus_entry: A file in the corpus
+    :rtype: Result
     """
+
     with open(os.path.join(corpus_path, corpus_entry), 'rb') as f:
         source = f.read()
 
@@ -56,32 +58,49 @@ def minify_test(corpus_path, corpus_entry):
     return result
 
 
-def corpus_test(corpus_path, results_path, sha):
+def corpus_test(corpus_path, results_path, sha, regenerate_results):
     """
+    Test the minifier on the entire corpus
+
+    The results are written to a csv file in the results directory.
+    The name of the file is results_<python_version>_<sha>.csv
+
+    If the file already exists and regenerate_results is False, the test is skipped.
+
     :param str corpus_path: Path to the corpus
     :param str results_path: Path to the results directory
     :param str sha: The python-minifier sha we are testing
+    :param bool regenerate_results: Regenerate results even if they are present
     """
     corpus_entries = os.listdir(corpus_path)
 
     python_version = '.'.join([str(s) for s in sys.version_info[:2]])
     results_file_path = os.path.join(results_path, 'results_' + python_version + '_' + sha + '.csv')
 
+    if os.path.isfile(results_file_path) and not regenerate_results:
+        print('Results file already exists: %s', results_file_path)
+        return
+
     with ResultWriter(results_file_path) as result_writer:
         for entry in corpus_entries:
             print(entry)
-            result = minify_test(corpus_path, entry)
+            result = minify_corpus_entry(corpus_path, entry)
             result_writer.write(result)
 
 
+def bool_parse(value):
+    return value == 'true'
+
+
 def main():
     parser = argparse.ArgumentParser(description='Test python-minifier on a corpus of Python files.')
     parser.add_argument('corpus_dir', type=str, help='Path to corpus directory', default='corpus')
     parser.add_argument('results_dir', type=str, help='Path to results directory', default='results')
     parser.add_argument('minifier_sha', type=str, help='The python-minifier sha we are testing')
+    parser.add_argument('regenerate_results', type=bool_parse, help='Regenerate results even if they are present', default='false')
     args = parser.parse_args()
 
-    corpus_test(args.corpus_dir, args.results_dir, args.minifier_sha)
+    corpus_test(args.corpus_dir, args.results_dir, args.minifier_sha, args.regenerate_results)
 
 
 if __name__ == '__main__':