test: provide more guidance about working with gold files

And make it easier to add new gold tests.

Author: nedbat

tests/gold/README.rst

@@ -9,16 +9,26 @@ these comparisons is in tests/goldtest.py.
 
 If gold tests are failing, you may need to update the gold files by copying the
 current output of the tests into the gold files. When a test fails, the actual
-output is in the tests/actual directory. Do not commit those files to git.
+output is in the tests/actual directory. Those files are ignored by git.
 
-You can run just the failed tests again with::
+There's a Makefile in the html directory for working with gold files and their
+associated support files.  If actual outputs have changed so that comparisons
+are failing, but the new output is correct, you can use "make update-gold" to
+copy the actual output as the new gold files.
+
+If you have added a gold test, you'll need to manually copy the tests/actual
+files to tests/gold.
+
+Once you've copied the actual results to the gold files, or to check your work
+again, you can run just the failed tests again with::
 
     tox -e py39 -- -n 0 --lf
 
 The saved HTML files in the html directories can't be viewed properly without
 the supporting CSS and Javascript files. But we don't want to save copies of
-those files in every subdirectory.  There's a Makefile in the html directory
-for working with the saved copies of the support files.
+those files in every subdirectory. The make target "make complete" in
+tests/gold/html will copy the support file so you can open the HTML files to
+see how they look.
 
 If the output files are correct, you can update the gold files with "make
 update-gold".  If there are version-specific gold files (for example,

tests/gold/html/Makefile

@@ -17,7 +17,7 @@ complete: 	## Copy support files into directories so the HTML can be viewed prop
 clean:		## Remove the effects of this Makefile.
 	@git clean -fq .
 
-update-gold:	## Copy output files from latest tests to gold files.
+update-gold:	## Copy actual output files from latest tests to gold files.
 	@for sub in ../../actual/html/*; do \
 		rsync --verbose --existing --recursive $$sub/ $$(basename $$sub) ; \
 	done ; \

tests/goldtest.py

@@ -46,6 +46,8 @@ def compare(
     """
     __tracebackhide__ = True    # pytest, please don't show me this function.
     assert os_sep("/gold/") in expected_dir
+    assert os.path.exists(actual_dir)
+    os.makedirs(expected_dir, exist_ok=True)
 
     dc = filecmp.dircmp(expected_dir, actual_dir)
     diff_files = _fnmatch_list(dc.diff_files, file_pattern)
@@ -56,9 +58,11 @@ def save_mismatch(f: str) -> None:
         """Save a mismatched result to tests/actual."""
         save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/"))
         os.makedirs(save_path, exist_ok=True)
-        with open(os.path.join(save_path, f), "w") as savef:
+        save_file = os.path.join(save_path, f)
+        with open(save_file, "w") as savef:
             with open(os.path.join(actual_dir, f)) as readf:
                 savef.write(readf.read())
+                print(os_sep(f"Saved actual output to '{save_file}': see tests/gold/README.rst"))
 
     # filecmp only compares in binary mode, but we want text mode.  So
     # look through the list of different files, and compare them

tests/test_goldtest.py

@@ -13,7 +13,7 @@
 from tests.coveragetest import CoverageTest, TESTS_DIR
 from tests.goldtest import compare, gold_path
 from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain
-from tests.helpers import re_line, remove_tree
+from tests.helpers import os_sep, re_line, remove_tree
 
 GOOD_GETTY = """\
 Four score and seven years ago our fathers brought forth upon this continent, a
@@ -73,6 +73,10 @@ def test_bad(self) -> None:
         assert "+ Five score" in stdout
         assert re_line(rf"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout)
         assert re_line(rf"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout)
+        assert (
+            os_sep(f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst")
+            in os_sep(stdout)
+        )
         assert "  D/D/D, Gxxx, Pennsylvania" in stdout
 
         # The actual file was saved.