Merge pull request #181 from rocky/test_crossversion

Crossversion Testing wip update.

Author: 2elli

test_crossversion/Makefile

@@ -1,4 +1,4 @@
-.PHONY: help clean get_sources setup_pyenv compile prepare test 
+.PHONY: help clean get_sources setup_pyenv setup_uv compile prepare .prepare_msg test .mkdirs
 
 SOURCE=./templates/source/
 COMPILED=./templates/compiled/
@@ -29,18 +29,23 @@ clean:
 	rm -rf $(COMPILED)/*
 	rm -rf $(SERIALIZED)/*
 
+.mkdirs:
+	mkdir -p "$(SOURCE)" "$(COMPILED)" "$(SERIALIZED)"
+
 #: copy all .py files in ./ -> ./templates/source/
 get_sources:
-	cp -f *.py $(SOURCE)
+	cp $(shell find . -name "*.py" | head -n 10 | xargs) $(SOURCE)
 
 .python-version:
 	tox --listenvs | xargs pyenv local
+
 #: setup local pyenv versions to be used by tox
-setup_pyenv: .python-version
+setup_pyenv: .mkdirs .python-version
 
 #: setup tox-uv so tox can use uv
-setup_uv:
+setup_uv: .mkdirs
 	uv tool install tox --with tox-uv
+	tox --listenvs | xargs uv python install
 
 #: with each tox env, compile all sources in ./templates/source/ to ./templates/compiled/, then serialize with dis to ./templates/serialized/
 compile:

test_crossversion/USAGE.md

@@ -1,11 +1,11 @@
 # Automated crossversion testing
 This testing suite is used for automatic testing of differences found between xdis and dis.
-This is done by having a way to identically "serialize" important attributes in xdis and dis bytecodes.
-We then can check a diff between a serialized xdis and dis bytecode to find if xdis is parsing something incorrectly.
+This is done by having a way to identically "serialize" important attributes in bytecodes with xdis and dis.
+Using a diff between a bytecode serialized with xdis and dis, we can find if xdis is parsing something incorrectly.
 Most tests should be ran using the makefile.
 
 # Parsing results
-When running `make test`, tox will serialize bytecode to be put into text form. Given a bytecode compiled in 3.11 and natively disassembled, we go through each of our test versions, say 3.9 ... 3.14, and disassemble the same 3.11 bytecode.
+When running `make test`, two main steps take place. 1st, we compile and serialize bytecode in each target version. 2nd, we disasm the same bytecode using xdis, serialize again, and check the diff. In other words, given a bytecode compiled in 3.11 and natively disassembled (dis), we go through each of our test versions, say 3.9 ... 3.14, and disassemble the same 3.11 bytecode.
 Given the 3.11 serialized disasembly, disassembled from 3.11, we take the diff between that and a serialized 3.11 disassembled from any of our test versions.
 This lets us see if there are any differences in how xdis handles native vs cross version.
 
@@ -22,18 +22,20 @@ for native in test_vers:
 
 Pytest will fail early, so not all tests may be ran.
 
-# System Requirements
+# Usage
+## Requirements
 - `uv`
     - uv will handle everything on its own
+run `make setup_uv`
 
 ---OR---
 
 - `pyenv` and `pyenv-virtualenv`
     - Each version needing to be tested should be installed with pyenv
 - `tox`
+run `make setup_pyenv`
 
-# Usage
-## Makefile
+## Running tests
 Run `make` or `make help` to show the help menu for running and preparing tests, or with `remake`, `remake --tasks`.
 
 To simply run tests, `make test` will copy some sources, prepare template files, and run tests.

test_crossversion/test_xdis.py

@@ -28,8 +28,9 @@ def __init__(self, pyc: Path, serialized_txt: Path) -> None:
         # read serialized bytecode
         self.serialized_dis = serialized_txt.read_text()
         self.serialized_xdis = serialize_pyc(pyc, use_xdis=True, output_file=None)
-        # debug message
+        # debug messages
         self.message = f"{SYS_VERSION}: Checking equivalence: {self.pyc_path} <---> {self.serialized_txt_path}"
+        self.fail_message = f"Running version {SYS_VERSION}, failed equivalence; xdis:{self.pyc_path.name} != dis:{self.serialized_txt_path.name}"
 
     def __str__(self) -> str:
         return self.message
@@ -64,12 +65,12 @@ def get_tests_by_version(v: str) -> Iterable[SerializedTestCase]:
 def test_version(version):
     """Test each version in compiled template folder."""
     for case in get_tests_by_version(version):
-        assert case.serialized_dis.splitlines() == case.serialized_xdis.splitlines()
+        assert case.serialized_dis.splitlines() == case.serialized_xdis.splitlines(), case.fail_message
 
 
 ### LESS VERBOSE (fail early) ###
 #@pytest.mark.parametrize(
 #    "case", chain.from_iterable(get_tests_by_version(v) for v in get_versions())
 #)
 #def test_case(case: SerializedTestCase) -> None:
-#    assert case.serialized_dis.splitlines() == case.serialized_xdis.splitlines()
+#    assert case.serialized_dis.splitlines() == case.serialized_xdis.splitlines(), case.fail_message