Refactor conformance test case ingestion (cloud-custodian#131)

* Refactor conformance test ingestion -- eliminate an old, clumsy process and replace with a single, clean operation.

* Convert CI from poetry to uv

* Refine dependencies, links, and code style per code review

* Add specific failure code to noqa comments

* Use wip.toml to specify test cases that must be marked @wip because the Python implementation lags behind the baseline Golang implementation.

* Update docs and help for dev workflows

* regenerate conformance tests with whitespace tweaks and @wip flags

* regerenate docs with updates

* fix Python 3.9 type union complaints

* Move conformance test config docstring

Author: hudlow

.github/workflows/ci.yaml

@@ -12,15 +12,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+
+      - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ env.DEFAULT_PY_VERSION }}
-      - name: Linting
-        env:
-          RUFF_OUTPUT_FORMAT: github
-        run: |
-          pip install ruff
-          ruff check src
+
+      - run: uv run tox -e lint
 
   Tests:
     needs: Lint
@@ -41,48 +38,26 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Install poetry
-        shell: bash
-        run: pipx install poetry==${{ env.POETRY_VERSION }}
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+      - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Test Runner
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-
-      - name: Install Deps
-        run: |
-          tox -e ${{ matrix.tox-target }} --notest
-
-      - name: Test
-        run: |
-          tox -e ${{ matrix.tox-target }}
+      - run: uv run tox -e ${{ matrix.tox-target }}
 
   Coverage:
     needs: Tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install poetry
-        shell: bash
-        run: pipx install poetry==${{ env.POETRY_VERSION }}
 
-      - uses: actions/setup-python@v1
+      - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ env.DEFAULT_PY_VERSION }}
 
-      - name: Install Test Runner
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
+      - run: uv run tox -e lint
 
-      - name: Upload Code Coverage
-        uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v4
         with:
           files: ./coverage.xml
           name: codecov

Makefile

@@ -13,45 +13,107 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-# Tools require GO. Typically, the following kinds of setup is required.
-#        export PATH="/usr/local/go/bin:/usr/local/bin:$PATH"
-#        export GOPATH="~/go"
+HELP_TEXT := ""
+HELP_TEXT += "Available commands:\n"
 
+.PHONY: help
+help:
+	@echo $(HELP_TEXT)
+
+HELP_TEXT += "  build         runs ``uv build`` to create a distribution kit\n"
+.PHONY: build
 build:
 	uv build
 
-install-tools:
-	cd tools && docker pull golang && docker build -t mkgherkin .
+HELP_TEXT += "  build-all     alias of build, conformance, and docs\n"
+.PHONY: build-all
+build-all:
+	$(MAKE) build
+	$(MAKE) conformance
+	$(MAKE) docs
 
-test:
-	cd features && $(MAKE) all
-	tox run -e py312
+HELP_TEXT += "\n"
 
-test-all:
-	cd features && $(MAKE) all
-	tox run
+HELP_TEXT += "  test          runs the Python 3.12 test environment to execute a quick test\n"
+.PHONY: test
+test: conformance
+	uv run tox run -e py312
+
+HELP_TEXT += "  test-all      runs the full test suite\n"
+.PHONY: test-all
+test-all: conformance
+	uv run tox run
 
-test-wip:
+HELP_TEXT += "  test-<env>    runs tests for any of the available tox environments:\n"
+HELP_TEXT += "\n$$(uv run tox list | awk '{ print "     " $$0 }')\n"
+.PHONY: test-%
+test-%: conformance
+	uv run tox run -e $*
+
+HELP_TEXT += "\n"
+
+HELP_TEXT += "  conformance   generates conformance tests\n"
+.PHONY: conformance
+conformance:
 	cd features && $(MAKE) all
-	tox run -e wip
 
-test-tools:
-	tox run -e tools
-	cd features && $(MAKE) scan
+HELP_TEXT += "  conf-clean    cleans generated conformance tests\n"
+.PHONY: conf-clean
+conf-clean:
+	cd features && $(MAKE) clean
+
+HELP_TEXT += "\n"
 
+HELP_TEXT += "  docs          generates HTML documentation\n"
+.PHONY: docs
 docs: $(wildcard docs/source/*.rst)
-	PYTHONPATH=src python -m doctest docs/source/*.rst
-	export PYTHONPATH=$(PWD)/src:$(PWD)/tools && cd docs && $(MAKE) html
+	$(MAKE) docs-lint
+	cd docs && $(MAKE) html
 
+HELP_TEXT += "  docs-clean    lints documentation sources\n"
+.PHONY: docs-lint
+docs-lint:
+	uv run python -m doctest docs/source/*.rst
+
+HELP_TEXT += "  docs-clean    cleans generated HTML documentation\n"
+.PHONY: docs-clean
+docs-clean:
+	cd docs && $(MAKE) clean
+
+HELP_TEXT += "\n"
+
+HELP_TEXT += "  lint          runs code coverage, type hint checking, and other lint checks\n"
+HELP_TEXT += "                (alias of test-lint)\n"
+.PHONY: lint
 lint:
-	tox run -e lint
+	uv run tox run -e lint
 
+HELP_TEXT += "\n"
+
+HELP_TEXT += "  format        runs code formatting\n"
+.PHONY: format
+format:
+	uv run ruff format src tools
+
+HELP_TEXT += "  coverage      generates code coverage reports\n"
+.PHONY: coverage
 coverage:
-	coverage report -m
+	uv run tox run -e coverage
 
+HELP_TEXT += "  clean         cleans all content ignored by git\n"
+.PHONY: clean
 clean:
-	rm -rf .tox .Python bin include lib pip-selfcheck.json
+	git clean -Xfd
+
+HELP_TEXT += "  clean-all     alias of clean, conformance-clean, and docs-clean\n"
+.PHONY: clean-all
+clean-all:
+	$(MAKE) clean
+	$(MAKE) test-clean
+	$(MAKE) docs-clean
 
+HELP_TEXT += "  benchmarks    runs performance benchmarks\n"
+.PHONY: benchmarks
 benchmarks:
-	PYTHONPATH=src python benches/large_resource_set.py TagAssetBenchmark
-	PYTHONPATH=src python benches/complex_expression.py
+	PYTHONPATH=src uv run python benches/large_resource_set.py TagAssetBenchmark
+	PYTHONPATH=src uv run python benches/complex_expression.py

docs/Makefile

@@ -7,10 +7,12 @@
 # You can set these variables from the command line, and also
 # from the environment for the first two.
 SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
+SPHINXBUILD   ?= uv run sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build
 
+export GRAPHVIZ_DOT ?= $(shell which dot)
+
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@@ -20,4 +22,6 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
+	@which java || (>&2 echo "Building docs requires java" ; exit 1)
+	@which dot || (>&2 echo "Building docs requires graphviz (specifically, the dot executable)" ; exit 1)
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)