Fix tests, run tests in CI (#64)

* tests: fix version calculation

This was incorrect on 3.10+

* workflows: ci

* travis: remove old travis CI config

* tests: disable some broken tests

These are seemingly intentionally incompatible with pytest.

* [WIP] experiment with CI

* [WIP] listgen: fix path

* [WIP] more experimenting

* [WIP] listgen: syntax

* [WIP] listgen: disable 2.7 temporarily

* [WIP] disable 3.2

* [WIP]: 3.6+ only

* [WIP] listgen: tweak runner

* [WIP]: 3.7 and newer only

* [WIP] handle accelerator mods

* [WIP] do recursive step correctly

* [WIP] dedupe correctly

* [WIP]: use pkgutil

* [WIP] hackety hack

* [WIP] hackety hack

* [WIP] builtin_module_names

* [WIP] newer python support

* [WIP] fix path

* listgen: auto-PRs for 3.10+

* listgen: fix base branch

* walk-modules: be slightly smarter

* listgen: temporarily disable PR creation

* rewrite tests

* Makefile: enforce 100% coverage

* README: mention the new `sys` APIs

* listgen: run for old Pythons, drop pull_request trigger

* ci: enable caching

* ci: test on 3.10 and 3.11

* base: update docstrings

* test_base: add another consistency test

* tests: test long_versions as well for consistency

* listgen: every tuesday

Author: woodruffw

.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+
+      - name: test
+        run: make test INSTALL_EXTRA=test
+
+#   lint:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - uses: actions/checkout@v3
+#       - uses: actions/setup-python@v4
+#         with:
+#           python-version: "3.10"
+#       - name: lint
+#         run: make lint INSTALL_EXTRA=lint

.github/workflows/listgen.yml

@@ -0,0 +1,115 @@
+name: Generate stdlib lists
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 2"
+
+jobs:
+  pre-list-legacy:
+    strategy:
+      matrix:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          # NOTE: matrix.python is intentionally not used here.
+          python-version: "3.x"
+      - name: support deps
+        run: make dev INSTALL_EXTRA=support
+      - name: build pre-list
+        env:
+          LISTGEN_PYTHON_VERSION: "${{ matrix.python }}"
+        run: |
+          ./env/bin/python ./support/fetch-sphinx.py "${LISTGEN_PYTHON_VERSION}" > pre-list.txt
+      - name: upload pre-list
+        uses: actions/upload-artifact@v3
+        with:
+          name: pre-list-${{ matrix.python }}
+          path: pre-list.txt
+
+  expand-list-legacy:
+    needs: pre-list-legacy
+    strategy:
+      matrix:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: actions/download-artifact@v3
+        with:
+          name: pre-list-${{ matrix.python }}
+      - name: walk modules
+        run: |
+          python -V
+          python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt" < pre-list.txt
+          rm pre-list.txt
+          sort -o "${LISTGEN_PYTHON_VERSION}.txt" "${LISTGEN_PYTHON_VERSION}.txt"
+          mv "${LISTGEN_PYTHON_VERSION}.txt" ./stdlib_list/lists/
+      - name: create PR
+        uses: peter-evans/create-pull-request@v5
+        with:
+          commit-message: "[BOT] update list for ${{ matrix.python }}"
+          branch: update-stdlib-list-${{ matrix.python }}
+          base: main
+          branch-suffix: timestamp
+          title: "[BOT] update list for ${{ matrix.python }}"
+          body: |
+            This is an automated pull request, updating `${{ matrix.python }}.txt` after a detected change.
+
+            Please review manually before merging.
+          assignees: "woodruffw"
+          reviewers: "woodruffw"
+
+  expand-list:
+    strategy:
+      matrix:
+        python:
+          - "3.10"
+          - "3.11"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: walk modules
+        env:
+          LISTGEN_PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          python -V
+          python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt"
+          sort -o "${LISTGEN_PYTHON_VERSION}.txt" "${LISTGEN_PYTHON_VERSION}.txt"
+          mv "${LISTGEN_PYTHON_VERSION}.txt" ./stdlib_list/lists/
+
+      - name: create PR
+        uses: peter-evans/create-pull-request@v5
+        with:
+          commit-message: "[BOT] update list for ${{ matrix.python }}"
+          branch: update-stdlib-list-${{ matrix.python }}
+          base: main
+          branch-suffix: timestamp
+          title: "[BOT] update list for ${{ matrix.python }}"
+          body: |
+            This is an automated pull request, updating `${{ matrix.python }}.txt` after a detected change.
+
+            Please review manually before merging.
+          assignees: "woodruffw"
+          reviewers: "woodruffw"

.travis.yml

@@ -70,7 +70,7 @@ reformat:
 test tests: $(VENV)/pyvenv.cfg
 	. $(VENV_BIN)/activate && \
 		pytest --cov=$(PY_MODULE) $(T) $(TEST_ARGS) && \
-		python -m coverage report -m $(COV_ARGS)
+		python -m coverage report -m $(COV_ARGS) --fail-under 100
 
 .PHONY: doc
 doc: $(VENV)/pyvenv.cfg

Makefile

@@ -4,6 +4,11 @@ This package includes lists of all of the standard libraries for Python 2.6,
 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, and 3.9 along with the code for
 scraping the official Python docs to get said lists.
 
+**IMPORTANT**: If you're on Python 3.10 or newer, you **probably don't need this library**.
+See [`sys.stdlib_module_names`](https://docs.python.org/3/library/sys.html#sys.stdlib_module_names)
+and [`sys.builtin_module_names`](https://docs.python.org/3/library/sys.html#sys.builtin_module_names)
+for similar functionality.
+
 ## Installation
 
 `stdlib-list` is available on PyPI:

README.md

@@ -32,6 +32,9 @@ test = ["pytest", "pytest-cov", "coverage[toml]"]
 lint = ["black", "mypy", "ruff"]
 doc = ["sphinx", "sphinx_rtd_theme"]
 dev = ["build", "stdlib-list[test,lint,doc]"]
+# CI only: used for list generation for Python versions < 3.10.
+support = ["sphobjinv"]
+
 
 [tool.black]
 line-length = 100

pyproject.toml

@@ -8,3 +8,11 @@
     short_versions,
     long_versions,
 )
+
+__all__ = [
+    "stdlib_list",
+    "in_stdlib",
+    "get_canonical_version",
+    "short_versions",
+    "long_versions",
+]

stdlib_list/__init__.py

@@ -6,7 +6,20 @@
 
 from functools import lru_cache
 
-long_versions = ["2.6.9", "2.7.9", "3.2.6", "3.3.6", "3.4.3", "3.5", "3.6", "3.7", "3.8", "3.9"]
+long_versions = [
+    "2.6.9",
+    "2.7.9",
+    "3.2.6",
+    "3.3.6",
+    "3.4.3",
+    "3.5",
+    "3.6",
+    "3.7",
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+]
 
 short_versions = [".".join(x.split(".")[:2]) for x in long_versions]
 
@@ -23,11 +36,10 @@ def get_canonical_version(version):
 def stdlib_list(version=None):
     """
     Given a ``version``, return a ``list`` of names of the Python Standard
-    Libraries for that version. These names are obtained from the Sphinx inventory
-    file (used in :py:mod:`sphinx.ext.intersphinx`).
+    Libraries for that version.
 
     :param str|None version: The version (as a string) whose list of libraries you want
-        (one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, or ``"3.5"``).
+        (formatted as ``X.Y``, e.g. ``"2.7"`` or ``"3.10"``).
 
         If not specified, the current version of Python will be used.
 
@@ -72,7 +84,7 @@ def in_stdlib(module_name, version=None):
 
     :param str|None module_name: The module name (as a string) to query for.
     :param str|None version: The version (as a string) whose list of libraries you want
-        (one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, or ``"3.5"``).
+        (formatted as ``X.Y``, e.g. ``"2.7"`` or ``"3.10"``).
 
         If not specified, the current version of Python will be used.
 

stdlib_list/base.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+# fetch-sphinx.py: retrieve a particular Python version's stdlib list
+# using its hosted Sphinx inventory.
+
+import sys
+import sphobjinv as soi
+
+if __name__ == "__main__":
+    vers = sys.argv[1]
+    inv = soi.Inventory(url=f"https://docs.python.org/{vers}/objects.inv")
+    modules = list(sorted(obj.name for obj in inv.objects if obj.role == "module"))
+    print("\n".join(modules))

support/fetch-sphinx.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+import inspect
+import pkgutil
+import sys
+
+SEEN_MODS = set()
+
+
+def walk_pkgutil(mod_name, mod, io):
+    for pkg in pkgutil.walk_packages(mod.__path__, mod_name + "."):
+        if pkg.name in SEEN_MODS:
+            continue
+        else:
+            # We don't recurse here because `walk_packages` takes care of
+            # it for us.
+            SEEN_MODS.add(pkg.name)
+            print(pkg.name, file=io)
+
+
+def walk_naive(mod_name, mod, io):
+    for attr in dir(mod):
+        attr_obj = getattr(mod, attr, None)
+        # Shouldn't happen, but who knows.
+        if attr_obj is None:
+            continue
+
+        # If this member isn't a module, skip it.
+        if not inspect.ismodule(attr_obj):
+            continue
+
+        # To filter "real" submodules from re-exports, we try
+        # and import the submodule by its qualified name.
+        # If the import fails, we know it's a re-exported module.
+        try:
+            submod_name = mod_name + "." + attr
+            __import__(submod_name)
+            walk(submod_name, io)
+        except ImportError:
+            # ...but sometimes we do want to include re-exports, since
+            # they might be things like "accelerator" modules that don't
+            # appear anywhere else.
+            # For example, `_bz2` might appear as a re-export.
+            try:
+                # Again, try and import to avoid module-looking object
+                # that don't actually appear on disk. Experimentally,
+                # there are a few of these (like "TK").
+                __import__(attr)
+                walk(attr, io)
+            except ImportError:
+                continue
+
+
+def walk(mod_name, io):
+    if mod_name in SEEN_MODS:
+        return
+    else:
+        SEEN_MODS.add(mod_name)
+        print(mod_name, file=io)
+
+    # Try and import it.
+    try:
+        mod = __import__(mod_name)
+
+        if hasattr(mod, "__path__"):
+            walk_pkgutil(mod_name, mod, io)
+        else:
+            walk_naive(mod_name, mod, io)
+
+    except ImportError:
+        pass
+
+
+if __name__ == "__main__":
+    output = sys.argv[1]
+
+    with open(output, mode="w") as io:
+        for mod_name in sys.builtin_module_names:
+            walk(mod_name, io)
+
+        if hasattr(sys, "stdlib_module_names"):
+            for mod_name in sys.stdlib_module_names:
+                walk(mod_name, io)
+        else:
+            for mod_name in sys.stdin:
+                walk(mod_name.rstrip("\n"), io)