Update CI/CD workflows, setup config files, and build wheels for Python >= 3.8 (#524)

* Update CI/CD workflows

* disable cython for 3.12 and pypy3.9

* forgot to disable tests for 3.7

* properly disable cython

* syntax is annoying

* we should only test with orjson if we have cython installed

* allow experimentals to fail

* a test should take no longer than ten minutes

* i guess we can't use continue-on-error that way then

* test cibuildwheel

* revert dist.yml changes

* disable testing for 3.12 until properly supported in August or so

* disable scripts/coverage call in scripts/tests because we already do it in a separate job

* remove old tests and disable timeout

* dsiable integ

* try skipping meticulous tests

* disable pypy tests for now

* fix typo

* extend timeout

* switch over setup.cfg to pyproject.toml

* fix linting

* flake8 is impossible to please apparently

* update gh-pages.yml

* fix all yamls

* fix requirements seutp

* cleanup

* should continue on false

* fix continue on error placement

* check linting only

* Update python-package.yml to use experimental

* Update python-package.yml

* Update tests to run meticulous again

* Update python-package.yml to allow 3.12 to fail

* just disable 3.12 for now

* Update python-package.yml to not test pypy until we figure this out

* Update python-package.yml to extend job timeout to 10m

* consolidate dist.yml into python-package.yml

* update pyproject.toml with proper cibuildwheel config

* forgot fetch-depth for wheel builds

* reformat

* skip builds for musllinux_x86_64

* enable check requirement

* revert drastic changes for linting

* pull in new changes from master

Author: wbarnha

.github/workflows/codeql-analysis.yml

@@ -1,3 +1,4 @@
+---
 # For most projects, this workflow file will not need changing; you simply need
 # to commit it to your repository.
 #
@@ -9,17 +10,15 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: "CodeQL"
-
+name: CodeQL
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [master]
   schedule:
-    - cron: '19 10 * * 6'
-
+    - cron: 19 10 * * 6
 jobs:
   analyze:
     name: Analyze
@@ -28,33 +27,31 @@ jobs:
       actions: read
       contents: read
       security-events: write
-
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'python' ]
+        language: [python]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -66,6 +63,5 @@ jobs:
     #- run: |
     #   make bootstrap
     #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/dist.yml

@@ -1,47 +1,47 @@
+---
 name: Pages
-
 on:
   push:
-    branches: ["master"]
+    branches: [master]
   pull_request:
-    branches: ["master"]
+    branches: [master]
   release:
     types: [created]
-    branches:
-      - 'master'
-
+    branches: [master]
 jobs:
   build:
-    name: "Build docs"
+    name: Build docs
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/setup-python@v4
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
-    - name: "Install runtime dependencies"
-      run: "scripts/install"
-    - name: "Install doc build deps and build with Sphinx"
-      run: make docs
-    - name: "Upload artifacts"
-      uses: actions/upload-pages-artifact@v1
-      with:
+      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # otherwise, you will failed to push refs to dest repo
+      - name: Install runtime dependencies
+        run: |
+          pip install .
+          pip install -r requirements/docs.txt
+      - name: Install doc build deps and build with Sphinx
+        run: make docs
+      - name: Upload artifacts
+        uses: actions/upload-pages-artifact@v1
+        with:
         # Upload built docs
-        path: "./Documentation"
+          path: ./Documentation
   deploy:
-    name: "Deploy docs"
+    name: Deploy docs
     if: github.event_name == 'release' && github.event.action == 'created'
     needs: build
     runs-on: ubuntu-latest
     # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
     permissions:
-      pages: write      # to deploy to Pages
-      id-token: write   # to verify the deployment originates from an appropriate source
+      pages: write  # to deploy to Pages
+      id-token: write  # to verify the deployment originates from an appropriate source
     # Deploy to the github-pages environment
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
-    - uses: actions/deploy-pages@v2
-      id: deployment
-      name: "Deploy to GitHub Pages"
+      - uses: actions/deploy-pages@v2
+        id: deployment
+        name: Deploy to GitHub Pages

.github/workflows/gh-pages.yml

@@ -1,44 +1,143 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: Python package
-
+---
+name: Test Python library
 on:
   push:
-    branches: ["master"]
+    branches: [master]
   pull_request:
-    branches: ["master"]
+    branches: [master]
+  release:
+    types: [created]
+    branches: [master]
+env:
+  FORCE_COLOR: '1'  # Make tools pretty.
+  PIP_DISABLE_PIP_VERSION_CHECK: '1'
+  PIP_NO_PYTHON_VERSION_WARNING: '1'
+  PYTHON_LATEST: '3.11'
 
+  # For re-actors/checkout-python-sdist
+  sdist-artifact: python-package-distributions
 jobs:
-  tests:
-    name: "Python ${{ matrix.python-version }}/Cython: ${{ matrix.use-cython }}"
-    runs-on: "ubuntu-latest"
-
+  lint:
+    name: Check linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_LATEST }}
+          cache: pip
+      - run: |
+          python -m pip install build
+          python -m pip install -r requirements/test.txt
+        name: Install core libraries for build and install
+      - name: Run linting checks
+        run: scripts/check
+  test-pytest:
+    name: 'Python ${{ matrix.python-version }}/Cython: ${{ matrix.use-cython }}'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10  # Maybe we should remove this someday but the PyPy tests are acting strange
     strategy:
       # Complete all jobs even if one fails, allows us to see
       # for example if a test fails only when Cython is enabled
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        use-cython: ["true", "false"]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+        use-cython: ['true', 'false']
+        experimental: [false]
+#         include:
+#           - python-version: pypy3.9
+#             experimental: true
+#             use-cython: 'false'
+#           - python-version: ~3.12.0-0
+#             experimental: true
+#             use-cython: 'false'
     env:
       USE_CYTHON: ${{ matrix.use-cython }}
-
+    continue-on-error: ${{ matrix.experimental }}
     steps:
-      - uses: "actions/checkout@v3"
-        with:
-            # You need to include this or setuptools_scm in GitHub runners won't detect the version
-            fetch-depth: 0
-      - uses: "actions/setup-python@v4"
-        with:
-          python-version: "${{ matrix.python-version }}"
-      - name: "Install dependencies"
-        run: "scripts/install"
-      - name: "Run linting checks"
-        run: "scripts/check"
-      - name: "Run tests"
-        run: "scripts/tests"
-      - name: "Enforce coverage"
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: requirements/test.txt
+      - name: Install dependencies
+        run: |
+          pip install -r requirements/test.txt
+          pip install .
+      - name: Run tests
+        run: scripts/tests
+      - name: Enforce coverage
         uses: codecov/codecov-action@v3
         with:
-            token: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+  check:  # This job does nothing and is only used for the branch protection
+    name: ✅ Ensure the required checks passing
+    if: always()
+    needs: [lint, test-pytest]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+  build_wheels:
+    name: 📦 Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: check
+    if: github.event_name == 'release' && github.event.action == 'created'
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, macos-11]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.10.1
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+  build_sdist:
+    name: 📦 Build the source distribution
+    runs-on: ubuntu-latest
+    needs: check
+    if: github.event_name == 'release' && github.event.action == 'created'
+    steps:
+      - uses: actions/checkout@v3
+        name: Checkout source repository
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+      - name: Build sdist
+        run: >
+          pip3 install pkgconfig cython --upgrade &&
+          python3 setup.py sdist
+      - uses: actions/upload-artifact@v3
+        name: Upload build artifacts
+        with:
+          path: dist/*.tar.gz
+  publish:
+    name: 📦 Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build_wheels, build_sdist]
+    permissions:
+      id-token: write
+    environment: pypi
+    if: github.event_name == 'release' && github.event.action == 'created'
+    steps:
+      - name: Download the sdist artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: dist
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}