Merge branch 'main' into fix/documentation-template

Author: Nicoretti

.github/workflows/checks.yml

@@ -40,14 +40,17 @@ jobs:
         run: |
           poetry run python -m nox -s docs:build
 
+  build-matrix:
+    name: Generate Build Matrix
+    uses: ./.github/workflows/matrix-python.yml
+
   Lint:
     name: Linting (Python-${{ matrix.python-version }})
-    needs: [ Version-Check ]
+    needs: [ Version-Check, build-matrix ]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+      matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
 
     steps:
       - name: SCM Checkout
@@ -70,12 +73,11 @@ jobs:
 
   Type-Check:
     name: Type Checking (Python-${{ matrix.python-version }})
-    needs: [ Version-Check ]
+    needs: [ Version-Check, build-matrix ]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+      matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
 
     steps:
       - name: SCM Checkout
@@ -91,12 +93,11 @@ jobs:
 
   Security:
     name: Security Checks (Python-${{ matrix.python-version }})
-    needs: [ Version-Check ]
+    needs: [ Version-Check, build-matrix ]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+      matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
 
     steps:
       - name: SCM Checkout
@@ -134,16 +135,14 @@ jobs:
         run: poetry run nox -s project:format
 
   Tests:
-    name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
-    needs: [ Documentation, Lint, Type-Check, Security, Format]
+    name: Unit-Tests (Python-${{ matrix.python-version }})
+    needs: [ Documentation, Lint, Type-Check, Security, Format, build-matrix ]
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN ||  secrets.GITHUB_TOKEN }}
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
-        exasol-version: [ "7.1.9" ]
+      matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
 
     steps:
       - name: SCM Checkout
@@ -155,7 +154,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Run Tests and Collect Coverage
-        run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }}
+        run: poetry run nox -s test:unit -- -- --coverage
 
       - name: Upload Artifacts
         uses: actions/[email protected]

.github/workflows/matrix-all.yml

@@ -0,0 +1,30 @@
+name: Build Matrix (All Versions)
+
+on:
+  workflow_call:
+    outputs:
+      matrix:
+        description: "Generates the all versions build matrix"
+        value: ${{ jobs.all_versions.outputs.matrix }}
+
+jobs:
+  all_versions:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: ./.github/actions/python-environment
+
+      - name: Generate matrix
+        run: poetry run nox -s matrix:all
+
+      - id: set-matrix
+        run: |
+          echo "matrix=$(poetry run nox -s matrix:all)" >> $GITHUB_OUTPUT
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}

.github/workflows/matrix-exasol.yml

@@ -0,0 +1,30 @@
+name: Build Matrix (Exasol)
+
+on:
+  workflow_call:
+    outputs:
+      matrix:
+        description: "Generates the exasol version build matrix"
+        value: ${{ jobs.exasol_versions.outputs.matrix }}
+
+jobs:
+  exasol_versions:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: ./.github/actions/python-environment
+
+      - name: Generate matrix
+        run: poetry run nox -s matrix:exasol
+
+      - id: set-matrix
+        run: |
+          echo "matrix=$(poetry run nox -s matrix:exasol)" >> $GITHUB_OUTPUT
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}

.github/workflows/matrix-python.yml

@@ -0,0 +1,30 @@
+name: Build Matrix (Python)
+
+on:
+  workflow_call:
+    outputs:
+      matrix:
+        description: "Generates the python version build matrix"
+        value: ${{ jobs.python_versions.outputs.matrix }}
+
+jobs:
+  python_versions:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: ./.github/actions/python-environment
+
+      - name: Generate matrix
+        run: poetry run nox -s matrix:python
+
+      - id: set-matrix
+        run: |
+          echo "matrix=$(poetry run nox -s matrix:python)" >> $GITHUB_OUTPUT
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}

.github/workflows/slow-checks.yml

@@ -8,8 +8,13 @@ on:
 
 jobs:
 
+  build-matrix:
+    name: Generate Build Matrix
+    uses: ./.github/workflows/matrix-all.yml
+
   Tests:
     name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
+    needs: [ build-matrix ]
     runs-on: ubuntu-latest
     # Even though the environment "manual-approval" will be created automatically,
     # it still needs to be configured to require interactive review.
@@ -19,9 +24,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN ||  secrets.GITHUB_TOKEN }}
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
-        exasol-version: [ "7.1.9" ]
+      matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
 
     steps:
       - name: SCM Checkout

doc/changes/unreleased.md

@@ -1,5 +1,42 @@
 # Unreleased
 
+## 🚨 Breaking Changes
+* **Matrices in CI/CD workflows will be generated automatically now**
+
+    Make sure you have installed all the latest workflow files, especially the newly added ones:
+
+    - `matrix-all.yml`
+    - `matrix-python.yml`
+    - `matrix-exasol.yml`
+
+
+## ✨ Added
+* Added support for dynamically generated workflow matrices.
+
+    This feature allows you to easily change the test matrices in one place: `noxconfig.py`.
+
+    Note: As usual, there are different ways a user can adjust or change the behavior. In the case of the build matrices, there are three obvious ways:
+
+    - Set the appropriate fields in the `noxconfig.py` project configuration (`PROJECT_CONFIG`):
+        * `python_versions = [ ... ]`
+        * `exasol_versions = [ ... ]`
+    - Overwrite the nox tasks:
+        * `matrix:all`
+        * `matrix:python`
+        * `matrix:exasol`
+    - Overwrite/replace the matrix generation workflows:
+        * `matrix-all.yml`
+        * `matrix-python.yml`
+        * `matrix-exasol.yml`
+
+    Among all of the above, the safest way is to set the matrix-related fields in your project config object in `noxconfig.py`.
+    
+    
+## 📚 Documentation
+
+* Added new entries to the frequently asked questions regarding `multiversion documentation`
+
+
 ## 🐞 Fixed
 
 * Fixed `index.rst` documentation template to provide the correct underlining length of the main heading

doc/faq.rst

@@ -23,3 +23,43 @@ Duplicated label error when building documentation
 --------------------------------------------------
 
 Similar error to :code:`Warning, treated as error: integration-test-docker-environment/doc/changes/changes_0.10.0.md:5:duplicate label summary, other instance in integration-test-docker-environment/doc/changes/changes_0.1.0.md'`, might be caused by sphinx extension `sphinx.ext.autosectionlabel`. Try to remove this extension in `doc/conf.py`.
+
+
+.. _faq_multiversion_build_warnings:
+
+Warning while building multiversion documentation
+--------------------------------------------------
+When running ``nox -s docs:multiversion``, I receive the following warnings during the build:
+
+.. code-block::
+
+    WARNING: unknown config value 'smv_metadata_path' in override, ignoring
+    WARNING: unknown config value 'smv_current_version' in override, ignoring
+
+If you receive the warnings above, it is very likely that the multiversion extension is not configured in your Sphinx configuration (``conf.py``). Try adding it to your configuration and rerun the build.
+
+.. code-block:: python
+
+    extensions = [
+        ...,
+        ...,
+        "exasol.toolbox.sphinx.multiversion",
+    ]
+
+
+.. _faq_multiversion_selection_missing:
+
+Missing Version Selection Box in Multiversion Documentation
+------------------------------------------------------------
+
+I have run ``nox -s docs:multiversion``, but I still do not see any version selection box in the upper right corner before the GitHub symbol.
+
+This is likely due to :ref:`faq_multiversion_build_warnings`
+
+
+.. _faq_multiversion_limited_versions:
+
+Limited Previous Versions in Multiversion Documentation
+-------------------------------------------------------
+
+If not all previous versions of the project are available via the version selection box of the multiversion documentation, it is likely due to the fact that the unavailable documentation for those versions was not in a compatible format (there hasn't been a compatible setup of a Sphinx-based documentation).

exasol/toolbox/nox/_ci.py

@@ -0,0 +1,59 @@
+import json
+import logging
+
+import nox
+from nox import Session
+
+from noxconfig import (
+    PROJECT_CONFIG,
+    Config,
+)
+
+_log = logging.getLogger(__name__)
+
+_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"]
+_EXASOL_VERSIONS = ["7.1.9"]
+
+
+def _python_matrix(config: Config):
+    attr = "python_versions"
+    python_versions = getattr(config, attr, _PYTHON_VERSIONS)
+    if not hasattr(config, attr):
+        _log.warning(
+            "Config does not contain '%s' setting. Using default: %s",
+            attr,
+            _PYTHON_VERSIONS,
+        )
+    return {"python-version": python_versions}
+
+
+def _exasol_matrix(config: Config):
+    attr = "exasol_versions"
+    exasol_versions = getattr(config, attr, _EXASOL_VERSIONS)
+    if not hasattr(config, attr):
+        _log.warning(
+            "Config does not contain '%s' setting. Using default: %s",
+            attr,
+            _EXASOL_VERSIONS,
+        )
+    return {"exasol-version": exasol_versions}
+
+
+@nox.session(name="matrix:python", python=False)
+def python_matrix(session: Session) -> None:
+    """Output the build matrix for Python versions as JSON."""
+    print(json.dumps(_python_matrix(PROJECT_CONFIG)))
+
+
+@nox.session(name="matrix:exasol", python=False)
+def exasol_matrix(session: Session) -> None:
+    """Output the build matrix for Exasol versions as JSON."""
+    print(json.dumps(_exasol_matrix(PROJECT_CONFIG)))
+
+
+@nox.session(name="matrix:all", python=False)
+def full_matrix(session: Session) -> None:
+    """Output the full build matrix for Python & Exasol versions as JSON."""
+    matrix = _python_matrix(PROJECT_CONFIG)
+    matrix.update(_exasol_matrix(PROJECT_CONFIG))
+    print(json.dumps(matrix))

exasol/toolbox/nox/tasks.py

@@ -59,14 +59,20 @@ def check(session: Session) -> None:
     clean_docs,
     open_docs,
 )
-from exasol.toolbox.nox._release import prepare_release
 from exasol.toolbox.nox._shared import (
     Mode,
     _context,
     _version,
     python_files,
 )
 
+from exasol.toolbox.nox._ci import (
+    python_matrix,
+    exasol_matrix,
+    full_matrix,
+)
+
+from exasol.toolbox.nox._release import prepare_release
 
 # isort: on
 # fmt: on