WIP: Run CI tests against multiple versions

Author: LucasG0

.github/workflows/ci.yml

@@ -201,8 +201,87 @@ jobs:
           pip install invoke toml codecov
       - name: "Install Package"
         run: "poetry install --all-extras"
+      - name: "Set environment variables for python_testcontainers"
+        run: |
+          echo INFRAHUB_TESTING_IMAGE_TAG=latest >> $GITHUB_ENV
       - name: "Integration Tests"
-        run: "poetry run pytest --cov infrahub_sdk tests/integration/"
+        run: |
+          poetry run pytest --cov infrahub_sdk tests/integration/
+      - name: "Upload coverage to Codecov"
+        run: |
+          codecov --flags integration-tests
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  integration-tests-against-local-infrahub-build:
+    if: |
+      always() && !cancelled() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled') &&
+      needs.files-changed.outputs.python == 'true' &&
+      (github.base_ref == 'stable' || github.base_ref == 'develop')
+    needs: ["files-changed", "yaml-lint", "python-lint"]
+    runs-on:
+      group: "huge-runners"
+    timeout-minutes: 30
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+
+      - name: "Extract target branch name"
+        id: extract_branch
+        run: echo "TARGET_BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV
+
+      - name: "Checkout infrahub repository"
+        uses: "actions/checkout@v4"
+        with:
+          repository: "opsmill/infrahub"
+          path: "infrahub-server"
+          ref: ${{ github.base_ref }}
+          submodules: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: "Setup git credentials prior dev.build"
+        run: |
+          cd infrahub-server
+          git config --global user.name 'Infrahub'
+          git config --global user.email '[email protected]'
+          git config --global --add safe.directory '*'
+          git config --global credential.usehttppath true
+          git config --global credential.helper /usr/local/bin/infrahub-git-credential
+
+      - name: "Set environment variables prior dev.build"
+        run: |
+          echo "INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }}" >> $GITHUB_ENV
+          RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/')
+          echo "PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp" >> $GITHUB_ENV
+          echo "INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "INFRAHUB_TESTING_IMAGE_TAG=local-${{ runner.name }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "INFRAHUB_TESTING_DOCKER_IMAGE=opsmill/infrahub" >> $GITHUB_ENV
+
+      - name: "Build container"
+        run: |
+          cd infrahub-server
+          inv dev.build
+
+      - name: "Setup environment"
+        run: |
+          pipx install poetry==1.8.5
+          poetry config virtualenvs.create true --local
+          pip install invoke toml codecov
+
+      - name: "Install Package"
+        run: "poetry install --all-extras"
+
+      - name: "Integration Tests"
+        run: |
+          echo "Running tests for version: $INFRAHUB_TESTING_IMAGE_TAG"
+          poetry run pytest --cov infrahub_sdk tests/integration/
+
       - name: "Upload coverage to Codecov"
         run: |
           codecov --flags integration-tests

infrahub_sdk/testing/conftest.py

@@ -0,0 +1,18 @@
+import os
+
+import pytest
+
+infrahub_version = os.getenv("INFRAHUB_TESTING_IMAGE_TAG", "latest")
+
+
+def pytest_addoption(parser):  # type: ignore # noqa: ANN001
+    parser.addoption(
+        "--infrahub-version", action="store", default=None, help="Specify the Infrahub version to use for tests"
+    )
+
+
+@pytest.fixture(autouse=True, scope="session")
+def set_infrahub_version(request):  # type: ignore # noqa: ANN001
+    # We want to provide both environment variable and pytest CLI to set infrahub testing version.
+    global infrahub_version
+    infrahub_version = request.config.getoption("--infrahub-version")

infrahub_sdk/testing/docker.py

@@ -1,10 +1,38 @@
+from __future__ import annotations
+
 import pytest
 from infrahub_testcontainers.helpers import TestInfrahubDocker
+from packaging.version import InvalidVersion, Version
 
 from .. import Config, InfrahubClient, InfrahubClientSync
+from .conftest import infrahub_version
+
+
+# TODO test it works
+def skip_version(min_infrahub_version: str | None = None, max_infrahub_version: str | None = None) -> bool:
+    """
+    Check if a test should be skipped depending on infrahub version.
+    """
+    try:
+        version = Version(infrahub_version)
+    except InvalidVersion:
+        # We would typically end up here for development purpose while running a CI test against
+        # unreleased versions of infrahub, like `stable` or `develop` branch.
+        # For now, we consider this means we are testing against the most recent version of infrahub,
+        # so we skip if the test should not be ran against a maximum version.
+        return max_infrahub_version is None
+
+    if min_infrahub_version is not None and version <= Version(min_infrahub_version):
+        return True
+
+    return max_infrahub_version is not None and version <= Version(max_infrahub_version)
 
 
 class TestInfrahubDockerClient(TestInfrahubDocker):
+    @pytest.fixture(scope="class")
+    def infrahub_version(self) -> str:
+        return infrahub_version
+
     @pytest.fixture(scope="class")
     def client(self, infrahub_port: int) -> InfrahubClient:
         return InfrahubClient(

tests/integration/test_infrahub_client.py

@@ -16,10 +16,6 @@
 
 
 class TestInfrahubNode(TestInfrahubDockerClient, SchemaAnimal):
-    @pytest.fixture(scope="class")
-    def infrahub_version(self) -> str:
-        return "1.1.0"
-
     @pytest.fixture(scope="class")
     async def base_dataset(
         self,

tests/integration/test_node.py

@@ -11,10 +11,6 @@
 
 
 class TestInfrahubNode(TestInfrahubDockerClient, SchemaCarPerson):
-    @pytest.fixture(scope="class")
-    def infrahub_version(self) -> str:
-        return "1.1.0"
-
     @pytest.fixture(scope="class")
     async def initial_schema(self, default_branch: str, client: InfrahubClient, schema_base: SchemaRoot) -> None:
         await client.schema.wait_until_converged(branch=default_branch)

tests/integration/test_repository.py

@@ -2,8 +2,6 @@
 
 from typing import TYPE_CHECKING
 
-import pytest
-
 from infrahub_sdk.testing.docker import TestInfrahubDockerClient
 from infrahub_sdk.testing.repository import GitRepo
 from infrahub_sdk.utils import get_fixtures_dir
@@ -13,10 +11,6 @@
 
 
 class TestInfrahubRepository(TestInfrahubDockerClient):
-    @pytest.fixture(scope="class")
-    def infrahub_version(self) -> str:
-        return "1.1.0"
-
     async def test_add_repository(self, client: InfrahubClient, remote_repos_dir):
         src_directory = get_fixtures_dir() / "integration/mock_repo"
         repo = GitRepo(name="mock_repo", src_directory=src_directory, dst_directory=remote_repos_dir)