Adapt CI to use pixi

chrisburr · chrisburr · commit 84abe062d5d7 · 2025-05-08T22:46:33.000+02:00
diff --git a/.github/workflows/extensions.yml b/.github/workflows/extensions.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - gubbins
   pull_request:
     branches:
       - main
@@ -15,59 +14,6 @@ defaults:
     shell: bash -el {0}
 
 jobs:
-  unittest:
-    name: Unit test - ${{ matrix.package }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        # In principle, the dependencies could be limited to the extension packages.
-        # However, we want to use the local packages, and not those published on pypi
-        include:
-          - package: "./extensions/gubbins/gubbins-core"
-            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./diracx-core"
-          - package: "./extensions/gubbins/gubbins-db"
-            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core"
-          - package: "./extensions/gubbins/gubbins-logic"
-            dependencies: "./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-db ./diracx-core ./diracx-logic"
-          - package: "./extensions/gubbins/gubbins-routers"
-            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-logic ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-logic ./diracx-core ./diracx-routers"
-          - package: "./extensions/gubbins/gubbins-client"
-            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core"
-          - package: "./extensions/gubbins/gubbins-cli"
-            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-cli ./diracx-client ./diracx-core ./diracx-api"
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v2
-        with:
-          # TODO: Use a conda environment file used for the diracx/base container image
-          environment-name: test-env
-          create-args: >-
-            python=3.11
-            m2crypto
-            python-gfal2
-            mypy
-            pip
-          init-shell: bash
-          post-cleanup: 'all'
-      - name: Set up environment
-        run: |
-          pip install pytest-github-actions-annotate-failures
-          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ${{ matrix.dependencies }} ${{ matrix.package }}[types]
-      - name: Run mypy
-        run: |
-          mypy ${{ matrix.package }}/src
-      - name: Run pytest
-        if: ${{ matrix.package != './extensions/gubbins/gubbins-logic' }}
-        run: |
-          cd ${{ matrix.package }}
-          pip install .[testing]
-          pytest --cov-report=xml:coverage.xml --junitxml=report.xml
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v5.4.2
-
 
   build-wheels:
     name: Build wheels
@@ -178,16 +124,9 @@ jobs:
           docker image ls -a
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v2
+      - uses: prefix-dev/setup-pixi@v0.8.3
         with:
-          environment-file: environment.yml
-          init-shell: bash
-          post-cleanup: 'all'
-      - name: Set up environment
-        run: |
-          pip install pytest-github-actions-annotate-failures
-          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-logic/[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing]
+          environments: default-gubbins
       - name: Start demo
         run: |
           git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
@@ -221,7 +160,7 @@ jobs:
       - name: Run pytest
         run: |
           cd extensions/gubbins
-          pytest --demo-dir=../../../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
+          pixi run pytest-diracx --demo-dir=../../../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
       - name: Collect demo coverage
         run: |
           DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
@@ -245,7 +184,6 @@ jobs:
             ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")"
           done
 
-
           mkdir -p "${fake_module}/src/gubbins"
           for fn in "${PWD}"/extensions/gubbins/*/src/gubbins/*; do
             ln -sf "${fn}" "${fake_module}/src/gubbins/$(basename "${fn}")"
@@ -260,25 +198,3 @@ jobs:
         uses: codecov/codecov-action@v5.4.2
         with:
           files: ./coverage-pytest.xml,./coverage-demo.xml
-
-  client-generation:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v2
-        with:
-          environment-file: environment.yml
-          init-shell: bash
-          post-cleanup: 'all'
-      - name: Set up environment
-        run: |
-          micromamba install -c conda-forge nodejs pre-commit
-          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-logic/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-logic[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing]
-          npm install -g autorest
-      - name: Run autorest
-        run: |
-          autorest --python --help
-          $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools
-          pytest --no-cov --regenerate-client extensions/gubbins/gubbins-client/tests/test_regenerate.py
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -12,75 +12,50 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-defaults:
-  run:
-    shell: bash -el {0}
-
 jobs:
 
   shellcheck:
     runs-on: ubuntu-latest
-    if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx'
-    timeout-minutes: 30
-
     steps:
-    - uses: actions/checkout@v4
-    - name: Run shellcheck
-      # Excluded codes related to sourcing files
-      #     SC1090: Can't follow non-constant source
-      #     SC1091: Not following sourced file
-      run: |
-        find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.8.3
+        with:
+          environments: shellcheck
+      - name: Run shellcheck
+        run: pixi run shellcheck
 
   unittest:
     name: Unit test - ${{ matrix.package }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - package: "./diracx-core"
-            dependencies: "./diracx-testing"
-          - package: "./diracx-db"
-            dependencies: "./diracx-testing ./diracx-core"
-          - package: "./diracx-logic"
-            dependencies: "./diracx-core ./diracx-db"
-          - package: "./diracx-routers"
-            dependencies: "./diracx-testing ./diracx-core ./diracx-db ./diracx-logic"
-          - package: "./diracx-client"
-            dependencies: "./diracx-testing ./diracx-core"
-          - package: "./diracx-api"
-            dependencies: "./diracx-testing ./diracx-core ./diracx-client"
-          - package: "./diracx-cli"
-            dependencies: "./diracx-testing ./diracx-core ./diracx-client ./diracx-api"
+        package:
+          - diracx-core
+          - diracx-db
+          - diracx-logic
+          - diracx-routers
+          - diracx-client
+          - diracx-api
+          - diracx-cli
+          - gubbins-core
+          - gubbins-db
+          - gubbins-logic
+          - gubbins-routers
+          - gubbins-client
+          - gubbins-api
+          - gubbins-cli
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v2
+      - uses: prefix-dev/setup-pixi@v0.8.3
         with:
-          # TODO: Use a conda environment file used for the diracx/base container image
-          environment-name: test-env
-          environment-file: environment.yml
-          init-shell: bash
-          post-cleanup: 'all'
-      - name: Set up environment
-        run: |
-          pip install pytest-github-actions-annotate-failures
-          # Note: DIRAC will install pretty much everything
-          # from diracx so installing just the dependency may
-          # be a bit useless
-          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ${{ matrix.dependencies }}
+          environments: ${{ matrix.package }}
       - name: Run pytest
-        if: ${{ matrix.package != './diracx-logic' }}
-        run: |
-          cd ${{ matrix.package }}
-          pip install .[testing]
-          pytest --cov-report=xml:coverage.xml --junitxml=report.xml
-      - name: Run mypy
-        run: |
-          pip install mypy ${{ matrix.package }}[types]
-          mypy ${{ matrix.package }}/src
+        env:
+          EXTRA_PYTEST_ARGS: --cov-report=xml:coverage.xml --junitxml=report.xml
+        run: pixi run unit-${{ matrix.package }}
       - name: Upload coverage report
         uses: codecov/codecov-action@v5.4.2
 
@@ -89,16 +64,9 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v2
+      - uses: prefix-dev/setup-pixi@v0.8.3
         with:
-          environment-file: environment.yml
-          init-shell: bash
-          post-cleanup: 'all'
-      - name: Set up environment
-        run: |
-          pip install pytest-github-actions-annotate-failures
-          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-db/[testing] ./diracx-testing/
+          environments: default
       - name: Start demo
         run: |
           git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
@@ -123,7 +91,7 @@ jobs:
           fi
       - name: Run pytest
         run: |
-          pytest --demo-dir=../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
+          pixi run pytest-diracx --demo-dir=../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
       - name: Collect demo coverage
         run: |
           DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
@@ -155,22 +123,15 @@ jobs:
 
   client-generation:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [diracx, gubbins]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v2
+      - uses: prefix-dev/setup-pixi@v0.8.3
         with:
-          environment-file: environment.yml
-          init-shell: bash
-          post-cleanup: 'all'
-      - name: Set up environment
-        run: |
-          micromamba install -c conda-forge nodejs pre-commit
-          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/ ./diracx-api/ ./diracx-cli/ -e ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-db/ ./diracx-testing/
-          npm install -g autorest
+          environments: ${{ matrix.package }}-generate-client
       - name: Run autorest
-        run: |
-          autorest --python --help
-          $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools
-          pytest --no-cov --regenerate-client diracx-client/tests/test_regenerate.py
+        run: pixi run -e ${{ matrix.package }}-generate-client generate-client
