ci: optimize runs (#448)

Author: nicoloboschi

.github/changes-filter.yaml

@@ -2,7 +2,9 @@
 e2e_tests:
   - "libs/e2e-tests/**"
   - "pyproject.toml"
-  - "**/pyproject.toml"
+  - "libs/llamaindex/pyproject.toml"
+  - "libs/langchain/pyproject.toml"
+  - "libs/tests-utils/pyproject.toml"
   - "./.github/actions/**"
   - "./.github/workflows/ci-e2e-tests.yml"
   - "./.github/workflows/_run_e2e_tests.yml"
@@ -15,8 +17,23 @@ notebooks:
   - "./.github/workflows/_run_e2e_tests.yml"
 integration_tests:
   - "libs/colbert/**"
-  - "libs/**"
-  - "pyproject.toml"
-  - "**/pyproject.toml"
+  - "libs/llamaindex/**"
+  - "libs/langchain/**"
+  - "./.github/actions/**"
+  - "./.github/workflows/ci-unit-tests.yml"
+docker_examples:
+  - "docker/examples/**"
   - "./.github/actions/**"
   - "./.github/workflows/ci-unit-tests.yml"
+libs_colbert:
+  - "libs/colbert/**"
+libs_langchain:
+  - "libs/langchain/**"
+  - "libs/colbert/**"
+libs_llamaindex:
+  - "libs/llamaindex/**"
+  - "libs/colbert/**"
+libs_knowledge_graph:
+  - "libs/knowledge-graph/**"
+libs_knowledge_store:
+  - "libs/knowledge-graph/**"

.github/workflows/ci-security-scans.yml

@@ -9,8 +9,7 @@ on:
       - ".github/workflows/ci-security-scans.yml"
       - "scripts/**"
       - "docker/**"
-      - "pyproject.toml"
-      - "poetry.lock"
+      - "**/pyproject.toml"
     branches:
       - main
 
@@ -38,6 +37,11 @@ jobs:
       - name: Move snyk vulnerabilities file
         if: ${{ always() }}
         run: |
+          # cleanup snyk scan docker resources
+          sudo docker volume prune -f
+          sudo docker system prune -f
+          du -sh *
+
           python -m pip install poetry>=1.7.1
           mkdir -p snyk_reports
           mv snyk-vuln.json snyk_reports/snyk-vuln.json

.github/workflows/ci-unit-tests.yml

@@ -9,9 +9,25 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  preconditions:
+    name: Preconditions
+    runs-on: ubuntu-latest
+    outputs:
+      notebooks: ${{ steps.filter.outputs.notebooks }}
+      e2e_tests: ${{ steps.filter.outputs.e2e_tests }}
+      docker_examples: ${{ steps.filter.outputs.docker_examples }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: ./.github/changes-filter.yaml
+
   build-docker:
     name: Docker
     runs-on: ubuntu-latest
+    needs: ["preconditions"]
+    if: ${{ needs.preconditions.outputs.docker_examples == 'true' }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -33,6 +49,7 @@ jobs:
 
   unit-tests:
     name: Unit Tests (Python ${{ matrix.python-version }})
+    needs: ["preconditions"]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -55,50 +72,47 @@ jobs:
         uses: ./.github/actions/lint
 
       - name: "Unit tests (root)"
-        if: ${{ matrix.python-version != '3.12' }}
         run: tox -e unit-tests && rm -rf .tox
 
       - name: "Unit tests (colbert)"
+        if: ${{ needs.preconditions.outputs.libs_colbert == 'true'}}
         run: tox -e unit-tests -c libs/colbert && rm -rf libs/colbert/.tox
 
       - name: "Unit tests (langchain)"
+        if: ${{ needs.preconditions.outputs.libs_langchain == 'true'}}
         run: tox -e unit-tests -c libs/langchain && rm -rf libs/langchain/.tox
 
       - name: "Unit tests (llamaindex)"
+        if: ${{ needs.preconditions.outputs.libs_llamaindex == 'true'}}
         run: tox -e unit-tests -c libs/llamaindex && rm -rf libs/llamaindex/.tox
 
       - name: "Unit tests (knowledge-store)"
-        if: ${{ matrix.python-version != '3.9' }}
+        if: ${{ needs.preconditions.outputs.libs_knowledge_store == 'true' && matrix.python-version != '3.9' }}
         env:
           OPENAI_API_KEY: "${{ secrets.E2E_TESTS_OPEN_AI_KEY }}"
         run: tox -e unit-tests -c libs/knowledge-store && rm -rf libs/knowledge-store/.tox
 
       - name: "Unit tests (knowledge-graph)"
-        if: ${{ matrix.python-version != '3.9' && matrix.python-version != '3.10' }}
+        if: ${{ needs.preconditions.outputs.libs_knowledge_graph == 'true' && matrix.python-version != '3.9' && matrix.python-version != '3.10' }}
         env:
           OPENAI_API_KEY: "${{ secrets.E2E_TESTS_OPEN_AI_KEY }}"
         run: tox -e unit-tests -c libs/knowledge-graph && rm -rf libs/knowledge-graph/.tox
 
   integration-tests:
     name: Integration Tests
     runs-on: ubuntu-latest
+    if: ${{ needs.preconditions.outputs.integration_tests == 'true' }}
+    needs: ["preconditions"]
     timeout-minutes: 30
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
 
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: ./.github/changes-filter.yaml
-
       - name: "Setup: Python 3.11"
-        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         uses: ./.github/actions/setup-python
 
       - name: Setup AstraDB
         uses: nicoloboschi/setup-astradb@v1
-        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         id: astra-db
         with:
           token: ${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}
@@ -107,7 +121,6 @@ jobs:
           cloud: "gcp"
 
       - name: "Integration tests"
-        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         env:
           ASTRA_DB_TOKEN: "${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}"
           ASTRA_DB_ID: "${{ steps.astra-db.outputs.id }}"
@@ -126,7 +139,7 @@ jobs:
 
       - name: Cleanup AstraDB
         uses: nicoloboschi/cleanup-astradb@v1
-        if: ${{ always() && steps.filter.outputs.integration_tests == 'true' }}
+        if: ${{ always() && steps.astra-db.outputs.name != '' }}
         with:
           token: ${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}
           name: ${{ steps.astra-db.outputs.name }}