ci: improve astra usage and cleanups (#319)

Author: nicoloboschi

.github/workflows/ci-e2e-tests.yml

@@ -2,8 +2,11 @@ name: CI - E2E tests
 on:
   workflow_dispatch: {}
   schedule:
-    # astra dev and dse - every 2 hours
-    - cron: "0 */2 * * *"
+    # keep different crons to reduce the n. of databases at the same time on astra
+    - cron: "0 0/4 * * *"
+    - cron: "0 1/4 * * *"
+    - cron: "0 2/4 * * *"
+    - cron: "0 3/4 * * *"
   pull_request:
     paths-ignore:
       - "scripts/**"
@@ -29,6 +32,10 @@ jobs:
       astradb-prod-region: "us-east-1"
       astradb-prod-cloud: "aws"
       is-scheduled: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+      is-ragstack-dev-cron: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '0 0/4 * * *') }}
+      is-ragstack-latest-release-cron: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '0 1/4 * * *') }}
+      is-langchain-dev-cron: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '0 2/4 * * *') }}
+      is-llamaindex-dev-cron: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '0 3/4 * * *') }}
     steps:
       - uses: actions/checkout@v4
       - name: Run lint
@@ -56,7 +63,7 @@ jobs:
   ragstack-dev-astradev-tests:
     name: "RAGStack dev / AstraDB dev"
     needs: ["preconditions"]
-    if: ${{ needs.preconditions.outputs.is-scheduled == 'true' || needs.preconditions.outputs.e2e_tests == 'true' }}
+    if: ${{ needs.preconditions.outputs.is-ragstack-dev-cron == 'true' || needs.preconditions.outputs.e2e_tests == 'true' }}
     uses: ./.github/workflows/_run_e2e_tests.yml
     secrets: inherit
     with:
@@ -90,7 +97,7 @@ jobs:
   langchain-astradev-tests:
     name: "LangChain dev / AstraDB dev"
     needs: ["preconditions"]
-    if: ${{ needs.preconditions.outputs.is-scheduled == 'true' }}
+    if: ${{ needs.preconditions.outputs.is-langchain-dev-cron == 'true' }}
     uses: ./.github/workflows/_run_e2e_tests.yml
     secrets: inherit
     with:
@@ -124,7 +131,7 @@ jobs:
   llamaindex-astradev-tests:
     name: "LLamaIndex dev / AstraDB dev"
     needs: ["preconditions"]
-    if: ${{ needs.preconditions.outputs.is-scheduled == 'true' }}
+    if: ${{ needs.preconditions.outputs.is-llamaindex-dev-cron == 'true' }}
     uses: ./.github/workflows/_run_e2e_tests.yml
     secrets: inherit
     with:
@@ -158,7 +165,7 @@ jobs:
   ragstack-latest-release-astradev-tests:
     name: "RAGStack latest / AstraDB dev"
     needs: ["preconditions"]
-    if: ${{ needs.preconditions.outputs.is-scheduled == 'true' }}
+    if: ${{ needs.preconditions.outputs.is-ragstack-latest-release-cron == 'true' }}
     uses: ./.github/workflows/_run_e2e_tests.yml
     secrets: inherit
     with:

.github/workflows/clean-astra-env.yml

@@ -3,7 +3,7 @@ name: Clean Astra env
 on:
   workflow_dispatch: {}
   schedule:
-    - cron: "*/30 * * * *"
+    - cron: "*/5 * * * *"
 
 jobs:
   e2e-tests:
@@ -13,9 +13,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          #          - name: Astra PROD
-          #            astra_token_secret: E2E_TESTS_ASTRA_PROD_DB_TOKEN
-          #            env: PROD
+          - name: Astra PROD
+            astra_token_secret: E2E_TESTS_ASTRA_PROD_DB_TOKEN
+            env: PROD
           - name: Astra DEV
             astra_token_secret: E2E_TESTS_ASTRA_DEV_DB_TOKEN
             env: DEV