new packages docs (#386)

Author: nicoloboschi

.github/changes-filter.yaml

@@ -13,3 +13,10 @@ notebooks:
   - "./.github/actions/**"
   - "./.github/workflows/ci-e2e-tests.yml"
   - "./.github/workflows/_run_e2e_tests.yml"
+integration_tests:
+  - "libs/colbert/**"
+  - "libs/**"
+  - "pyproject.toml"
+  - "**/pyproject.toml"
+  - "./.github/actions/**"
+  - "./.github/workflows/ci-unit-tests.yml"

.github/workflows/ci-unit-tests.yml

@@ -64,17 +64,25 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v4
 
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: ./.github/changes-filter.yaml
+
       - name: "Setup: Python 3.11"
+        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         uses: ./.github/actions/setup-python
 
       - name: Compute db name
+        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         id: db-name
         run: |
           echo "db-name=${{ github.run_id}}-$(echo $(for((i=1;i<=8;i++))
           do printf '%s' "${RANDOM:0:1}"; done) | tr '[0-9]' '[a-z]')" >> $GITHUB_OUTPUT
 
       - name: Setup AstraDB
         uses: ./.github/actions/setup-astra-db
+        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         id: astra-db
         with:
           astra-token: ${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}
@@ -84,6 +92,7 @@ jobs:
           cloud: "gcp"
 
       - name: "Integration tests"
+        if: ${{ steps.filter.outputs.integration_tests == 'true' }}
         env:
           ASTRA_DB_TOKEN: "${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}"
           ASTRA_DB_ID: "${{ steps.astra-db.outputs.db_id }}"
@@ -102,7 +111,7 @@ jobs:
 
       - name: Cleanup AstraDB
         uses: ./.github/actions/cleanup-astra-db
-        if: ${{ always() }}
+        if: ${{ always() && steps.filter.outputs.integration_tests == 'true' }}
         with:
           astra-token: ${{ secrets.E2E_TESTS_ASTRA_DEV_DB_TOKEN }}
           db-name: ${{ steps.db-name.outputs.db-name }}

docs/modules/ROOT/nav.adoc

@@ -2,6 +2,7 @@
 * xref:ROOT:quickstart.adoc[]
 * xref:ROOT:what-is-rag.adoc[]
 * xref:ROOT:dev-environment.adoc[]
+* xref:ROOT:packages.adoc[]
 * xref:ROOT:migration.adoc[]
 
 .RAG Default architecture

docs/modules/ROOT/pages/packages.adoc

@@ -0,0 +1,44 @@
+= RAGStack Python packages
+
+RAGStack comes with a set of Python packages that provide the necessary tools to implement the RAG pattern in your applications.
+
+. `ragstack-ai`: All-in-one package that contains all components supported by RAGStack. While this is the most convenient package to use, it may be heavier than necessary for some use cases.
+. `ragstack-ai-langchain`: This package is meant for users who want to use RAGStack with the LangChain framework.
+. `ragstack-ai-llamaindex`: This package is meant for users who want to use RAGStack with the LlamaIndex framework.
+. `ragstack-ai-colbert`: This package contains the implementation of the ColBERT retrieval.
+
+
+== Supported integrations for `ragstack-ai-langchain`
+The `ragstack-ai-langchain` package includes the minimum set of dependencies for using LangChain with {astra_db}.
+LLMs, embeddings, and third-party providers are not included in this package by default, expect for OpenAI and Azure OpenAI.
+
+To use LLMs, embeddings, or third-party providers, you can leverage `ragstack-ai-langchain` extras:
+
+. `ragstack-ai-langchain[google]` lets you work with https://python.langchain.com/docs/integrations/platforms/google[Google Vertex AI and Google Gemini API]{external-link-icon}.
+. `ragstack-ai-langchain[nvidia]` lets you work with https://python.langchain.com/docs/integrations/providers/nvidia/[NVIDIA hosted API endpoints for NVIDIA AI Foundation Models]{external-link-icon}.
+
+Additional LangChain packages should work out of the box, although you need to manage the packages and their dependencies yourself.
+
+
+== Supported integrations for `ragstack-ai-llamaindex`
+
+The `ragstack-ai-llamaindex` package includes the minimum set of dependencies for using LlamaIndex with {astra_db}.
+LLMs, embeddings, and third-party providers are not included in this package by default, except for OpenAI.
+
+To use LLMs, embeddings, or third-party providers, you can leverage `ragstack-ai-llamaindex` extras:
+
+. `ragstack-ai-llamaindex[google]` lets you work with https://docs.llamaindex.ai/en/stable/examples/llm/vertex/[Google Vertex AI]{external-link-icon} and https://docs.llamaindex.ai/en/stable/examples/llm/gemini/[Google Gemini API]{external-link-icon}.
+. `ragstack-ai-llamaindex[azure]` lets you work with https://docs.llamaindex.ai/en/stable/examples/llm/azure_openai/[Azure OpenAI]{external-link-icon}.
+. `ragstack-ai-llamaindex[bedrock]` lets you work with https://docs.llamaindex.ai/en/stable/examples/llm/bedrock/[AWS Bedrock]{external-link-icon}.
+
+Additional LLamaIndex packages should work out of the box, although you need to manage the packages and their dependencies yourself.
+
+
+== Colbert with `ragstack-ai-langchain` and `ragstack-ai-llamaindex`
+
+The `colbert` module provides a vanilla implementation for COLBert retrieval. It is not tied to any specific framework and can be used with any of the RAGStack packages.
+
+If you want to use COLBert with LangChain or LLamaIndex, you can use the following the extras:
+
+. `ragstack-ai-langchain[colbert]`
+. `ragstack-ai-llamaindex[colbert]`