Merge pull request #58 from red-hat-data-services/sync-midstream-2

[sync] Sync midstream

Author: ruivieira

.github/pull.yml

@@ -0,0 +1,8 @@
+version: "1"
+rules:
+  - base: main
+    upstream: trustyai-explainability:main
+    mergeMethod: merge
+    mergeUnstable: false
+label: "bot/merge-upstream"
+conflictLabel: "merge-conflict"

.github/workflows/docs.yml

@@ -0,0 +1,67 @@
+name: Build and Deploy Documentation
+
+on:
+  push:
+    branches: [main, docs]
+    paths:
+      - 'docs/**'
+      - 'antora-playbook.yml'
+      - '.github/workflows/docs.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'docs/**'
+      - 'antora-playbook.yml'
+      - '.github/workflows/docs.yml'
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+          cache: 'npm'
+
+      - name: Install Antora
+        run: |
+          npm install -g @antora/[email protected] @antora/[email protected]
+          npm install @asciidoctor/tabs
+
+      - name: Build documentation
+        run: antora antora-playbook.yml
+
+      - name: Upload documentation artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./docs-dist
+
+  deploy:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/docs'
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/release.yaml

@@ -0,0 +1,33 @@
+name: Publish to PyPi
+on:
+  release:
+    types: [published]
+jobs:
+  pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/llama-stack-provider-ragas
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Build
+        run: uv build
+      - name: Local smoke test (wheel)
+        run: uv run --isolated --no-project --with dist/*.whl -- python -c "import llama_stack_provider_ragas"
+      - name: Local smoke test (source distribution)
+        run: uv run --isolated --no-project --with dist/*.tar.gz -- python -c "import llama_stack_provider_ragas"
+      - name: Publish
+        run: uv publish
+      - name: Test install from pypi
+        run: |
+          uv run --isolated --no-project --with llama-stack-provider-ragas -- python -c "import llama_stack_provider_ragas; import importlib.metadata; print(f'LLS Ragas Provider Version: {importlib.metadata.version(\"llama-stack-provider-ragas\")}')"

.gitignore

@@ -95,3 +95,7 @@ _build/
 
 # Local development notes
 notes.org
+
+# Documentation
+docs-dist/
+node_modules/

.vscode/launch.json

@@ -6,11 +6,21 @@
     "configurations": [
 
         {
-            "name": "Debug Ragas Distribution",
+            "name": "Debug Ragas Distribution -- Remote",
             "type": "debugpy",
             "request": "launch",
             "module": "llama_stack.cli.llama",
-            "args": ["stack", "run", "distribution/run.yaml"],
+            "args": ["stack", "run", "distribution/run-remote.yaml"],
+            "cwd": "${workspaceFolder}",
+            "envFile": "${workspaceFolder}/.env",
+            "justMyCode": false
+        },
+        {
+            "name": "Debug Ragas Distribution -- Inline",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "llama_stack.cli.llama",
+            "args": ["stack", "run", "distribution/run-inline.yaml"],
             "cwd": "${workspaceFolder}",
             "envFile": "${workspaceFolder}/.env",
             "justMyCode": false

Containerfile

@@ -5,4 +5,4 @@ WORKDIR /usr/local/src/kfp/components
 
 COPY . .
 
-RUN pip install --no-cache-dir -e .
+RUN pip install --no-cache-dir -e ".[remote]"

README.md

@@ -1,6 +1,11 @@
-# `trustyai-ragas` <br> Ragas as an Out-of-Tree Llama Stack Provider
+<p align="center">
+  <img src="https://raw.githubusercontent.com/trustyai-explainability/llama-stack-provider-ragas/main/docs/_static/provider-logo.png" alt="Llama Stack Provider" height="120">
+</p>
+
+# Ragas as an External Provider for Llama Stack
+
+[![PyPI version](https://img.shields.io/pypi/v/llama_stack_provider_ragas.svg)](https://pypi.org/project/llama-stack-provider-ragas/)
 
-⚠️ Warning! This project is in early stages of development!
 
 ## About
 This repository implements [Ragas](https://github.com/explodinggradients/ragas) as an out-of-tree [Llama Stack](https://github.com/meta-llama/llama-stack) evaluation provider.
@@ -34,24 +39,58 @@ There are two versions of the provider:
     ```bash
     uv pip install -e ".[dev]"
     ```
-- Run the Llama Stack server with the distribution configs. The distribution is a simple LS distribution that uses Ollama for inference and embeddings, and includes both the inline and remote Ragas providers. Counting the number of `run`s in this command is left as an exercise for the reader:
-    ```bash
-    dotenv run uv run llama stack run distribution/run.yaml
-    ```
+- The sample LS distributions (one for inline and one for remote provider) is a simple LS distribution that uses Ollama for inference and embeddings. See the provider-specific sections below for setup and run commands.
+
+### Remote provider (default)
+
+Create a `.env` file with the following:
+```bash
+# Required for both inline and remote
+EMBEDDING_MODEL=all-MiniLM-L6-v2
+
+# Required for remote provider
+KUBEFLOW_LLAMA_STACK_URL=<your-llama-stack-url>
+KUBEFLOW_PIPELINES_ENDPOINT=<your-kfp-endpoint>
+KUBEFLOW_NAMESPACE=<your-namespace>
+KUBEFLOW_BASE_IMAGE=quay.io/diegosquayorg/my-ragas-provider-image:latest
+KUBEFLOW_PIPELINES_TOKEN=<your-pipelines-token>
+KUBEFLOW_RESULTS_S3_PREFIX=s3://my-bucket/ragas-results
+KUBEFLOW_S3_CREDENTIALS_SECRET_NAME=<secret-name>
+```
+
+Where:
+- `KUBEFLOW_LLAMA_STACK_URL`: The URL of the llama stack server that the remote provider will use to run the evaluation (LLM generations and embeddings, etc.). If you are running Llama Stack locally, you can use [ngrok](https://ngrok.com/) to expose it to the remote provider.
+- `KUBEFLOW_PIPELINES_ENDPOINT`: You can get this via `kubectl get routes -A | grep -i pipeline` on your Kubernetes cluster.
+- `KUBEFLOW_NAMESPACE`: The name of the data science project where the Kubeflow Pipelines server is running.
+- `KUBEFLOW_PIPELINES_TOKEN`: Kubeflow Pipelines token with access to submit pipelines. If not provided, the token will be read from the local kubeconfig file.
+- `KUBEFLOW_BASE_IMAGE`: The image used to run the Ragas evaluation in the remote provider. See `Containerfile` for details. There is a public version of this image at `quay.io/diegosquayorg/my-ragas-provider-image:latest`.
+- `KUBEFLOW_RESULTS_S3_PREFIX`: S3 location (bucket and prefix folder) where evaluation results will be stored, e.g., `s3://my-bucket/ragas-results`.
+- `KUBEFLOW_S3_CREDENTIALS_SECRET_NAME`: Name of the Kubernetes secret containing AWS credentials with write access to the S3 bucket. Create with:
+  ```bash
+  oc create secret generic <secret-name> \
+    --from-literal=AWS_ACCESS_KEY_ID=your-access-key \
+    --from-literal=AWS_SECRET_ACCESS_KEY=your-secret-key \
+    --from-literal=AWS_DEFAULT_REGION=us-east-1
+  ```
+
+Run the server:
+```bash
+dotenv run uv run llama stack run distribution/run-remote.yaml
+```
+
+### Inline provider (need to specify `.inline` in the module name)
 
-### Inline provider
+Create a `.env` file with the required environment variable:
+```bash
+EMBEDDING_MODEL=all-MiniLM-L6-v2
+```
 
-### Remote provider
-- Create a `.env` file with the following:
-    - `LLAMA_STACK_URL`
-        - This is the url of the llama stack server that the remote provider will use to run the evaluation (LLM generations and embeddings, etc.). If you are running Llama Stack locally, you can use [ngrok](https://ngrok.com/) to expose it to the remote provider.
-    - `KUBEFLOW_PIPELINES_ENDPOINT`
-        - You can get this via `kubectl get routes -A | grep -i pipeline` on your Kubernetes cluster.
-    - `KUBEFLOW_NAMESPACE`
-        - This is the name of the data science project where the Kubeflow Pipelines server is running.
-    - `KUBEFLOW_BASE_IMAGE`
-        - This is the image used to run the Ragas evaluation in the remote provider. See `Containerfile` for details. There is a public version of this image at `quay.io/diegosquayorg/my-ragas-provider-image:latest`.
+Run the server:
+```bash
+dotenv run uv run llama stack run distribution/run-inline.yaml
+```
 
+You will notice that `run-inline.yaml` file has the module name as `llama_stack_provider_ragas.inline`, in order to specify the inline provider.
 
 ## Usage
 See the demos in the `demos` directory.

antora-playbook.yml

@@ -0,0 +1,27 @@
+site:
+  title: Ragas provider for Llama Stack
+  url: https://trustyai-explainability.github.io/llama-stack-provider-ragas
+  start_page: llama-stack-provider-ragas::index.adoc
+
+content:
+  sources:
+  - url: ./
+    branches: HEAD
+    start_path: docs
+
+ui:
+  bundle:
+    url: https://gitlab.com/antora/antora-ui-default/-/jobs/artifacts/HEAD/raw/build/ui-bundle.zip?job=bundle-stable
+    snapshot: true
+
+asciidoc:
+  attributes:
+    source-highlighter: highlight.js
+    page-pagination: true
+    hide-uri-scheme: true
+  extensions:
+  - '@asciidoctor/tabs'
+  - 'asciidoctor-kroki'
+
+output:
+  dir: ./docs-dist