Embedding Service

.github/workflows/backend-test.yml

@@ -29,6 +29,84 @@ jobs:
     name: Backend tests
     runs-on: ubuntu-latest
 
+    # 👇 Only apply environment when running on GitHub (not act)
+    environment: test
+
+    env:
+      # Database configuration with isolation
+      SQLALCHEMY_DB_MODE: test
+      SQLALCHEMY_DB_USER: ${{ secrets.SQLALCHEMY_DB_USER }}
+      SQLALCHEMY_DB_PASS: ${{ secrets.SQLALCHEMY_DB_PASS }}
+      SQLALCHEMY_DB_HOST: localhost
+      SQLALCHEMY_DB_NAME: ${{ secrets.SQLALCHEMY_DB_NAME }}_${{ github.run_id }}
+      SQLALCHEMY_DATABASE_TEST_URL: postgresql://${{ secrets.SQLALCHEMY_DB_USER }}:${{ secrets.SQLALCHEMY_DB_PASS }}@localhost:5432/${{ secrets.SQLALCHEMY_DB_NAME }}_${{ github.run_id }}
+
+      # Application configuration
+      LOG_LEVEL: ${{ secrets.LOG_LEVEL }}
+      BROKER_URL: ${{ secrets.BROKER_URL }}
+      CELERY_RESULT_BACKEND: ${{ secrets.CELERY_RESULT_BACKEND }}
+
+      # External services
+      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+      AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+      AZURE_OPENAI_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_DEPLOYMENT_NAME }}
+      AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      GEMINI_MODEL_NAME: ${{ secrets.GEMINI_MODEL_NAME }}
+      AUTH0_DOMAIN: ${{ secrets.AUTH0_DOMAIN }}
+      AUTH0_AUDIENCE: ${{ secrets.AUTH0_AUDIENCE }}
+      AUTH0_CLIENT_ID: ${{ secrets.AUTH0_CLIENT_ID }}
+      AUTH0_CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }}
+      AUTH0_SECRET_KEY: ${{ secrets.AUTH0_SECRET_KEY }}
+      JWT_SECRET_KEY: ${{ secrets.JWT_SECRET_KEY }}
+      JWT_ALGORITHM: ${{ secrets.JWT_ALGORITHM }}
+      JWT_ACCESS_TOKEN_EXPIRE_MINUTES: ${{ secrets.JWT_ACCESS_TOKEN_EXPIRE_MINUTES }}
+      FRONTEND_URL: ${{ secrets.FRONTEND_URL }}
+      SMTP_HOST: ${{ secrets.SMTP_HOST }}
+      SMTP_PORT: ${{ secrets.SMTP_PORT }}
+      SMTP_USER: ${{ secrets.SMTP_USER }}
+      SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
+      FROM_EMAIL: ${{ secrets.FROM_EMAIL }}
+      AGENT_EMAIL_BCC: ${{ secrets.AGENT_EMAIL_BCC }}
+      WELCOME_CALENDAR_LINK: ${{ secrets.WELCOME_CALENDAR_LINK }}
+      WELCOME_FROM_EMAIL: ${{ secrets.WELCOME_FROM_EMAIL }}
+      STORAGE_SERVICE_URI: ${{ secrets.STORAGE_SERVICE_URI }}
+      STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.STORAGE_SERVICE_ACCOUNT_KEY }}
+      LOCAL_STORAGE_PATH: ${{ secrets.LOCAL_STORAGE_PATH }}
+
+      # Encryption
+      DB_ENCRYPTION_KEY: ${{ secrets.DB_ENCRYPTION_KEY }}
+
+      # Observability (disable connector functionalities)
+      RHESIS_CONNECTOR_DISABLED: ${{ secrets.RHESIS_CONNECTOR_DISABLED || 'true' }}
+
+    services:
+      postgres:
+        image: pgvector/pgvector:pg15
+        env:
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          # Removed POSTGRES_HOST_AUTH_METHOD: trust for better security
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -50,8 +128,175 @@ jobs:
             ${{ runner.os }}-uv-backend-
 
       - name: Install dependencies
-        run: uv sync --dev --extra cpu
-        working-directory: apps/backend
+        run: |
+          cd apps/backend
+          uv sync --dev
+
+      - name: Verify SDK installation
+        run: |
+          cd apps/backend
+          uv pip show rhesis-sdk || {
+            echo "❌ SDK not installed correctly"
+            exit 1
+          }
+
+      - name: Debug environment detection
+        run: |
+          echo "🔍 Environment detection debug:"
+          echo "ACT: ${ACT:-not set}"
+          echo "GITHUB_ACTIONS: ${GITHUB_ACTIONS:-not set}"
+          echo "CI: ${CI:-not set}"
+          echo "Isolated database name: $SQLALCHEMY_DB_NAME"
+
+      - name: Install PostgreSQL client
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y postgresql-client
+
+      - name: Wait for PostgreSQL
+        run: |
+          echo "⏳ Waiting for PostgreSQL..."
+          until PGPASSWORD=postgres pg_isready -h localhost -p 5432 -U postgres; do
+            sleep 2
+          done
+          echo "✅ PostgreSQL ready"
+
+      - name: Create test database and user
+        run: |
+          echo "🔗 Setting up test database and user..."
+          echo "Creating isolated database: $SQLALCHEMY_DB_NAME"
+          echo "Database user: $SQLALCHEMY_DB_USER"
+
+          # Drop and recreate user to ensure clean state
+          PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c "
+            DROP USER IF EXISTS \"$SQLALCHEMY_DB_USER\";
+            CREATE USER \"$SQLALCHEMY_DB_USER\" WITH PASSWORD '$SQLALCHEMY_DB_PASS' CREATEDB;
+          " || exit 1
+
+          # Create isolated database for this run
+          echo "Creating database: $SQLALCHEMY_DB_NAME"
+          PGPASSWORD=postgres createdb -h localhost -U postgres -O "$SQLALCHEMY_DB_USER" "$SQLALCHEMY_DB_NAME" || {
+            echo "❌ Failed to create database $SQLALCHEMY_DB_NAME"
+            exit 1
+          }
+
+          # Grant comprehensive privileges
+          PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c "
+            GRANT ALL PRIVILEGES ON DATABASE \"$SQLALCHEMY_DB_NAME\" TO \"$SQLALCHEMY_DB_USER\";
+            GRANT ALL ON SCHEMA public TO \"$SQLALCHEMY_DB_USER\";
+          " || exit 1
+
+          # Connect to the new database and grant schema permissions
+          PGPASSWORD=postgres psql -h localhost -U postgres -d "$SQLALCHEMY_DB_NAME" -c "
+            CREATE EXTENSION IF NOT EXISTS vector;
+            GRANT ALL ON SCHEMA public TO \"$SQLALCHEMY_DB_USER\";
+            GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"$SQLALCHEMY_DB_USER\";
+            GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"$SQLALCHEMY_DB_USER\";
+            ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO \"$SQLALCHEMY_DB_USER\";
+            ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO \"$SQLALCHEMY_DB_USER\";
+          " || exit 1
+
+          # Verify database exists
+          PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c "
+            SELECT datname FROM pg_database WHERE datname = '$SQLALCHEMY_DB_NAME';
+          " | grep -q "$SQLALCHEMY_DB_NAME" || {
+            echo "❌ Database $SQLALCHEMY_DB_NAME was not created successfully"
+            exit 1
+          }
+
+          # Test connection to the new database with proper authentication
+          echo "Testing connection with user credentials..."
+          PGPASSWORD=$SQLALCHEMY_DB_PASS psql -h localhost -U "$SQLALCHEMY_DB_USER" -d "$SQLALCHEMY_DB_NAME" -c 'SELECT current_user, current_database();' || {
+            echo "❌ Cannot connect to database $SQLALCHEMY_DB_NAME with user $SQLALCHEMY_DB_USER"
+            echo "Testing if user can connect to postgres database..."
+            PGPASSWORD=$SQLALCHEMY_DB_PASS psql -h localhost -U "$SQLALCHEMY_DB_USER" -d postgres -c 'SELECT current_user;' || {
+              echo "❌ User cannot connect to any database - password issue"
+              exit 1
+            }
+            exit 1
+          }
+
+          echo "✅ Database and user created successfully"
+
+      - name: Run database migrations
+        run: |
+          cd apps/backend/src/rhesis/backend
+          echo "🔄 Running migrations on isolated database: $SQLALCHEMY_DB_NAME"
+          echo "Migration environment variables:"
+          echo "  SQLALCHEMY_DB_MODE: $SQLALCHEMY_DB_MODE"
+          echo "  SQLALCHEMY_DB_NAME: $SQLALCHEMY_DB_NAME"
+          echo "  SQLALCHEMY_DATABASE_TEST_URL: $SQLALCHEMY_DATABASE_TEST_URL"
+          echo "  SQLALCHEMY_DB_USER: $SQLALCHEMY_DB_USER"
+          echo "  SQLALCHEMY_DB_HOST: $SQLALCHEMY_DB_HOST"
+          echo "  SQLALCHEMY_DB_PASS: [REDACTED - $(if [ -n "$SQLALCHEMY_DB_PASS" ]; then echo "SET"; else echo "NOT SET"; fi)]"
+
+          # Verify database exists before running migrations
+          PGPASSWORD=$SQLALCHEMY_DB_PASS psql -h localhost -U "$SQLALCHEMY_DB_USER" -d "$SQLALCHEMY_DB_NAME" -c 'SELECT version();' || {
+            echo "❌ Cannot connect to database $SQLALCHEMY_DB_NAME for migrations"
+            echo "Listing available databases:"
+            PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c '\l'
+            exit 1
+          }
+
+          # Ensure environment variables take precedence over .env file
+          export SQLALCHEMY_DB_MODE="test"
+          export SQLALCHEMY_DB_NAME="$SQLALCHEMY_DB_NAME"
+          export SQLALCHEMY_DATABASE_TEST_URL="postgresql://$SQLALCHEMY_DB_USER:$SQLALCHEMY_DB_PASS@localhost:5432/$SQLALCHEMY_DB_NAME"
+          export SQLALCHEMY_DB_USER="$SQLALCHEMY_DB_USER"
+          export SQLALCHEMY_DB_PASS="$SQLALCHEMY_DB_PASS"
+          export SQLALCHEMY_DB_HOST="localhost"
+
+          echo "🔄 Running Alembic with exported environment variables..."
+          echo "Final environment check:"
+          echo "  SQLALCHEMY_DATABASE_TEST_URL: $SQLALCHEMY_DATABASE_TEST_URL"
+          uv run alembic upgrade head
+
+      - name: Verify database structure
+        run: |
+          echo "🔍 Verifying database structure..."
+          echo "Checking isolated database: $SQLALCHEMY_DB_NAME"
+          PGPASSWORD=$SQLALCHEMY_DB_PASS psql -h localhost -U "$SQLALCHEMY_DB_USER" -d "$SQLALCHEMY_DB_NAME" -c "
+            SELECT COUNT(*) as table_count
+            FROM information_schema.tables
+            WHERE table_schema = 'public';
+          " || exit 1
+          echo "✅ Database structure verified"
+
+      - name: Setup test data
+        id: setup_env
+        env:
+          LOG_LEVEL: WARNING
+        run: |
+          cd apps/backend
+          export PYTHONPATH="${{ github.workspace }}/apps/backend/src:${{ github.workspace }}/tests:$PYTHONPATH"
+          echo "🚀 Setting up test environment..."
+          echo "Using isolated database: $SQLALCHEMY_DB_NAME"
+          echo "Database URL: $SQLALCHEMY_DATABASE_TEST_URL"
+
+          # Setup test environment and capture API token
+          API_TOKEN=$(timeout 300 uv run python -c "
+          import os
+          from backend.fixtures.test_setup import setup_test_environment
+          setup_test_environment()
+          print(os.environ.get('RHESIS_API_KEY', ''))
+          " | tail -1) || exit 1
+
+          # Export the API token for subsequent steps
+          echo "RHESIS_API_KEY=${API_TOKEN}" >> $GITHUB_OUTPUT
+          echo "✅ Test environment ready"
+
+      - name: Determine pytest command
+        id: pytest_cmd
+        run: |
+          PYTEST_OPTS="-v --durations=10 --tb=short --maxfail=10 -x --junitxml=junit.xml -o junit_family=legacy"
+          if [ "${{ inputs.test_markers }}" = "all" ]; then
+            echo "cmd=uv run --project apps/backend python -m pytest tests/backend $PYTEST_OPTS" >> $GITHUB_OUTPUT
+          elif [ "${{ inputs.test_markers }}" = "custom" ]; then
+            echo "cmd=uv run --project apps/backend python -m pytest tests/backend -m \"${{ inputs.custom_markers }}\" $PYTEST_OPTS" >> $GITHUB_OUTPUT
+          else
+            echo "cmd=uv run --project apps/backend python -m pytest tests/backend -m \"${{ inputs.test_markers }}\" $PYTEST_OPTS" >> $GITHUB_OUTPUT
+          fi
+
 
       - name: Run tests
         run: make test

.github/workflows/backend.yml

@@ -454,8 +454,8 @@ jobs:
           GH_CLIENT_SECRET=${{ secrets.GH_CLIENT_SECRET }},
           GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }},
           GEMINI_MODEL_NAME=${{ secrets.GEMINI_MODEL_NAME }},
-          DEFAULT_GENERATION_MODEL=${{ secrets.DEFAULT_GENERATION_MODEL }},
-          DEFAULT_MODEL_NAME=${{ secrets.DEFAULT_MODEL_NAME }},
+          DEFAULT_LANGUAGE_MODEL_PROVIDER=${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER }},
+          DEFAULT_LANGUAGE_MODEL_NAME=${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME }},
           GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }},
           VERTEX_AI_LOCATION=${{ secrets.VERTEX_AI_LOCATION }},
           VERTEX_AI_PROJECT=${{ secrets.VERTEX_AI_PROJECT }},
@@ -525,8 +525,8 @@ jobs:
           GH_CLIENT_SECRET=${{ secrets.GH_CLIENT_SECRET }},
           GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }},
           GEMINI_MODEL_NAME=${{ secrets.GEMINI_MODEL_NAME }},
-          DEFAULT_GENERATION_MODEL=${{ secrets.DEFAULT_GENERATION_MODEL }},
-          DEFAULT_MODEL_NAME=${{ secrets.DEFAULT_MODEL_NAME }},
+          DEFAULT_LANGUAGE_MODEL_PROVIDER=${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER }},
+          DEFAULT_LANGUAGE_MODEL_NAME=${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME }},
           GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }},
           VERTEX_AI_LOCATION=${{ secrets.VERTEX_AI_LOCATION }},
           VERTEX_AI_PROJECT=${{ secrets.VERTEX_AI_PROJECT }},
@@ -716,8 +716,8 @@ jobs:
           GH_CLIENT_SECRET=${{ secrets.GH_CLIENT_SECRET }},
           GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }},
           GEMINI_MODEL_NAME=${{ secrets.GEMINI_MODEL_NAME }},
-          DEFAULT_GENERATION_MODEL=${{ secrets.DEFAULT_GENERATION_MODEL }},
-          DEFAULT_MODEL_NAME=${{ secrets.DEFAULT_MODEL_NAME }},
+          DEFAULT_LANGUAGE_MODEL_PROVIDER=${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER }},
+          DEFAULT_LANGUAGE_MODEL_NAME=${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME }},
           GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }},
           VERTEX_AI_LOCATION=${{ secrets.VERTEX_AI_LOCATION }},
           VERTEX_AI_PROJECT=${{ secrets.VERTEX_AI_PROJECT }},
@@ -794,8 +794,8 @@ jobs:
           GH_CLIENT_SECRET=${{ secrets.GH_CLIENT_SECRET }},
           GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }},
           GEMINI_MODEL_NAME=${{ secrets.GEMINI_MODEL_NAME }},
-          DEFAULT_GENERATION_MODEL=${{ secrets.DEFAULT_GENERATION_MODEL }},
-          DEFAULT_MODEL_NAME=${{ secrets.DEFAULT_MODEL_NAME }},
+          DEFAULT_LANGUAGE_MODEL_PROVIDER=${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER }},
+          DEFAULT_LANGUAGE_MODEL_NAME=${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME }},
           GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }},
           VERTEX_AI_LOCATION=${{ secrets.VERTEX_AI_LOCATION }},
           VERTEX_AI_PROJECT=${{ secrets.VERTEX_AI_PROJECT }},

.github/workflows/chatbot.yml

@@ -179,7 +179,7 @@ jobs:
             echo "📧 Rate limit notifications enabled (hello@rhesis.ai)"
           fi
 
-          echo "🤖 Model configuration: ${{ secrets.DEFAULT_GENERATION_MODEL || 'vertex_ai' }} / ${{ secrets.DEFAULT_MODEL_NAME || 'gemini-2.0-flash' }}"
+          echo "🤖 Model configuration: ${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER || 'vertex_ai' }} / ${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME || 'gemini-2.0-flash' }}"
 
           # Set environment-dependent resources
           if [ "${{ env.ENVIRONMENT }}" = "prd" ]; then
@@ -219,8 +219,8 @@ jobs:
             --max-instances=$MAX_INSTANCES \
             --min-instances=$MIN_INSTANCES \
             --set-env-vars="$(cat <<EOF
-          DEFAULT_GENERATION_MODEL=${{ secrets.DEFAULT_GENERATION_MODEL || 'vertex_ai' }},
-          DEFAULT_MODEL_NAME=${{ secrets.DEFAULT_MODEL_NAME || 'gemini-2.0-flash' }},
+          DEFAULT_LANGUAGE_MODEL_PROVIDER=${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER || 'vertex_ai' }},
+          DEFAULT_LANGUAGE_MODEL_NAME=${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME || 'gemini-2.0-flash' }},
           GOOGLE_APPLICATION_CREDENTIALS=${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }},
           VERTEX_AI_LOCATION=${{ secrets.VERTEX_AI_LOCATION }},
           VERTEX_AI_PROJECT=${{ secrets.VERTEX_AI_PROJECT }},

.github/workflows/worker.yml

@@ -281,8 +281,8 @@ jobs:
             --from-literal=GEMINI_API_KEY="${{ secrets.GEMINI_API_KEY }}" \
             --from-literal=GOOGLE_API_KEY="${{ secrets.GOOGLE_API_KEY }}" \
             --from-literal=GEMINI_MODEL_NAME="${{ secrets.GEMINI_MODEL_NAME }}" \
-            --from-literal=DEFAULT_GENERATION_MODEL="${{ secrets.DEFAULT_GENERATION_MODEL }}" \
-            --from-literal=DEFAULT_MODEL_NAME="${{ secrets.DEFAULT_MODEL_NAME }}" \
+            --from-literal=DEFAULT_LANGUAGE_MODEL_PROVIDER="${{ secrets.DEFAULT_LANGUAGE_MODEL_PROVIDER }}" \
+            --from-literal=DEFAULT_LANGUAGE_MODEL_NAME="${{ secrets.DEFAULT_LANGUAGE_MODEL_NAME }}" \
             --from-literal=GOOGLE_APPLICATION_CREDENTIALS="${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}" \
             --from-literal=VERTEX_AI_LOCATION="${{ secrets.VERTEX_AI_LOCATION }}" \
             --from-literal=VERTEX_AI_PROJECT="${{ secrets.VERTEX_AI_PROJECT }}" \

apps/backend/src/rhesis/backend/alembic/versions/2e3f4g5h6i7j_add_dimension_column_to_model_table.py

@@ -0,0 +1,36 @@
+"""add dimension column to model table
+
+Revision ID: 2e3f4g5h6i7j
+Revises: 1776e6dd47d3
+Create Date: 2026-02-12 15:00:00.000000
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from typing import Union, Sequence
+
+
+# revision identifiers, used by Alembic.
+revision: str = "2e3f4g5h6i7j"
+down_revision: Union[str, None] = "1776e6dd47d3"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add dimension column to model table for embedding models."""
+    op.add_column(
+        "model",
+        sa.Column(
+            "dimension",
+            sa.Integer(),
+            nullable=True,
+            comment="Embedding dimension (384, 768, 1024, or 1536). Only applicable for embedding models.",
+        ),
+    )
+
+
+def downgrade() -> None:
+    """Remove dimension column from model table."""
+    op.drop_column("model", "dimension")