ci.yml

name: CI

on:
  pull_request:
    branches:
      - main
      - legacy-lancedb

  push:
    branches:
      - main
      - legacy-lancedb
    tags:
      - "v*"

jobs:
  check_changes:
    name: Check for changes
    runs-on: ubuntu-latest
    outputs:
      has-changes: ${{ steps.check.outputs.has-changes }}
    steps:
      - name: Check
        id: check
        uses: jiahuei/check-changes-action@v0
        with:
          watch-dirs: "clients/python/ services/api/ docker/ .github/"

  sdk_tests_noop:
    # This job is needed so that status checks can still pass
    # This is because strategy matrix is evaluated after if condition
    name: SDK unit tests
    runs-on: ubuntu-latest
    needs: check_changes
    if: ${{ !(needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push') }}
    strategy:
      matrix:
        jamai-mode: ["oss"]
        test-group: [group1, group2, group3, group4]
    timeout-minutes: 2
    steps:
      - name: No-op
        run: echo Tests skipped !!!

  sdk_tests:
    name: SDK unit tests
    runs-on: ubuntu-latest-l
    needs: check_changes
    if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push'
    strategy:
      matrix:
        jamai-mode: ["oss"]
        test-group: [group1, group2, group3, group4]
    timeout-minutes: 90
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
      cancel-in-progress: true

    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          lfs: true

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Inspect git version
        run: git --version

      - name: Check Docker Version
        run: docker version

      - name: Check Docker Compose Version
        run: docker compose version

      - name: Remove cloud-only modules
        if: matrix.jamai-mode == 'oss'
        run: bash scripts/remove_cloud_modules.sh

      - name: Inspect directory tree
        run: tree

      - name: Install jamaibase & owl
        run: |
          pushd clients/python
          uv pip install --system -e .[test]
          popd
          pushd services/api
          uv pip install --system -e .[test]

      - name: Inspect jamaibase environment
        run: |
          uv pip list

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Edit env file
        run: |
          mv .env.example .env

          ORGS=$(printenv | grep API_KEY | xargs -I {} echo {} | cut -d '=' -f 1)
          KEYS=$(printenv | grep API_KEY | xargs -I {} echo {} | cut -d '=' -f 2-)
          # Convert them into arrays
          ORG_ARRAY=($ORGS)
          KEY_ARRAY=($KEYS)
          # Loop through the ORG_ARRAY
          for i in "${!ORG_ARRAY[@]}"; do
            # Get the org and key
            org="${ORG_ARRAY[$i]}"
            key="${KEY_ARRAY[$i]}"
            # Replace the org with the key in the .env file
            sed -i "s/$org=.*/$org=$key/g" .env
          done
          echo "OWL_DB_INIT=0" >> .env
          echo "OWL_COMPUTE_STORAGE_PERIOD_SEC=15" >> .env
          echo "OWL_STRIPE_WEBHOOK_SECRET_TEST=${OWL_STRIPE_WEBHOOK_SECRET_TEST}" >> .env
          echo "OWL_STRIPE_PUBLISHABLE_KEY_TEST=${OWL_STRIPE_PUBLISHABLE_KEY_TEST}" >> .env
          echo 'OWL_SERVICE_KEY=lalala' >> .env
          echo 'JAMAI_TOKEN=lalala' >> .env
          echo 'JAMAI_API_BASE=http://localhost:6969/api' >> .env
          echo 'OWL_FLUSH_CLICKHOUSE_BUFFER_SEC=5' >> .env
        env:
          OWL_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OWL_COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
          OWL_DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          OWL_ELLM_API_KEY: ${{ secrets.CUSTOM_API_KEY }}
          OWL_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          OWL_HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }}
          OWL_JINA_AI_API_KEY: ${{ secrets.JINA_AI_API_KEY }}
          OWL_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OWL_TOGETHER_AI_API_KEY: ${{ secrets.TOGETHER_AI_API_KEY }}
          OWL_STRIPE_WEBHOOK_SECRET_TEST: ${{ secrets.OWL_STRIPE_WEBHOOK_SECRET_TEST }}
          OWL_STRIPE_PUBLISHABLE_KEY_TEST: ${{ secrets.OWL_STRIPE_PUBLISHABLE_KEY_TEST }}

      - name: Configure Docker daemon
        run: |
          DAEMON_JSON=/etc/docker/daemon.json
          EXISTING=$(sudo cat "$DAEMON_JSON" 2>/dev/null || echo '{}')
          echo "$EXISTING" | jq '. + {"max-concurrent-downloads": 3, "max-download-attempts": 5}' | sudo tee "$DAEMON_JSON"
          sudo systemctl restart docker

      - name: Launch services
        id: launch_services
        timeout-minutes: 20
        if: always()
        run: |
          docker compose -p jm -f docker/compose.ci.yml up --quiet-pull -d --wait
        env:
          COMPOSE_DOCKER_CLI_BUILD: 1
          DOCKER_BUILDKIT: 1
          COMPOSE_HTTP_TIMEOUT: 30
          DOCKER_CLIENT_TIMEOUT: 30

      - name: Inspect owl logs if failed to launch
        timeout-minutes: 1
        if: failure() && steps.launch_services.outcome == 'failure'
        run: docker compose -p jm -f docker/compose.ci.yml logs owl

      - name: Inspect owl UV and Python version
        run: |
          docker exec jm-owl-1 uv -V
          docker exec jm-owl-1 $(docker exec jm-owl-1 uv python find) -V

      - name: Inspect owl environment
        if: always()
        run: docker exec jm-owl-1 uv pip list

      - name: Copy OpenAPI JSON
        id: copy_openapi
        if: always() && matrix.test-group == 'group1'
        run: |
          curl localhost:6969/api/public/openapi.json > openapi.json

      - name: Generate OpenAPI Redoc HTML page
        id: generate_redoc_html
        if: always() && matrix.test-group == 'group1' && steps.copy_openapi.outcome == 'success'
        run: |
          npx @redocly/cli@latest build-docs openapi.json
          mkdir openapi
          mv redoc-static.html openapi
          mv openapi.json openapi

      - name: Upload Redoc HTML
        id: upload_redoc_html
        uses: actions/upload-artifact@v4
        if: always() && matrix.test-group == 'group1' && steps.generate_redoc_html.outcome == 'success'
        with:
          name: redoc-html-${{ matrix.jamai-mode }}
          path: openapi

      - name: Publish Redoc HTML link as PR comment
        uses: thollander/actions-comment-pull-request@v3
        if: always() && matrix.test-group == 'group1' && github.event_name == 'pull_request' && steps.upload_redoc_html.outcome == 'success'
        with:
          message: |
            [Link to OpenAPI Redoc HTML (${{ matrix.jamai-mode }})](${{ steps.upload_redoc_html.outputs.artifact-url }})
          comment-tag: redoc_html_comment_${{ matrix.jamai-mode }}

      - name: Python SDK tests
        id: python_sdk_test
        if: always() && steps.launch_services.outcome == 'success'
        run: |
          cp .env services/api/.env
          pushd services/api

          if [ "${{ matrix.test-group }}" = "group1" ]; then
            DIRS=(tests --ignore=tests/gen_table/test_row_ops.py --ignore=tests/gen_table/test_row_ops_v2.py --ignore=tests/routers --ignore=tests/utils)
          elif [ "${{ matrix.test-group }}" = "group2" ]; then
            DIRS=(tests/gen_table/test_row_ops.py)
          elif [ "${{ matrix.test-group }}" = "group3" ]; then
            DIRS=(tests/gen_table/test_row_ops_v2.py tests/utils)
          else
            DIRS=(tests/routers)
          fi

          coverage run --data-file=coverage/.coverage.${{ matrix.test-group }} --rcfile=pyproject.toml -m \
            pytest \
              --timeout 300 \
              --no-flaky-report \
              --junitxml=pytest_regular.xml \
              -m "not (${{ matrix.jamai-mode == 'cloud' && 'oss' || 'cloud' }} or stripe)" \
              "${DIRS[@]}"

          # Shut down owl to allow coverage data to be flushed
          popd
          docker compose -p jm -f docker/compose.ci.yml down
          # Copy Pytest coverage data
          sudo find docker_data -type f -name '.coverage.*' -exec cp -v -t services/api/coverage/. {} +
        env:
          OWL_DB_PATH: postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl
          OWL_CLICKHOUSE_HOST: localhost
          OWL_REDIS_HOST: localhost

      - name: Inspect owl logs
        if: always() && steps.launch_services.outcome == 'success'
        timeout-minutes: 1
        run: mkdir -p logs && docker compose -p jm -f docker/compose.ci.yml logs owl > logs/owl.log

      - name: Inspect starling logs
        if: always() && steps.launch_services.outcome == 'success'
        timeout-minutes: 1
        run: mkdir -p logs && docker compose -p jm -f docker/compose.ci.yml logs starling > logs/starling.log

      - name: Inspect test-llm logs
        timeout-minutes: 1
        if: always() && steps.launch_services.outcome == 'success'
        run: mkdir -p logs && docker compose -p jm -f docker/compose.ci.yml logs test-llm > logs/test-llm.log

      - name: Test Stripe integration (Cloud only)
        id: test_stripe
        if: matrix.jamai-mode == 'cloud' && matrix.test-group == 'group1' && steps.launch_services.outcome == 'success'
        run: |
          sudo rm -rf docker_data

          # Relaunch
          echo "OWL_STRIPE_API_KEY=${OWL_STRIPE_API_KEY}" >> .env
          docker compose -p jm -f docker/compose.ci.yml up --quiet-pull -d --wait --force-recreate

          # Install Stripe CLI
          curl -L https://github.com/stripe/stripe-cli/releases/download/v1.27.0/stripe_1.27.0_linux_x86_64.tar.gz --output stripe.tar.gz
          tar -xvf stripe.tar.gz

          # Listen for Stripe events and forward them to local endpoint
          nohup ./stripe listen \
            --forward-to http://localhost:6969/api/v2/organizations/webhooks/stripe &
            # --events customer.created,invoice.paid
          # Give stripe listen a moment to establish the tunnel
          sleep 5

          # Run tests
          pushd services/api
          coverage run --data-file=coverage/.coverage.stripe --rcfile=pyproject.toml -m \
            pytest \
              --timeout 300 \
              --no-flaky-report \
              --junitxml=pytest_stripe.xml \
              -m stripe \
              tests

          # Shut down owl to allow coverage data to be flushed
          popd
          docker compose -p jm -f docker/compose.ci.yml down
          # Move existing coverage data
          sudo find docker_data -type f -name '.coverage.*' -exec cp -v -t services/api/coverage/. {} +
        env:
          STRIPE_API_KEY: ${{ secrets.OWL_STRIPE_API_KEY }}
          OWL_STRIPE_API_KEY: ${{ secrets.OWL_STRIPE_API_KEY }}
          OWL_CLICKHOUSE_HOST: localhost
          OWL_REDIS_HOST: localhost

      - name: Inspect owl logs if Stripe integration failed
        timeout-minutes: 1
        if: failure() && matrix.jamai-mode == 'cloud' && steps.test_stripe.outcome == 'failure'
        run: mkdir -p logs && docker compose -p jm --env-file .env -f docker/compose.ci.yml logs owl > logs/owl_stripe.log

      # - name: TS/JS SDK tests
      #   id: ts_sdk_test
      #   if: always() && steps.launch_services.outcome == 'success'
      #   run: |
      #     cd clients/typescript
      #     echo "BASEURL=http://localhost:6969" >> __tests__/.env
      #     npm install
      #     npm run test

      - name: Upload logs
        id: upload_logs
        uses: actions/upload-artifact@v4
        if: always() && steps.launch_services.outcome == 'success'
        with:
          name: logs-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
          path: logs

      - name: Publish logs link as PR comment
        uses: thollander/actions-comment-pull-request@v3
        if: always() && github.event_name == 'pull_request' && steps.upload_logs.outcome == 'success'
        with:
          message: |
            [Link to logs (${{ matrix.jamai-mode }}, ${{ matrix.test-group }})](${{ steps.upload_logs.outputs.artifact-url }})
          comment-tag: logs_comment_${{ matrix.jamai-mode }}-${{ matrix.test-group }}

      - name: Merge JUnit XML and Coverage data
        id: merge_test_data
        if: always() && steps.launch_services.outcome == 'success'
        run: |
          # Shut down owl to allow coverage data to be flushed
          docker compose -p jm --env-file .env -f docker/compose.ci.yml down

          # Combine coverage data
          coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \
            services/api/coverage

          # Merge JUnit XML files
          mkdir -p services/api/junit_xml
          junitparser merge --glob "services/api/pytest_*.xml" services/api/junit_xml/pytest-${{ matrix.jamai-mode }}-${{ matrix.test-group }}.xml

      - name: Upload coverage data
        uses: actions/upload-artifact@v4
        if: always() && steps.merge_test_data.outcome == 'success'
        with:
          name: owl-coverage-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
          path: services/api/coverage
          include-hidden-files: true
          if-no-files-found: error

      - name: Upload JUnit XML file
        uses: actions/upload-artifact@v4
        if: always() && steps.merge_test_data.outcome == 'success'
        with:
          name: junit-xml-data-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
          path: services/api/junit_xml

      - name: Generate coverage reports
        id: generate_coverage_report
        if: always() && steps.merge_test_data.outcome == 'success'
        run: |
          cd services/api
          coverage html --data-file=coverage/.coverage -d coverage/html
          coverage xml --data-file=coverage/.coverage -o coverage/coverage.xml
          coverage report --data-file=coverage/.coverage

      - name: Upload coverage HTML report
        id: upload_coverage_html
        uses: actions/upload-artifact@v4
        if: always() && steps.merge_test_data.outcome == 'success'
        with:
          name: pytest-coverage-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
          path: services/api/coverage/html

  merge_coverage:
    name: Merge Coverage Reports
    runs-on: ubuntu-latest
    needs: sdk_tests
    if: always() && (needs.sdk_tests.result == 'success' || needs.sdk_tests.result == 'skipped')
    strategy:
      matrix:
        jamai-mode: ["oss"]
    steps:
      - name: Skip coverage merge (no tests ran)
        if: needs.sdk_tests.result == 'skipped'
        run: echo "Skipping coverage merge - no tests were executed (no changes detected)"

      - name: Checkout code
        if: needs.sdk_tests.result == 'success'
        uses: actions/checkout@v4
        with:
          lfs: true

      - name: Install uv
        if: needs.sdk_tests.result == 'success'
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true

      - name: Set up Python
        if: needs.sdk_tests.result == 'success'
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install jamaibase & owl
        if: needs.sdk_tests.result == 'success'
        run: |
          pushd services/api
          uv pip install --system -e .[test]

      - name: Download coverage data artifacts
        uses: actions/download-artifact@v4
        if: needs.sdk_tests.result == 'success'
        with:
          pattern: owl-coverage-*
          path: ./

      - name: Download junit xml artifacts
        uses: actions/download-artifact@v4
        if: needs.sdk_tests.result == 'success'
        with:
          pattern: junit-xml-data-*
          path: junit-xml-data

      - name: Log coverage data files
        if: needs.sdk_tests.result == 'success'
        run: |
          find owl-coverage-* -type f | head -50
          find junit-xml-data -type f | head -50

      - name: Merge JUnit XML and Coverage data (${{ matrix.jamai-mode }})
        id: merge_coverage
        if: needs.sdk_tests.result == 'success'
        run: |
          coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \
            ./owl-coverage-${{ matrix.jamai-mode }}-group[1-4]

          # Merge JUnit XML files
          junitparser merge --glob "junit-xml-data/junit-xml-data-${{ matrix.jamai-mode }}*/pytest-*.xml" \
            junit-xml-data/pytest-${{ matrix.jamai-mode }}.xml

      - name: Generate coverage report
        if: always() && steps.merge_coverage.outcome == 'success'
        run: |
          cd services/api
          coverage html --data-file=coverage/.coverage -d coverage/html-${{ matrix.jamai-mode }}
          coverage xml --data-file=coverage/.coverage -o coverage/coverage-${{ matrix.jamai-mode }}.xml
          coverage report --data-file=coverage/.coverage

      - name: Upload ${{ matrix.jamai-mode }} coverage HTML report
        id: upload_coverage_html
        uses: actions/upload-artifact@v4
        if: always() && steps.merge_coverage.outcome == 'success'
        with:
          name: pytest-coverage-${{ matrix.jamai-mode }}
          path: services/api/coverage/html-${{ matrix.jamai-mode }}

      - name: Pytest coverage comment
        uses: MishaKav/pytest-coverage-comment@main
        if: always() && github.event_name == 'pull_request' && steps.merge_coverage.outcome == 'success'
        with:
          title: Coverage Report (${{ matrix.jamai-mode }})
          pytest-xml-coverage-path: services/api/coverage/coverage-${{ matrix.jamai-mode }}.xml
          junitxml-path: junit-xml-data/pytest-${{ matrix.jamai-mode }}.xml
          unique-id-for-comment: coverage_report_comment_${{ matrix.jamai-mode }}
          report-only-changed-files: true

      - name: Merge All JUnit XML and Coverage data
        id: merge_all_test_data
        if: needs.sdk_tests.result == 'success' && matrix.jamai-mode == 'oss'
        run: |
          coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \
            ./owl-coverage-oss-group[1-4]

          # Merge JUnit XML files
          junitparser merge --glob "junit-xml-data/junit-xml-data-*/pytest-*.xml" junit-xml-data/pytest.xml

      - name: Generate coverage reports
        id: generate_coverage_report
        if: always() && steps.merge_all_test_data.outcome == 'success'
        run: |
          cd services/api
          coverage html --data-file=coverage/.coverage -d coverage/html
          coverage xml --data-file=coverage/.coverage -o coverage/coverage.xml
          coverage report --data-file=coverage/.coverage

      - name: Pytest coverage comment
        uses: MishaKav/pytest-coverage-comment@main
        if: always() && github.event_name == 'pull_request' && steps.merge_all_test_data.outcome == 'success'
        with:
          title: Coverage Report (all)
          pytest-xml-coverage-path: services/api/coverage/coverage.xml
          unique-id-for-comment: coverage_report_comment_all
          report-only-changed-files: true

      - name: Upload all coverage HTML report
        id: upload_all_coverage_html
        uses: actions/upload-artifact@v4
        if: always() && steps.merge_all_test_data.outcome == 'success'
        with:
          name: pytest-coverage-all
          path: services/api/coverage/html

      - name: Publish coverage HTML report link as PR comment
        uses: thollander/actions-comment-pull-request@v3
        if: always() && github.event_name == 'pull_request' && steps.upload_all_coverage_html.outcome == 'success'
        with:
          message: |
            [Link to coverage HTML report (All)](${{ steps.upload_all_coverage_html.outputs.artifact-url }})
          comment-tag: coverage_html_report_comment_all