[CI] Fix public repo sync workflow (#807)

Author: jiahuei

.github/workflows/ci.yml

@@ -205,7 +205,7 @@ jobs:
         if: always() && steps.launch_services.outcome == 'success'
         run: |
           cp .env services/api/.env
-          cd services/api
+          pushd services/api
 
           if [ "${{ matrix.test-group }}" = "group1" ]; then
             DIRS=(tests --ignore=tests/gen_table/test_row_ops.py --ignore=tests/gen_table/test_row_ops_v2.py --ignore=tests/routers --ignore=tests/utils)
@@ -224,6 +224,12 @@ jobs:
               --junitxml=pytest_regular.xml \
               -m "not (${{ matrix.jamai-mode == 'cloud' && 'oss' || 'cloud' }} or stripe)" \
               "${DIRS[@]}"
+
+          # Shut down owl to allow coverage data to be flushed
+          popd
+          docker compose -p jm -f docker/compose.ci.yml down
+          # Copy Pytest coverage data
+          sudo find docker_data -type f -name '.coverage.*' -exec cp -v -t services/api/coverage/. {} +
         env:
           OWL_DB_PATH: postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl
           OWL_CLICKHOUSE_HOST: localhost
@@ -243,10 +249,6 @@ jobs:
         id: test_stripe
         if: matrix.jamai-mode == 'cloud' && matrix.test-group == 'group1' && steps.launch_services.outcome == 'success'
         run: |
-          # Shut down owl to allow coverage data to be flushed
-          docker compose -p jm -f docker/compose.ci.yml down
-          # Copy Pytest coverage data
-          sudo cp -r docker_data docker_data_tmp
           sudo rm -rf docker_data
 
           # Relaunch
@@ -274,9 +276,11 @@ jobs:
               -m stripe \
               tests
 
-          # Move existing coverage data
+          # Shut down owl to allow coverage data to be flushed
           popd
-          mv docker_data_tmp/owl/db/* docker_data/owl/db/. 2>/dev/null || true
+          docker compose -p jm -f docker/compose.ci.yml down
+          # Move existing coverage data
+          sudo find docker_data -type f -name '.coverage.*' -exec cp -v -t services/api/coverage/. {} +
         env:
           STRIPE_API_KEY: ${{ secrets.OWL_STRIPE_API_KEY }}
           OWL_STRIPE_API_KEY: ${{ secrets.OWL_STRIPE_API_KEY }}
@@ -313,15 +317,6 @@ jobs:
             [Link to logs (${{ matrix.jamai-mode }}, ${{ matrix.test-group }})](${{ steps.upload_logs.outputs.artifact-url }})
           comment-tag: logs_comment_${{ matrix.jamai-mode }}-${{ matrix.test-group }}
 
-      - name: Upload pytest coverage file
-        uses: actions/upload-artifact@v4
-        if: always() && steps.python_sdk_test.outcome == 'success'
-        with:
-          name: pytest-coverage-data-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
-          path: services/api/coverage
-          include-hidden-files: true
-          if-no-files-found: error
-
       - name: Merge JUnit XML and Coverage data
         id: merge_test_data
         if: always() && steps.launch_services.outcome == 'success'
@@ -331,7 +326,7 @@ jobs:
 
           # Combine coverage data
           coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \
-            docker_data/owl/db services/api/coverage
+            services/api/coverage
 
           # Merge JUnit XML files
           mkdir -p services/api/junit_xml
@@ -341,8 +336,8 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always() && steps.merge_test_data.outcome == 'success'
         with:
-          name: docker_data-coverage-data-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
-          path: docker_data/owl/db
+          name: owl-coverage-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
+          path: services/api/coverage
           include-hidden-files: true
           if-no-files-found: error
 
@@ -353,12 +348,6 @@ jobs:
           name: junit-xml-data-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
           path: services/api/junit_xml
 
-      - name: Log coverage data files
-        run: |
-          find docker_data/owl/db -type f | head -50
-          find services/api/coverage -type f | head -50
-          find services/api/junit_xml -type f | head -50
-
       - name: Generate coverage reports
         id: generate_coverage_report
         if: always() && steps.merge_test_data.outcome == 'success'
@@ -413,18 +402,11 @@ jobs:
           pushd services/api
           uv pip install --system -e .[test]
 
-      - name: Download pytest coverage data artifacts
-        uses: actions/download-artifact@v4
-        if: needs.sdk_tests.result == 'success'
-        with:
-          pattern: pytest-coverage-data-*
-          path: ./
-
       - name: Download coverage data artifacts
         uses: actions/download-artifact@v4
         if: needs.sdk_tests.result == 'success'
         with:
-          pattern: docker_data-coverage-data-*
+          pattern: owl-coverage-*
           path: ./
 
       - name: Download junit xml artifacts
@@ -437,29 +419,36 @@ jobs:
       - name: Log coverage data files
         if: needs.sdk_tests.result == 'success'
         run: |
-          find docker_data-coverage-data-* -type f | head -50
-          find pytest-coverage-data-* -type f | head -50
+          find owl-coverage-* -type f | head -50
           find junit-xml-data -type f | head -50
 
       - name: Merge JUnit XML and Coverage data (${{ matrix.jamai-mode }})
         id: merge_coverage
         if: needs.sdk_tests.result == 'success'
         run: |
           coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \
-            ./docker_data-coverage-data-${{ matrix.jamai-mode }}-group[1-4] \
-            ./pytest-coverage-data-${{ matrix.jamai-mode }}-group[1-4]
+            ./owl-coverage-${{ matrix.jamai-mode }}-group[1-4]
 
           # Merge JUnit XML files
-          junitparser merge --glob "junit-xml-data/junit-xml-data-${{ matrix.jamai-mode }}-*/pytest_*.xml" \
+          junitparser merge --glob "junit-xml-data/junit-xml-data-${{ matrix.jamai-mode }}*/pytest-*.xml" \
             junit-xml-data/pytest-${{ matrix.jamai-mode }}.xml
 
       - name: Generate coverage report
         if: always() && steps.merge_coverage.outcome == 'success'
         run: |
           cd services/api
+          coverage html --data-file=coverage/.coverage -d coverage/html-${{ matrix.jamai-mode }}
           coverage xml --data-file=coverage/.coverage -o coverage/coverage-${{ matrix.jamai-mode }}.xml
           coverage report --data-file=coverage/.coverage
 
+      - name: Upload ${{ matrix.jamai-mode }} coverage HTML report
+        id: upload_coverage_html
+        uses: actions/upload-artifact@v4
+        if: always() && steps.merge_coverage.outcome == 'success'
+        with:
+          name: pytest-coverage-${{ matrix.jamai-mode }}
+          path: services/api/coverage/html-${{ matrix.jamai-mode }}
+
       - name: Pytest coverage comment
         uses: MishaKav/pytest-coverage-comment@main
         if: always() && github.event_name == 'pull_request' && steps.merge_coverage.outcome == 'success'
@@ -475,11 +464,10 @@ jobs:
         if: needs.sdk_tests.result == 'success' && matrix.jamai-mode == 'oss'
         run: |
           coverage combine --keep --data-file=services/api/coverage/.coverage --rcfile=services/api/pyproject.toml \
-            ./docker_data-coverage-data-oss-group[1-4] \
-            ./pytest-coverage-data-oss-group[1-4]
+            ./owl-coverage-oss-group[1-4]
 
           # Merge JUnit XML files
-          junitparser merge --glob "junit-xml-data/junit-xml-data-*/pytest_*.xml" junit-xml-data/pytest.xml
+          junitparser merge --glob "junit-xml-data/junit-xml-data-*/pytest-*.xml" junit-xml-data/pytest.xml
 
       - name: Generate coverage reports
         id: generate_coverage_report
@@ -496,7 +484,6 @@ jobs:
         with:
           title: Coverage Report (all)
           pytest-xml-coverage-path: services/api/coverage/coverage.xml
-          junitxml-path: junit-xml-data/pytest.xml
           unique-id-for-comment: coverage_report_comment_all
           report-only-changed-files: true
 

.github/workflows/lint.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Node.js
         uses: actions/setup-node@v5

clients/python/src/jamaibase/client.py

@@ -3304,6 +3304,9 @@ async def import_table(
                 otherwise OkResponse.
         """
         migrate = kwargs.pop("migrate", False)  # Temporary, may be removed anytime
+        reupload = (
+            kwargs.pop("reupload", False) or not migrate
+        )  # Temporary, may be removed anytime, but if migrate == False then this must be True
         timeout = None if migrate else (kwargs.pop("timeout", None) or self.file_upload_timeout)
         v = "v1" if kwargs.pop("v1", False) else "v2"
         mime_type = "application/octet-stream"
@@ -3317,7 +3320,7 @@ async def import_table(
                 files={
                     "file": (filename, f, mime_type),
                 },
-                data=dict(**self._process_body(request), migrate=migrate),
+                data=dict(**self._process_body(request), migrate=migrate, reupload=reupload),
                 timeout=timeout,
                 **kwargs,
             )

docker/compose.oss.yml

@@ -0,0 +1,4 @@
+include:
+  - path:
+      - compose.base.yml
+      - override.oss.yml

docker/override.oss.yml

@@ -0,0 +1,21 @@
+services:
+  vmauth:
+    ports:
+      - 8427:8427
+
+  pgbouncer:
+    ports:
+      - 5432:5432
+
+  minio:
+    ports:
+      - 9000:9000
+      - 9001:9001
+
+  owl:
+    ports:
+      - "${API_PORT:-6969}:${OWL_PORT:-6969}"
+
+  frontend:
+    ports:
+      - "${FRONTEND_PORT:-4000}:4000"

docs/migration_v1_to_v2.md

@@ -0,0 +1,116 @@
+# JamAIBase v1 to v2 migration guide
+
+## Overview
+
+This guide helps migrate your JamAIBase instance from v1 to v2.
+
+## Pre-requisites
+
+- During migration, you will need to have both instances of JamAIBase running concurrently
+- For the existing v1 JamAIBase, please re-launch with the new port mapping, to prevent clashing with the v2 instance.
+
+## Launching v1 with updated port mapping
+
+- Update the .env
+
+```
+API_PORT=26969
+```
+
+- Update the docker/compose.cpu.yml
+
+```
+# under the owl service
+# from
+# image: jamai/owl
+# to
+image: jamai/owl:v1
+```
+
+- Launch v1 JamAIBase with a subset of the services
+
+```
+cd /path/to/your/v1/dir
+docker compose --env-file .env -f docker/compose.cpu.yml up --scale infinity=0 --scale starling=0 --scale frontend=0 --scale docling=0
+```
+
+> if on windows (powershell)
+
+```
+cd C:\path\to\your\v1\dir
+docker compose -p jamai --env-file .env -f docker/compose.oss.yml up
+```
+
+---
+
+## Launching v2
+
+- Launch v2 JamAIBase
+
+```
+cd /path/to/your/v2/dir
+docker compose -p jamai --env-file .env -f docker/compose.oss.yml up
+```
+
+> if on windows (powershell)
+
+```
+cd C:\path\to\your\v2\dir
+docker compose -p jamai --env-file .env -f docker/compose.oss.yml up
+```
+
+## Run the migration
+
+- Install JamAIBase locally
+
+```
+cd /path/to/your/v2/dir
+cd services/api
+pip install -e .
+cd -
+cd clients/python
+pip install -e .
+```
+
+> if on windows (powershell)
+
+```
+cd C:\path\to\your\v2\dir
+
+cd services\api
+pip install -e .
+
+cd ..\..
+cd clients\python
+pip install -e .
+```
+
+- Run the migration scripts
+
+```
+# Check your v1 db path
+export OWL_V1_DB=/path/to/your/v1/db/main.db
+export OWL_V1_FILE=/path/to/your/v1/file # path to your v1 file directory
+export OWL_REDIS_HOST=localhost
+export OWL_DB_PATH=postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl
+
+python scripts/oss_migrate.py --db_v1_path ${OWL_V1_DB} --migrate --reset --api_base_dst http://localhost:6969/api --api_base_src http://localhost:26969/api --v1_file_path ${OWL_V1_FILE} --s3_endpoint_dst http://localhost:9000
+```
+
+> if on windows (powershell)
+
+```
+# Set environment variables for the current PowerShell session
+# Use Windows-style paths for your files and databases
+ $env:OWL_V1_DB = "C:\path\to\your\v1\db\main.db"
+ $env:OWL_V1_FILE = "C:\path\to\your\v1\file"
+ $env:OWL_REDIS_HOST = "localhost"
+ $env:OWL_DB_PATH = "postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl"
+
+# Run the migration script
+# Note the use of '.\' to execute a script in the current directory
+# and $env:VAR_NAME to reference environment variables
+python .\scripts\oss_migrate.py --db_v1_path $env:OWL_V1_DB --migrate --reset --api_base_dst http://localhost:6969/api --api_base_src http://localhost:26969/api --v1_file_path $env:OWL_V1_FILE --s3_endpoint_dst http://localhost:9000
+```
+
+- If everything successful, you can close the v1 instance