Merge from main into spiceai

.github/dependabot.yml

@@ -0,0 +1,16 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: 'github-actions' # See documentation for possible values
+    directory: '/' # Location of package manifests
+    schedule:
+      interval: 'weekly'
+
+  - package-ecosystem: 'cargo' # See documentation for possible values
+    directory: '/' # Location of package manifests
+    schedule:
+      interval: 'weekly'

.github/workflows/pr.yaml

@@ -13,10 +13,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: dtolnay/rust-toolchain@stable
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -25,132 +28,125 @@ jobs:
       - run: cargo clippy --all-features -- -D warnings
 
   build:
-    name: Build
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: dtolnay/rust-toolchain@stable
-
-      # Putting this into a GitHub Actions matrix will run a separate job per matrix item, whereas in theory
-      # this can re-use the existing build cache to go faster.
-      - name: Build without default features
-        run: cargo check --no-default-features
-
-      - name: Build with only duckdb
-        run: cargo check --no-default-features --features duckdb
-
-      - name: Build with only postgres
-        run: cargo check --no-default-features --features postgres
-
-      - name: Build with only sqlite
-        run: cargo check --no-default-features --features sqlite
-
-      - name: Build with only mysql
-        run: cargo check --no-default-features --features mysql
-
-  integration-test-mysql:
-    name: Tests mysql
+    name: Build (${{ matrix.features }})
     runs-on: ubuntu-latest
 
-    env:
-      MYSQL_DOCKER_IMAGE: public.ecr.aws/ubuntu/mysql:8.0-22.04_beta
+    strategy:
+      matrix:
+        features:
+          - 'no-default-features'
+          - 'clickhouse'
+          - 'duckdb'
+          - 'postgres'
+          - 'sqlite'
+          - 'mysql'
+          - 'flight'
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - name: Pull the MySQL images
-        run: |
-          docker pull ${{ env.MYSQL_DOCKER_IMAGE }}
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
 
-      - uses: ./.github/actions/setup-integration-test
-
-      - name: Run tests
-        run: cargo test --features mysql
-
-  integration-test-postgres:
-    name: Tests postgres
+      - name: Build with ${{ matrix.features }}
+        run: |
+          if [ "${{ matrix.features }}" = "no-default-features" ]; then
+            cargo check --no-default-features
+          else
+            cargo check --no-default-features --features ${{ matrix.features }}
+          fi
+
+  integration-test:
+    name: Tests
     runs-on: ubuntu-latest
 
     env:
-      PG_DOCKER_IMAGE: public.ecr.aws/docker/library/postgres:latest
+      PG_DOCKER_IMAGE: ghcr.io/cloudnative-pg/postgresql:16-bookworm
+      MYSQL_DOCKER_IMAGE: public.ecr.aws/ubuntu/mysql:8.0-22.04_beta
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - uses: ./.github/actions/setup-integration-test
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
 
-      - name: Pull the Postgres images
+      - name: Pull the Postgres/MySQL images
         run: |
           docker pull ${{ env.PG_DOCKER_IMAGE }}
+          docker pull ${{ env.MYSQL_DOCKER_IMAGE }}
 
-      - name: Run tests
-        run: cargo test --features postgres
-
-  integration-test-sqlite:
-    name: Tests sqlite
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: dtolnay/rust-toolchain@stable
-
-      - uses: ./.github/actions/setup-integration-test
-
-      - name: Run tests
-        run: cargo test --features sqlite
-
-  integration-test-duckdb:
-    name: Tests duckdb
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: dtolnay/rust-toolchain@stable
-
-      - uses: ./.github/actions/setup-integration-test
+      - name: Free Disk Space
+        run: |
+          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+          sudo rm -rf \
+            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
+            /usr/lib/jvm || true
+          echo "some directories deleted"
+          sudo apt install aptitude -y >/dev/null 2>&1
+          sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
+            esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
+            google-cloud-sdk imagemagick \
+            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
+            mercurial apt-transport-https mono-complete libmysqlclient \
+            yarn chrpath libssl-dev libxft-dev \
+            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
+            snmp pollinate libpq-dev postgresql-client powershell ruby-full \
+            sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
+            -y -f >/dev/null 2>&1
+          sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1
+          sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
+          sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
+          sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
+          sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
+          sudo apt-get autoremove -y >/dev/null 2>&1
+          sudo apt-get autoclean -y >/dev/null 2>&1
+          echo "some packages purged"
+          df -h
+
+      - name: Install ODBC & Sqlite
+        run: |
+          sudo apt-get install -y unixodbc-dev
+          sudo apt-get install -y libsqlite3-dev
 
       - name: Run tests
-        run: cargo test --features duckdb
+        run: make test
 
-  integration-test-flight:
-    name: Tests flight
+  python-integration-test:
+    name: Python Tests
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - uses: ./.github/actions/setup-integration-test
-
-      - name: Run tests
-        run: cargo test --features flight
-
-  integration-test-mongodb:
-    name: Tests mongoDB
-    runs-on: ubuntu-latest
-
-    env:
-      MONGODB_DOCKER_IMAGE: public.ecr.aws/docker/library/mongo:7
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: dtolnay/rust-toolchain@stable
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            .
+            python
 
-      - uses: ./.github/actions/setup-integration-test
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: 'python/pyproject.toml'
 
-      - name: Pull the MongoDB images
+      - name: Install ODBC, Sqlite and Roapi
         run: |
-          docker pull ${{ env.MONGODB_DOCKER_IMAGE }}
+          sudo apt-get install -y unixodbc-dev libsqliteodbc
+          sudo apt-get install -y libsqlite3-dev
+          cargo install --locked --git https://github.com/roapi/roapi --branch main --bins roapi
 
-      - name: Run tests
-        run: cargo test --features mongodb
+      - name: Build Python package and run tests
+        run: |
+          cd python
+          uv sync --dev --no-install-package datafusion
+          uv run --no-project maturin develop --uv --release
+          cd python/tests
+          uv run --no-project pytest -v .

.gitignore

@@ -2,6 +2,7 @@
 # will have compiled files and executables
 debug/
 target/
+.cargo
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
@@ -17,3 +18,4 @@ target/
 .DS_Store
 
 .idea
+.vscode