Use a larger runner for Windows

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.x86_64-pc-windows-msvc]
+linker = "rust-lld.exe"

.github/workflows/ci_test.yml

@@ -3,36 +3,36 @@ name: 🐂📝 Continuous integration - Test Suite
 on:
   workflow_call:
 
+env:
+  AWS_REGION: us-west-1
+  AWS_ROLE: arn:aws:iam::213020545630:role/ci-test-integration-role
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
-  test:
-    name: Test Suite
+  rust-tests:
+    name: Rust Tests
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: ${{ matrix.shell }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-x64-8cpu-32ram-300ssd]
         include:
           - os: ubuntu-latest
             platform: linux
             shell: bash -leo pipefail {0}
           - os: macos-latest
             platform: macos
             shell: bash -leo pipefail {0}
-          - os: windows-latest
+          - os: windows-x64-8cpu-32ram-300ssd
             platform: windows
             shell: powershell
       fail-fast: false
 
-    env:
-      AWS_REGION: us-west-1
-      AWS_ROLE: arn:aws:iam::213020545630:role/ci-test-integration-role
-
-    permissions:
-      id-token: write
-      contents: read
-
     steps:
       - name: Free up disk space
         if: matrix.platform == 'linux'
@@ -55,10 +55,10 @@ jobs:
           df -h
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: ${{ env.AWS_ROLE }}
           role-duration-seconds: 3600
@@ -68,68 +68,27 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           rustflags: ${{ matrix.platform == 'windows' && '-D warnings -C link-arg=/DEBUG:NONE' || '-D warnings' }}
+          cache: true # This is the default--making it explicit. This uses Swatinem/rust-cache under the hood.
+          cache-all-crates: true
+          cache-workspace-crates: true
+          cache-key: rust
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: "3.11" # Test against the earliest supported Python version
 
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Only cache builds from main. Otherwise our caches will be too big and start to churn. (10 GB limit in GitHub Actions)
-          # Other branches will still use the cache but won't save it.
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-          workspaces: "."
-
       - name: Install dependencies (macOS)
         if: matrix.platform == 'macos'
         run: |
           brew update
           brew install redis pkg-config
           brew services start redis
 
-          # NOTE: This adds a lot of bloat and takes a long time to build.
-          #       we have ffmpeg disabled by default in the CI pipeline
-          # Set PKG_CONFIG_PATH so pkg-config can find FFmpeg libraries
-          # brew install ffmpeg@7 imagemagick
-          # echo "PKG_CONFIG_PATH=$(brew --prefix ffmpeg@7)/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV
-
-      # NOTE: This adds a lot of bloat to the Windows image and takes a long time to build.
-      #       we have ffmpeg disabled by default in the CI pipeline
-      # - name: Install dependencies (Linux)
-      #   if: matrix.platform == 'linux'
-      #   run: |
-      #     sudo apt-get update
-      #     sudo apt install -y clang libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev pkg-config
-
-      # - name: Install dependencies (Windows)
-      #   if: matrix.platform == 'windows'
-      #   run: |
-      #     # Install clang (via llvm) and pkg-config via Chocolatey
-      #     choco install -y llvm pkgconfiglite
-
-      #     # Install vcpkg for FFmpeg static development libraries (libavcodec, libavformat, libavutil, libavfilter, libavdevice)
-      #     # Using x64-windows-static-md for static libraries with dynamic MSVC runtime (better compatibility)
-      #     git clone https://github.com/microsoft/vcpkg.git C:\vcpkg
-      #     cd C:\vcpkg
-      #     .\bootstrap-vcpkg.bat
-
-      #     # Install FFmpeg (includes libavutil, libavcodec, libavformat, libavfilter, libavdevice)
-      #     # FFmpeg installation provides libavutil.pc and other .pc files needed by pkg-config
-      #     .\vcpkg.exe install ffmpeg:x64-windows-static-md
-
-      #     # Set VCPKG_ROOT for cargo to find FFmpeg libraries
-      #     echo "VCPKG_ROOT=C:\vcpkg" >> $env:GITHUB_ENV
-
-      #     # Set PKG_CONFIG_PATH so pkg-config can find libavutil.pc and other FFmpeg .pc files
-      #     # vcpkg installs pkg-config files in the installed/<triplet>/lib/pkgconfig directory
-      #     # This must be set before cargo build runs, as ffmpeg-sys-next uses pkg-config during build
-      #     $pkgConfigPath = "C:\vcpkg\installed\x64-windows-static-md\lib\pkgconfig"
-      #     echo "PKG_CONFIG_PATH=$pkgConfigPath" >> $env:GITHUB_ENV
-
       - name: Install nextest runner
-        uses: taiki-e/install-action@nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: nextest@0.9.114
 
       # Oxen Rust Tests
       - name: Setup test directories (Linux/macOS)
@@ -144,11 +103,7 @@ jobs:
           mkdir .\data\test\runs
 
       - name: Build oxen rust project
-        run: |
-          cargo build --workspace
-        env:
-          TEMP: ${{ matrix.platform == 'windows' && 'D:\a\_temp' || runner.temp }}
-          TMP: ${{ matrix.platform == 'windows' && 'D:\a\_temp' || runner.temp }}
+        run: cargo build --workspace
 
       - name: Setup oxen-server user (Linux/macOS)
         if: matrix.platform != 'windows'
@@ -171,7 +126,7 @@ jobs:
       - name: Run oxen rust tests (Windows)
         if: matrix.platform == 'windows'
         run: |
-          Start-Process -FilePath "${{ github.workspace }}\target\debug\oxen-server.exe" -WindowStyle Hidden -ArgumentList "start"
+          Start-Process ".\target\debug\oxen-server.exe" -ArgumentList "start"
           cargo nextest run --workspace --profile ci
 
       # CLI Tests
@@ -187,45 +142,158 @@ jobs:
       - name: Run CLI tests (Windows)
         if: matrix.platform == 'windows'
         env:
-          PATH: ${{ env.PATH }};${{ github.workspace }}/target/debug
+          PATH: ${{ env.PATH }};${{ github.workspace }}\target\debug
         run: |
           cd ${{ github.workspace }}\cli-test
           uv sync
           uv run pytest tests -v
 
-      # Oxen Python Tests
-      - name: Copy binaries for Python tests (Linux/macOS)
+  python-tests:
+    name: Python Tests
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-x64-8cpu-32ram-300ssd]
+        include:
+          - os: ubuntu-latest
+            platform: linux
+            shell: bash -leo pipefail {0}
+          - os: macos-latest
+            platform: macos
+            shell: bash -leo pipefail {0}
+          - os: windows-x64-8cpu-32ram-300ssd
+            platform: windows
+            shell: powershell
+      fail-fast: false
+
+    steps:
+      - name: Free up disk space
+        if: matrix.platform == 'linux'
+        run: |
+          echo "=== Before cleanup ==="
+          df -h
+
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/share/swift
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo apt-get clean
+
+          echo "=== After cleanup ==="
+          df -h
+
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ env.AWS_ROLE }}
+          role-duration-seconds: 3600
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Setup Rust Toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: ${{ matrix.platform == 'windows' && '-D warnings -C link-arg=/DEBUG:NONE' || '-D warnings' }}
+          cache: true
+          cache-all-crates: true
+          cache-workspace-crates: true
+          cache-key: python
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies (macOS)
+        if: matrix.platform == 'macos'
+        run: |
+          brew update
+          brew install redis pkg-config
+          brew services start redis
+
+      - name: Setup Python venv
+        run: |
+          cd ${{ github.workspace }}/oxen-python
+          uv sync --no-install-project
+
+      # maturin compiles liboxen and all shared deps for the Python extension (dev profile).
+      # cargo build then (hopefully) reuses those artifacts and only links the CLI/server binaries.
+      - name: Build oxen (Linux/macOS)
+        if: matrix.platform != 'windows'
+        run: |
+          cd ${{ github.workspace }}/oxen-python
+          source .venv/bin/activate
+          maturin develop
+          cargo build -p oxen-cli -p oxen-server
+
+      - name: Build oxen (Windows)
+        if: matrix.platform == 'windows'
+        run: |
+          cd ${{ github.workspace }}\oxen-python
+          .venv\Scripts\Activate.ps1
+          maturin develop
+          cargo build -p oxen-cli -p oxen-server
+
+      - name: Setup test directories (Linux/macOS)
+        if: matrix.platform != 'windows'
+        run: |
+          mkdir -p data/test/runs
+          mkdir -p /tmp/oxen_sync/
+
+      - name: Setup test directories (Windows)
+        if: matrix.platform == 'windows'
+        run: |
+          mkdir .\data\test\runs
+
+      - name: Setup oxen-server user (Linux/macOS)
+        if: matrix.platform != 'windows'
+        run: |
+          ./target/debug/oxen-server add-user --email ox@oxen.ai --name Ox --output user_config.toml
+          cp user_config.toml data/test/config/user_config.toml
+
+      - name: Setup oxen-server user (Windows)
+        if: matrix.platform == 'windows'
+        run: |
+          .\target\debug\oxen-server.exe add-user --email ox@oxen.ai --name Ox --output user_config.toml
+          copy user_config.toml data\test\config\user_config.toml
+
+      - name: Start oxen-server (Linux/macOS)
+        if: matrix.platform != 'windows'
+        run: ./target/debug/oxen-server start &
+
+      - name: Start oxen-server (Windows)
+        if: matrix.platform == 'windows'
+        run: Start-Process ".\target\debug\oxen-server.exe" -ArgumentList "start"
+
+      - name: Configure oxen user (Linux/macOS)
         if: matrix.platform != 'windows'
         run: |
           cp ${{ github.workspace }}/target/debug/oxen ~/oxen
-          chmod +x ~/oxen ~/oxen
+          chmod +x ~/oxen
+          ~/oxen config --name "Bessie Testington" --email "bessie@yourcompany.com"
 
-      - name: Copy binaries for Python tests (Windows)
+      - name: Configure oxen user (Windows)
         if: matrix.platform == 'windows'
         run: |
           copy target\debug\oxen.exe ${{ github.workspace }}\oxen.exe
+          ${{ github.workspace }}\oxen.exe config --name "Bessie Testington" --email "bessie@yourcompany.com"
 
       - name: Run oxen-python tests (Linux/macOS)
         if: matrix.platform != 'windows'
         run: |
           cd ${{ github.workspace }}/oxen-python
-
-          ~/oxen config --name "Bessie Testington" --email "bessie@yourcompany.com"
-
-          uv sync --no-install-project
           source .venv/bin/activate
-          maturin develop
           pytest -s tests
 
       - name: Run oxen-python tests (Windows)
         if: matrix.platform == 'windows'
         run: |
           cd ${{ github.workspace }}\oxen-python
-
-          ${{ github.workspace }}\oxen.exe config --name "Bessie Testington" --email "bessie@yourcompany.com"
-
-          uv sync --no-install-project
-          dir .venv
           .venv\Scripts\Activate.ps1
-          maturin develop
           pytest -s tests --ignore=tests/test_data_frame.py --ignore=tests/test_embeddings.py --ignore=tests/test_fsspec_backend.py

bin/test-rust

@@ -53,6 +53,9 @@ cleanup() {
         echo "==> Removing test data..."
         rm -rf ./data/ox
         rm -rf ./data/test/runs
+        if [ -n "${OXEN_TEST_RUN_DIR:-}" ]; then
+            rm -rf "$OXEN_TEST_RUN_DIR"
+        fi
     else
         echo "==> Keeping test data in data/ox and data/test/runs"
     fi

crates/lib/src/core/v_latest/workspaces/files.rs

@@ -727,7 +727,7 @@ pub fn decompress_zip(zip_filepath: &PathBuf) -> Result<Vec<PathBuf>, OxenError>
 
     // Get the canonical (absolute) path of the parent directory
     let parent = match zip_filepath.parent() {
-        Some(p) => p.canonicalize()?,
+        Some(p) => crate::util::fs::canonicalize(p)?,
         None => std::env::current_dir()?,
     };
 

crates/lib/src/test.rs

@@ -45,7 +45,10 @@ pub static REPO_ROOT: LazyLock<PathBuf> = LazyLock::new(|| {
 pub static TEST_DATA_DIR: LazyLock<PathBuf> = LazyLock::new(|| REPO_ROOT.join("data"));
 
 pub fn test_run_dir() -> PathBuf {
-    TEST_DATA_DIR.join("test").join("runs")
+    match std::env::var("OXEN_TEST_RUN_DIR") {
+        Ok(dir) => PathBuf::from(dir),
+        Err(_) => TEST_DATA_DIR.join("test").join("runs"),
+    }
 }
 
 pub fn test_host() -> String {

crates/oxen-py/Cargo.toml

@@ -28,5 +28,5 @@ uuid = { workspace = true }
 [build-dependencies]
 bindgen = { version = "0.71.1", default-features = false, features = ["runtime"] }
 cc = { version = "1.0", features = ["parallel"] }
-glob = "0.3"
+glob = { workspace = true }
 pkg-config = { version = "0.3", optional = true }

crates/server/src/test.rs

@@ -16,8 +16,7 @@ use liboxen::test::init_test_env;
 
 pub fn get_sync_dir() -> Result<PathBuf, OxenError> {
     init_test_env();
-    let sync_dir =
-        liboxen::test::REPO_ROOT.join(format!("data/test/runs/{}", uuid::Uuid::new_v4()));
+    let sync_dir = liboxen::test::test_run_dir().join(uuid::Uuid::new_v4().to_string());
     util::fs::create_dir_all(&sync_dir)?;
     Ok(sync_dir)
 }