feat: upgrade to latest kernel and delta-rs

.devcontainer/devcontainer.json

@@ -0,0 +1,40 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/dotnet
+{
+	"name": "C# (.NET)",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/dotnet:1-9.0-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/dotnet:2": {
+			"version": "8.0"
+		},
+		"ghcr.io/devcontainers/features/rust:1": {
+			"version": "1.88"
+		},
+		// "ghcr.io/lee-orr/rusty-dev-containers/cargo-llvm-cov:0": {},
+		"ghcr.io/devcontainers-community/features/llvm": {
+			"version": "20"
+		}
+	},
+	"onCreateCommand": "bash tools/clangsharp/install_libraries.sh",
+	"postStartCommand": "git submodule update"
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [5000, 5001],
+	// "portsAttributes": {
+	//		"5001": {
+	//			"protocol": "https"
+	//		}
+	// }
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "dotnet restore",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.github/codeql-config.yml

@@ -0,0 +1,4 @@
+paths:
+ - 'src'
+paths-ignore:
+ - 'src/DeltaLake/Kernel/delta-kernel-rs/**'

.github/workflows/build.yml

@@ -16,9 +16,8 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         include:
-          - os: windows-latest
-            checkTarget: true
           - os: ubuntu-latest
+            checkTarget: true
             docsTarget: true
             # Code coverage doesn't seem to work when PR's are opened from forked repositories, see discussion:
             #
@@ -45,22 +44,41 @@ jobs:
       - name: Setup Rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          workspaces: src/DeltaLake/Bridge
+          workspaces: |
+            src/DeltaLake/Bridge
+            src/DeltaLake/Kernel/delta-kernel-rs
 
       - name: Setup .NET
         uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: | 
+            8.0.x
+            9.0.x
 
       - name: Build
         # We intentionally just build debug mode in this CI. We build before
         # regen to confirm no interop header changes.
-        run: dotnet build
+        run: dotnet build -c Release
+
+      - name: Install LLVM and Clang
+        if: ${{ matrix.checkTarget }}
+        uses: KyleMayes/install-llvm-action@v2
+        with:
+          version: "20.1.2"
 
       - name: Regen confirm unchanged
         if: ${{ matrix.checkTarget }}
         # Needs to be 16.0.0 until https://github.com/dotnet/ClangSharp/pull/506 released
         run: |
-          dotnet tool install --global --version 16.0.0 ClangSharpPInvokeGenerator
-          # ClangSharpPInvokeGenerator @src/DeltaLake/Bridge/GenerateInterop.rsp
+          dotnet tool install --global --version 20.1.2 ClangSharpPInvokeGenerator
+          curl -L  https://www.nuget.org/api/v2/package/libclang.runtime.linux-x64/20.1.2 -o /tmp/libclang.zip
+          curl -L https://www.nuget.org/api/v2/package/libClangSharp.runtime.linux-x64/20.1.2 -o /tmp/libclangsharp.zip
+          unzip -od /tmp/libclang /tmp/libclang.zip
+          unzip -od /tmp/libclangsharp /tmp/libclangsharp.zip
+          cp /tmp/libclang/runtimes/linux-x64/native/libclang.so /home/runner/.dotnet/tools/.store/clangsharppinvokegenerator/20.1.2/clangsharppinvokegenerator/20.1.2/tools/net8.0/any/
+          cp /tmp/libclangsharp/runtimes/linux-x64/native/libClangSharp.so /home/runner/.dotnet/tools/.store/clangsharppinvokegenerator/20.1.2/clangsharppinvokegenerator/20.1.2/tools/net8.0/any/
+          # temporarily disable generation test
+          # make generate-bindings
           npx doctoc README.md
           git config --global core.safecrlf false
           git diff --exit-code

.github/workflows/codeql.yml

@@ -55,6 +55,13 @@ jobs:
       - name: Checkout specific delta-kernel-rs release
         shell: bash
         run: chmod -R +x .scripts/*.sh && .scripts/checkout-delta-kernel-rs.sh
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: | 
+            8.0.x
+            9.0.x
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/package.yml

@@ -9,44 +9,42 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-rust-libraries:
+  build-bridge-library:
     strategy:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, ubuntu-arm, macos-latest, macos-arm, windows-latest]
         include:
           - os: ubuntu-latest
             bridge-out-file: libdelta_rs_bridge.so
-            kernel-out-file: libdelta_kernel_ffi.so
             out-prefix: linux-x64
+            # We use cross to build on x64 for compatibility with older glibc
+            use-cross: true
+            target: x86_64-unknown-linux-gnu
           - os: ubuntu-arm
             bridge-out-file: libdelta_rs_bridge.so
-            kernel-out-file: libdelta_kernel_ffi.so
             out-prefix: linux-arm64
             runsOn: ubuntu-latest
             use-cross: true
-            alternative-target: aarch64-unknown-linux-gnu
+            target: aarch64-unknown-linux-gnu
           - os: macos-latest
             bridge-out-file: libdelta_rs_bridge.dylib
-            kernel-out-file: libdelta_kernel_ffi.dylib
             out-prefix: osx-x64
-            alternative-target: x86_64-apple-darwin
+            target: x86_64-apple-darwin
           - os: macos-arm
             bridge-out-file: libdelta_rs_bridge.dylib
-            kernel-out-file: libdelta_kernel_ffi.dylib
             out-prefix: osx-arm64
             runsOn: macos-latest
           - os: windows-latest
             bridge-out-file: delta_rs_bridge.dll
-            kernel-out-file: delta_kernel_ffi.dll
             out-prefix: win-x64
     runs-on: ${{ matrix.runsOn || matrix.os }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
         with:
           submodules: false
-      
+
       - name: Checkout specific delta-kernel-rs release
         shell: bash
         run: chmod -R +x .scripts/*.sh && .scripts/checkout-delta-kernel-rs.sh
@@ -60,83 +58,151 @@ jobs:
         uses: Swatinem/rust-cache@v2.7.3
         with:
           workspaces: src/DeltaLake/Bridge
-          key: ${{ matrix.os }}
+          key: ${{ matrix.os }}-bridge
+
+      - name: Compute target variables
+        id: target-info
+        run: |
+          if [ -n "${{ matrix.target }}" ]; then
+            echo "target-arg=--target ${{ matrix.target }}" >> $GITHUB_OUTPUT
+            echo "bridge-out-path=src/DeltaLake/Bridge/target/${{ matrix.target }}/release/${{ matrix.bridge-out-file }}" >> $GITHUB_OUTPUT
+          else
+            echo "target-arg=" >> $GITHUB_OUTPUT
+            echo "bridge-out-path=src/DeltaLake/Bridge/target/release/${{ matrix.bridge-out-file }}" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
 
       - name: Add alternative Rust target
-        if: ${{ matrix.alternative-target != ''}}
+        if: ${{ matrix.target != ''}}
         run: |
-          rustup target install ${{ matrix.alternative-target }}
+          rustup target install ${{ matrix.target }}
 
       - name: Install cross
         if: ${{ matrix.use-cross }}
         run: |
           cargo install cross
 
-      - name: Build alternative target
-        if: ${{ matrix.alternative-target != '' && !matrix.use-cross }}
-        run: |
-          cargo build --manifest-path src/DeltaLake/Bridge/Cargo.toml --release --target ${{ matrix.alternative-target }}
-          cargo build --manifest-path src/DeltaLake/Kernel/delta-kernel-rs/ffi/Cargo.toml --release --features "delta_kernel/cloud" --target ${{ matrix.alternative-target }}
+      - name: Build Bridge
+        if: ${{ !matrix.use-cross }}
+        run: cargo build --manifest-path src/DeltaLake/Bridge/Cargo.toml --release ${{ steps.target-info.outputs.target-arg }}
 
-      - name: Build cross target
+      - name: Build Bridge with cross
         if: ${{ matrix.use-cross }}
         run: |
-          CROSS_CONFIG=src/DeltaLake/Bridge/cross/${{ matrix.alternative-target }}/Cross.toml \
-          cross build --manifest-path src/DeltaLake/Bridge/Cargo.toml --release --target ${{ matrix.alternative-target }}
-
-          CROSS_CONFIG=src/DeltaLake/Kernel/Build/cross/${{ matrix.alternative-target }}/Cross.toml \
-              cross build --manifest-path src/DeltaLake/Kernel/delta-kernel-rs/ffi/Cargo.toml --release \
-                          --features "delta_kernel/cloud" \
-                          --target ${{ matrix.alternative-target }}
-
-      - name: Build Bridge
-        if: ${{ !matrix.alternative-target }}
-        run: cargo build --manifest-path src/DeltaLake/Bridge/Cargo.toml --release
-
-      - name: Build Kernel
-        if: ${{ !matrix.alternative-target }}
-        run: cargo build --manifest-path src/DeltaLake/Kernel/delta-kernel-rs/ffi/Cargo.toml --release --features "delta_kernel/cloud"
+          CROSS_CONFIG=src/DeltaLake/Bridge/Cross.toml \
+          cross build --manifest-path src/DeltaLake/Bridge/Cargo.toml --release --target ${{ matrix.target }}
 
       - name: Upload bridge library
-        if: ${{ !matrix.alternative-target }}
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.out-prefix }}-bridge
-          path: src/DeltaLake/Bridge/target/release/${{ matrix.bridge-out-file }}
-
-      - name: Upload kernel library
-        if: ${{ !matrix.alternative-target }}
-        uses: actions/upload-artifact@v4
+          path: ${{ steps.target-info.outputs.bridge-out-path }}
+          if-no-files-found: "error"
+
+  build-kernel-library:
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest, ubuntu-arm, macos-latest, macos-arm, windows-latest]
+        include:
+          - os: ubuntu-latest
+            kernel-out-file: libdelta_kernel_ffi.so
+            out-prefix: linux-x64
+            # We use cross to build on x64 for compatibility with older glibc
+            use-cross: true
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-arm
+            kernel-out-file: libdelta_kernel_ffi.so
+            out-prefix: linux-arm64
+            runsOn: ubuntu-latest
+            use-cross: true
+            target: aarch64-unknown-linux-gnu
+          - os: macos-latest
+            kernel-out-file: libdelta_kernel_ffi.dylib
+            out-prefix: osx-x64
+            target: x86_64-apple-darwin
+          - os: macos-arm
+            kernel-out-file: libdelta_kernel_ffi.dylib
+            out-prefix: osx-arm64
+            runsOn: macos-latest
+          - os: windows-latest
+            kernel-out-file: delta_kernel_ffi.dll
+            out-prefix: win-x64
+    runs-on: ${{ matrix.runsOn || matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
         with:
-          name: ${{ matrix.out-prefix }}-kernel
-          path: src/DeltaLake/Kernel/delta-kernel-rs/target/release/${{ matrix.kernel-out-file }}
+          submodules: false
 
-      - name: Upload bridge library alternative target
-        if: ${{ matrix.alternative-target != '' }}
-        uses: actions/upload-artifact@v4
+      - name: Checkout specific delta-kernel-rs release
+        shell: bash
+        run: chmod -R +x .scripts/*.sh && .scripts/checkout-delta-kernel-rs.sh
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
         with:
-          name: ${{ matrix.out-prefix }}-bridge
-          path: src/DeltaLake/Bridge/target/${{ matrix.alternative-target }}/release/${{ matrix.bridge-out-file }}
-          if-no-files-found: "error"
-
-      - name: Upload kernel library alternative target
-        if: ${{ matrix.alternative-target != '' }}
+          toolchain: stable
+
+      - name: Setup Rust cache
+        uses: Swatinem/rust-cache@v2.7.3
+        with:
+          workspaces: src/DeltaLake/Kernel/delta-kernel-rs
+          key: ${{ matrix.os }}-kernel
+
+      - name: Compute target variables
+        id: target-info
+        run: |
+          if [ -n "${{ matrix.target }}" ]; then
+            echo "target-arg=--target ${{ matrix.target }}" >> $GITHUB_OUTPUT
+            echo "kernel-out-path=src/DeltaLake/Kernel/delta-kernel-rs/target/${{ matrix.target }}/release/${{ matrix.kernel-out-file }}" >> $GITHUB_OUTPUT
+          else
+            echo "target-arg=" >> $GITHUB_OUTPUT
+            echo "kernel-out-path=src/DeltaLake/Kernel/delta-kernel-rs/target/release/${{ matrix.kernel-out-file }}" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
+
+      - name: Add alternative Rust target
+        if: ${{ matrix.target != ''}}
+        run: |
+          rustup target install ${{ matrix.target }}
+
+      - name: Install cross
+        if: ${{ matrix.use-cross }}
+        run: |
+          cargo install cross
+
+      - name: Build Kernel
+        if: ${{ !matrix.use-cross }}
+        run: cargo build --manifest-path src/DeltaLake/Kernel/delta-kernel-rs/ffi/Cargo.toml --release --no-default-features --features "default-engine-rustls" ${{ steps.target-info.outputs.target-arg }}
+
+      - name: Build Kernel with cross
+        if: ${{ matrix.use-cross }}
+        run: |
+          CROSS_CONFIG=src/DeltaLake/Kernel/Build/Cross.toml \
+          cross build --manifest-path src/DeltaLake/Kernel/delta-kernel-rs/ffi/Cargo.toml --release \
+                      --no-default-features \
+                      --features "default-engine-rustls" \
+                      --target ${{ matrix.target }}
+
+      - name: Upload kernel library
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.out-prefix }}-kernel
-          path: src/DeltaLake/Kernel/delta-kernel-rs/target/${{ matrix.alternative-target }}/release/${{ matrix.kernel-out-file }}
+          path: ${{ steps.target-info.outputs.kernel-out-path }}
           if-no-files-found: "error"
 
   build-nuget-package:
     needs:
-      - build-rust-libraries
+      - build-bridge-library
+      - build-kernel-library
     runs-on: windows-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
         with:
           submodules: false
-      
+
       - name: Checkout specific delta-kernel-rs release
         shell: bash
         run: chmod -R +x .scripts/*.sh && .scripts/checkout-delta-kernel-rs.sh
@@ -148,6 +214,10 @@ jobs:
 
       - name: Setup .NET
         uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: |
+            8.0.x
+            9.0.x
 
       - name: Build package
         run: dotnet pack src/DeltaLake/DeltaLake.csproj -c Release `

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "src/DeltaLake/Kernel/delta-kernel-rs"]
 	path = src/DeltaLake/Kernel/delta-kernel-rs
-	url = https://github.com/delta-incubator/delta-kernel-rs
+	url = https://github.com/delta-io/delta-kernel-rs