updated to kernel v0.14 and added a dev container

Author: mightyshazam

.devcontainer/devcontainer.json

@@ -0,0 +1,40 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/dotnet
+{
+	"name": "C# (.NET)",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/dotnet:1-9.0-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/dotnet:2": {
+			"version": "8.0"
+		},
+		"ghcr.io/devcontainers/features/rust:1": {
+			"version": "1.88"
+		},
+		// "ghcr.io/lee-orr/rusty-dev-containers/cargo-llvm-cov:0": {},
+		"ghcr.io/devcontainers-community/features/llvm": {
+			"version": "20"
+		}
+	},
+	"onCreateCommand": "bash tools/clangsharp/install_libraries.sh",
+	"postStartCommand": "git submodule update"
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [5000, 5001],
+	// "portsAttributes": {
+	//		"5001": {
+	//			"protocol": "https"
+	//		}
+	// }
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "dotnet restore",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.github/workflows/build.yml

@@ -55,14 +55,15 @@ jobs:
       - name: Build
         # We intentionally just build debug mode in this CI. We build before
         # regen to confirm no interop header changes.
-        run: dotnet build
+        run: dotnet build -C Release
 
       - name: Regen confirm unchanged
         if: ${{ matrix.checkTarget }}
         # Needs to be 16.0.0 until https://github.com/dotnet/ClangSharp/pull/506 released
         run: |
-          dotnet tool install --global --version 16.0.0 ClangSharpPInvokeGenerator
-          # ClangSharpPInvokeGenerator @src/DeltaLake/Bridge/GenerateInterop.rsp
+          dotnet tool install --global --version 20.1.2 ClangSharpPInvokeGenerator
+          ClangSharpPInvokeGenerator @src/DeltaLake/Bridge/GenerateInterop.rsp
+          ClangSharpPInvokeGenerator -D DEFINE_DEFAULT_ENGINE_BASE=1 @src/DeltaLake/Kernel/GenerateInterop.rsp
           npx doctoc README.md
           git config --global core.safecrlf false
           git diff --exit-code

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "src/DeltaLake/Kernel/delta-kernel-rs"]
 	path = src/DeltaLake/Kernel/delta-kernel-rs
-	url = https://github.com/mightyshazam/delta-kernel-rs
+	url = https://github.com/delta-io/delta-kernel-rs

CONTRIBUTING.md

@@ -4,6 +4,13 @@
   - [Build C# Interop](#build-c-interop)
 - [Regenerating API docs](#regenerating-api-docs)
 
+### Using the dev container
+The devcontainer comes preloaded with all of the requisite dependencies.
+It includes a `Makefile` to generate bindings.  
+Use `make generate-bindings` to generate everything.
+Use `make generate-bridge-bindings` for the bridge code.
+Use `make generate-kernel-bindings` for the kernel code
+
 ### Rebuilding Rust extension and interop layer
 
 To regen core interop from header, install
@@ -67,11 +74,11 @@ $CSHARP_FRIENDLY_FFI_HEADER = "$GIT_ROOT/src/DeltaLake/Kernel/include/${HEADER_F
 
 Copy-Item -Path $GENERATED_FFI_HEADER -Destination $CSHARP_FRIENDLY_FFI_HEADER -Force
 
-ClangSharpPInvokeGenerator -D DEFINE_DEFAULT_ENGINE=1 @src/DeltaLake/Kernel/GenerateInterop.rsp
+ClangSharpPInvokeGenerator -D DEFINE_DEFAULT_ENGINE_BASE=1 @src/DeltaLake/Kernel/GenerateInterop.rsp
 Alternatively, if you have an environment that is not particularly cooperative, you can use the included
 docker container and run 
 ```bash
-docker run --mount type=bind,src=.,dst=/app clangsharp -D DEFINE_DEFAULT_ENGINE=1 @src/DeltaLake/Kernel/GenerateInterop.rsp
+docker run --mount type=bind,src=.,dst=/app clangsharp -D DEFINE_DEFAULT_ENGINE_RUSTLS=1 @src/DeltaLake/Kernel/GenerateInterop.rsp
 ```
 # Post-processing script to remove Mangled entrpoints - ClangSharpPInvokeGenerator
 # does not seem to have a built-in arg that does this.

Makefile

@@ -0,0 +1,10 @@
+generate-kernel-bindings:
+	cargo build --manifest-path src/DeltaLake/Kernel/delta-kernel-rs/ffi/Cargo.toml --no-default-features --features "default-engine-rustls"
+	cp src/DeltaLake/Kernel/delta-kernel-rs/target/ffi-headers/delta_kernel_ffi.h src/DeltaLake/Kernel/include/
+	ClangSharpPInvokeGenerator -I "$$(llvm-config --libdir)/clang/20/include" -D DEFINE_DEFAULT_ENGINE_BASE=1 @src/DeltaLake/Kernel/GenerateInterop.rsp
+
+generate-bridge-bindings:
+	cargo build --manifest-path src/DeltaLake/Bridge/Cargo.toml
+	ClangSharpPInvokeGenerator -I "$$(llvm-config --libdir)/clang/20/include" @src/DeltaLake/Bridge/GenerateInterop.rsp
+
+generate-bindings: generate-bridge-bindings generate-kernel-bindings

src/DeltaLake/Bridge/Interop/Interop.cs

@@ -176,7 +176,7 @@ internal unsafe partial struct TableCreatOptions
     internal unsafe partial struct TableOptions
     {
         [NativeTypeName("int64_t")]
-        public long version;
+        public IntPtr version;
 
         [NativeTypeName("struct Map *")]
         public Map* storage_options;
@@ -258,7 +258,7 @@ internal unsafe partial struct VacuumOptions
         public byte dry_run;
 
         [NativeTypeName("uint64_t")]
-        public ulong retention_hours;
+        public UIntPtr retention_hours;
 
         [NativeTypeName("bool")]
         public byte enforce_retention_duration;
@@ -303,31 +303,31 @@ internal unsafe partial struct Dictionary
     internal unsafe partial struct TableMetadata
     {
         [NativeTypeName("const char *")]
-        public sbyte* id;
+        public byte* id;
 
         [NativeTypeName("const char *")]
-        public sbyte* name;
+        public byte* name;
 
         [NativeTypeName("const char *")]
-        public sbyte* description;
+        public byte* description;
 
         [NativeTypeName("const char *")]
-        public sbyte* format_provider;
+        public byte* format_provider;
 
         [NativeTypeName("struct Dictionary")]
         public Dictionary format_options;
 
         [NativeTypeName("const char *")]
-        public sbyte* schema_string;
+        public byte* schema_string;
 
         [NativeTypeName("char **")]
-        public sbyte** partition_columns;
+        public byte** partition_columns;
 
         [NativeTypeName("uintptr_t")]
         public UIntPtr partition_columns_count;
 
         [NativeTypeName("int64_t")]
-        public long created_time;
+        public IntPtr created_time;
 
         [NativeTypeName("struct Dictionary")]
         public Dictionary configuration;
@@ -424,13 +424,13 @@ internal static unsafe partial class Methods
         public static extern void history([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("uintptr_t")] UIntPtr limit, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("GenericErrorCallback")] IntPtr callback);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void table_update_incremental([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] long max_version, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
+        public static extern void table_update_incremental([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] IntPtr max_version, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void table_load_version([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] long version, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
+        public static extern void table_load_version([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] IntPtr version, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void table_load_with_datetime([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] long ts_milliseconds, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
+        public static extern void table_load_with_datetime([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] IntPtr ts_milliseconds, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
         public static extern void table_merge([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* delta_table, [NativeTypeName("struct ByteArrayRef * _Nonnull")] ByteArrayRef* query, [NativeTypeName("void * _Nonnull")] void* stream, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("GenericErrorCallback")] IntPtr callback);
@@ -440,7 +440,7 @@ internal static unsafe partial class Methods
         public static extern ProtocolResponse table_protocol_versions([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
-        public static extern void table_restore([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] long version_or_timestamp, [NativeTypeName("bool")] byte is_timestamp, [NativeTypeName("bool")] byte ignore_missing_files, [NativeTypeName("bool")] byte protocol_downgrade_allowed, [NativeTypeName("struct Map *")] Map* custom_metadata, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
+        public static extern void table_restore([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("int64_t")] IntPtr version_or_timestamp, [NativeTypeName("bool")] byte is_timestamp, [NativeTypeName("bool")] byte ignore_missing_files, [NativeTypeName("bool")] byte protocol_downgrade_allowed, [NativeTypeName("struct Map *")] Map* custom_metadata, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("TableEmptyCallback")] IntPtr callback);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
         public static extern void table_update([NativeTypeName("struct Runtime * _Nonnull")] Runtime* runtime, [NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table, [NativeTypeName("struct ByteArrayRef * _Nonnull")] ByteArrayRef* query, [NativeTypeName("const struct CancellationToken *")] CancellationToken* cancellation_token, [NativeTypeName("GenericErrorCallback")] IntPtr callback);
@@ -469,7 +469,7 @@ internal static unsafe partial class Methods
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
         [return: NativeTypeName("int64_t")]
-        public static extern long table_version([NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table_handle);
+        public static extern IntPtr table_version([NativeTypeName("struct RawDeltaTable * _Nonnull")] RawDeltaTable* table_handle);
 
         [DllImport("delta_rs_bridge", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
         [return: NativeTypeName("struct MetadataOrError")]