From 737496dabd0ec76d747a3a4f2914c2e45fd10712 Mon Sep 17 00:00:00 2001 From: tecc Date: Sun, 17 Aug 2025 19:07:22 +0200 Subject: [PATCH 01/28] feat: Add struct `git_odb_stream` and enum `git_odb_stream_t` Added type definition for struct `libgit2-sys::git_odb_stream`. Previously incorrectly defined as an opaque enum. See git2/odb_backend.h line 196. Added type definition for enum `git_odb_stream_t`. I believe it corresponds to the `mode` field of `git_odb_stream`, but that's just a guess. See git2/odb_backend.h line 182. --- libgit2-sys/lib.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index 7e1ac1e998..599c86ba44 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -87,7 +87,28 @@ pub enum git_reflog_entry {} pub enum git_describe_result {} pub enum git_packbuilder {} pub enum git_odb {} -pub enum git_odb_stream {} + +#[repr(C)] +pub struct git_odb_stream { + pub backend: *mut git_odb_backend, + pub mode: c_uint, + pub hash_ctx: *mut c_void, + pub declared_size: git_object_size_t, + pub received_bytes: git_object_size_t, + pub read: Option c_int>, + pub write: Option c_int>, + pub finalize_write: Option c_int>, + pub free: Option +} + +git_enum! { + pub enum git_odb_stream_t: c_int { + GIT_STREAM_RDONLY = 2, + GIT_STREAM_WRONLY = 4, + GIT_STREAM_RW = 6, + } +} + pub enum git_odb_object {} pub enum git_worktree {} pub enum git_transaction {} From 39509e7cb8770d8b93b0a0cdc4045eb99bd75fdf Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 01:39:18 +0200 Subject: [PATCH 02/28] feat: Add API from `git2/sys/config.h` Added struct `git_config_backend_entry`. See git2/sys/config.h line 27. Added struct `git_config_iterator`. Previously incorrectly defined as an empty enum. See git2/sys/config.h line 49. Added struct `git_config_backend`. See git2/sys/config.h line 69. Added constant `GIT_CONFIG_BACKEND_VERSION`. See git2/sys/config.h line 103. Added struct `git_config_backend_memory_options`. See git2/sys/config.h line 148. Added constant `GIT_CONFIG_BACKEND_MEMORY_OPTIONS_VERSION`. See git2/sys/config.h line 165. Added function `git_config_add_backend`. See git2/sys/config.h line 140. Added function `git_config_backend_from_string`. See git2/sys/config.h line 181. Added function `git_config_backend_from_values`. See git2/sys/config.h line 197. Added function `git_config_init_backend`. See git2/sys/config.h line 116. --- libgit2-sys/lib.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index 599c86ba44..abe74dbfb0 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -55,7 +55,68 @@ pub enum git_branch_iterator {} pub enum git_blame {} pub enum git_commit {} pub enum git_config {} -pub enum git_config_iterator {} + +#[repr(C)] +pub struct git_config_backend_entry { + pub entry: git_config_entry, + pub free: Option, +} + +#[repr(C)] +pub struct git_config_iterator { + pub backend: *mut git_config_backend, + pub flags: c_uint, + pub next: Option< + extern "C" fn(*mut *mut git_config_backend_entry, *mut git_config_iterator) -> c_int, + >, + pub free: Option, +} + +#[repr(C)] +pub struct git_config_backend { + pub version: c_uint, + pub readonly: c_int, + pub cfg: *mut git_config, + pub open: Option< + extern "C" fn(*mut git_config_backend, git_config_level_t, *const git_repository) -> c_int, + >, + pub get: Option< + extern "C" fn( + *mut git_config_backend, + *const c_char, + *mut *mut git_config_backend_entry, + ) -> c_int, + >, + pub set: Option c_int>, + pub set_multivar: Option< + extern "C" fn( + *mut git_config_backend, + *const c_char, + *const c_char, + *const c_char, + ) -> c_int, + >, + pub del: Option c_int>, + pub del_multivar: + Option c_int>, + pub snapshot: + Option c_int>, + pub lock: Option c_int>, + pub unlock: Option c_int>, + pub free: Option, +} + +pub const GIT_CONFIG_BACKEND_VERSION: c_uint = 1; + +#[repr(C)] +pub struct git_config_backend_memory_options { + pub version: c_uint, + pub backend_type: *const c_char, + pub origin_path: *const c_char +} + +pub const GIT_CONFIG_BACKEND_MEMORY_OPTIONS_VERSION: c_uint = 1; + pub enum git_index {} pub enum git_index_conflict_iterator {} pub enum git_object {} @@ -98,7 +159,7 @@ pub struct git_odb_stream { pub read: Option c_int>, pub write: Option c_int>, pub finalize_write: Option c_int>, - pub free: Option + pub free: Option, } git_enum! { @@ -3161,6 +3222,13 @@ extern "C" { ) -> c_int; // config + pub fn git_config_add_backend( + cfg: *mut git_config, + file: *mut git_config_backend, + level: git_config_level_t, + repo: *const git_repository, + force: c_int, + ) -> c_int; pub fn git_config_add_file_ondisk( cfg: *mut git_config, path: *const c_char, @@ -3168,6 +3236,18 @@ extern "C" { repo: *const git_repository, force: c_int, ) -> c_int; + pub fn git_config_backend_backend_from_string( + out: *mut *mut git_config_backend, + cfg: *const c_char, + len: size_t, + opts: *mut git_config_backend_memory_options, + ) -> c_int; + pub fn git_config_backend_backend_from_values( + out: *mut *mut git_config_backend, + values: *mut *const c_char, + len: size_t, + opts: *mut git_config_backend_memory_options, + ) -> c_int; pub fn git_config_delete_entry(cfg: *mut git_config, name: *const c_char) -> c_int; pub fn git_config_delete_multivar( cfg: *mut git_config, @@ -3214,6 +3294,7 @@ extern "C" { cfg: *const git_config, name: *const c_char, ) -> c_int; + pub fn git_config_init_backend(backend: *mut git_config_backend, version: c_uint) -> c_int; pub fn git_config_iterator_free(iter: *mut git_config_iterator); pub fn git_config_iterator_glob_new( out: *mut *mut git_config_iterator, From 0d1e96fc1fc7f6470312fcca7c02889a80bd740a Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 01:56:05 +0200 Subject: [PATCH 03/28] feat: Add struct `git_reference_iterator` Added struct `git_reference_iterator`. See git2/sys/refdb_backend.h line 35. --- libgit2-sys/lib.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index abe74dbfb0..caca95b75a 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -112,7 +112,7 @@ pub const GIT_CONFIG_BACKEND_VERSION: c_uint = 1; pub struct git_config_backend_memory_options { pub version: c_uint, pub backend_type: *const c_char, - pub origin_path: *const c_char + pub origin_path: *const c_char, } pub const GIT_CONFIG_BACKEND_MEMORY_OPTIONS_VERSION: c_uint = 1; @@ -121,7 +121,15 @@ pub enum git_index {} pub enum git_index_conflict_iterator {} pub enum git_object {} pub enum git_reference {} -pub enum git_reference_iterator {} + +#[repr(C)] +pub struct git_reference_iterator { + pub db: *mut git_refdb, + pub next: Option c_int>, + pub next_name: Option c_int>, + pub free: Option, +} + pub enum git_annotated_commit {} pub enum git_refdb {} pub enum git_refspec {} From b7ce5e626da9234202f0d5e46f5ab3648d43a359 Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 02:02:56 +0200 Subject: [PATCH 04/28] fix: Rename `git_commit_nth_gen_ancestor`'s first argument `git_commit_nth_gen_ancestor`'s first argument has been changed to `ancestor` from the previous `commit` to bring it closer to the actual definition (git2/commit.h line 282) and because the previous name is also used for the second argument. --- libgit2-sys/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index caca95b75a..e13dfac289 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -2991,7 +2991,7 @@ extern "C" { pub fn git_commit_message_encoding(commit: *const git_commit) -> *const c_char; pub fn git_commit_message_raw(commit: *const git_commit) -> *const c_char; pub fn git_commit_nth_gen_ancestor( - commit: *mut *mut git_commit, + ancestor: *mut *mut git_commit, commit: *const git_commit, n: c_uint, ) -> c_int; From d877928672d489ebb8898bbb6e61aa4b33c04dc1 Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 02:32:10 +0200 Subject: [PATCH 05/28] fix: Add `git2/sys/config.h` to systest headers, use extern "C" fn --- libgit2-sys/lib.rs | 6 +++--- systest/build.rs | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index e13dfac289..bfdb85e333 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -125,9 +125,9 @@ pub enum git_reference {} #[repr(C)] pub struct git_reference_iterator { pub db: *mut git_refdb, - pub next: Option c_int>, - pub next_name: Option c_int>, - pub free: Option, + pub next: Option c_int>, + pub next_name: Option c_int>, + pub free: Option, } pub enum git_annotated_commit {} diff --git a/systest/build.rs b/systest/build.rs index 9503af7e10..eb22065d4f 100644 --- a/systest/build.rs +++ b/systest/build.rs @@ -16,6 +16,7 @@ fn main() { .header("git2/sys/repository.h") .header("git2/sys/cred.h") .header("git2/sys/email.h") + .header("git2/sys/config.h") .header("git2/cred_helpers.h") .type_name(|s, _, _| s.to_string()); cfg.field_name(|_, f| match f { From a5af4764b11e1052067b9530c391597fcb6f58ea Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 02:34:35 +0200 Subject: [PATCH 06/28] fix: Add missing argument fromm `git_config_backend`, fix function names --- libgit2-sys/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index bfdb85e333..145e25b53c 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -102,7 +102,7 @@ pub struct git_config_backend { pub snapshot: Option c_int>, pub lock: Option c_int>, - pub unlock: Option c_int>, + pub unlock: Option c_int>, pub free: Option, } @@ -3244,13 +3244,13 @@ extern "C" { repo: *const git_repository, force: c_int, ) -> c_int; - pub fn git_config_backend_backend_from_string( + pub fn git_config_backend_from_string( out: *mut *mut git_config_backend, cfg: *const c_char, len: size_t, opts: *mut git_config_backend_memory_options, ) -> c_int; - pub fn git_config_backend_backend_from_values( + pub fn git_config_backend_from_values( out: *mut *mut git_config_backend, values: *mut *const c_char, len: size_t, From 833ed2f94fc0fe2c90d3bc17376c2a2eeb20221d Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 02:40:26 +0200 Subject: [PATCH 07/28] fix: Add missing `iterator` function to `git_config_backend` --- libgit2-sys/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index 145e25b53c..c30051cf6c 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -99,6 +99,8 @@ pub struct git_config_backend { pub del: Option c_int>, pub del_multivar: Option c_int>, + pub iterator: + Option c_int>, pub snapshot: Option c_int>, pub lock: Option c_int>, From 36aea269eebcaae67e4ade903b53f014af51843b Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 02:41:29 +0200 Subject: [PATCH 08/28] fix: Make `git_odb_stream_t` unsigned --- libgit2-sys/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index c30051cf6c..deb6e1bbfc 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -173,7 +173,7 @@ pub struct git_odb_stream { } git_enum! { - pub enum git_odb_stream_t: c_int { + pub enum git_odb_stream_t: c_uint { GIT_STREAM_RDONLY = 2, GIT_STREAM_WRONLY = 4, GIT_STREAM_RW = 6, From 60d0dc425037ce2cb07c0ee50013168819384ec1 Mon Sep 17 00:00:00 2001 From: tecc Date: Mon, 18 Aug 2025 13:38:19 +0200 Subject: [PATCH 09/28] fix: Remove int type from `git_odb_stream_t` --- libgit2-sys/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libgit2-sys/lib.rs b/libgit2-sys/lib.rs index deb6e1bbfc..a1b640c89a 100644 --- a/libgit2-sys/lib.rs +++ b/libgit2-sys/lib.rs @@ -173,7 +173,7 @@ pub struct git_odb_stream { } git_enum! { - pub enum git_odb_stream_t: c_uint { + pub enum git_odb_stream_t { GIT_STREAM_RDONLY = 2, GIT_STREAM_WRONLY = 4, GIT_STREAM_RW = 6, From b72fff02f458b8159f5dc11ef42e390e2c93378e Mon Sep 17 00:00:00 2001 From: tecc Date: Fri, 15 Aug 2025 21:31:05 +0200 Subject: [PATCH 10/28] fix(odb): Fix memory leak if `git_odb_add_backend` fails `CustomOdbBackend` does not drop the inner `Backend` value, leaving it to libgit2 calling the backend's `free` function. This function cannot be called if the `git_odb_add_backend` call fails. Now, `Odb::add_custom_backend` first creates the inner `Backend` value and only when the value has successfully been passed to libgit2 do we stop managing the memory. --- src/lib.rs | 1 + src/odb.rs | 13 ++ src/odb_backend.rs | 290 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 304 insertions(+) create mode 100644 src/odb_backend.rs diff --git a/src/lib.rs b/src/lib.rs index 675680a8e5..1c3ede548e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -690,6 +690,7 @@ mod util; pub mod build; pub mod cert; +pub mod odb_backend; pub mod oid_array; pub mod opts; pub mod string_array; diff --git a/src/odb.rs b/src/odb.rs index 2019908c48..81393991e3 100644 --- a/src/odb.rs +++ b/src/odb.rs @@ -12,6 +12,7 @@ use crate::util::Binding; use crate::{ raw, Error, IndexerProgress, Mempack, Object, ObjectType, OdbLookupFlags, Oid, Progress, }; +use crate::odb_backend::{CustomOdbBackend, OdbBackend}; /// A structure to represent a git object database pub struct Odb<'repo> { @@ -269,6 +270,18 @@ impl<'repo> Odb<'repo> { Ok(Mempack::from_raw(mempack)) } } + + /// Adds a custom backend to this odb with the given priority. + /// Returns a handle to the backend. + /// + /// `backend` will be dropped when this Odb is dropped. + pub fn add_custom_backend<'odb, B: OdbBackend + 'odb>(&'odb self, backend: B, priority: i32) -> Result, Error> { + let mut inner = CustomOdbBackend::new_inner(backend); + unsafe { + try_call!(raw::git_odb_add_backend(self.raw, ptr::from_mut(inner.as_mut()).cast(), priority as c_int)); + Ok(CustomOdbBackend::new(inner)) + } + } } /// An object from the Object Database. diff --git a/src/odb_backend.rs b/src/odb_backend.rs new file mode 100644 index 0000000000..18ca79cf5a --- /dev/null +++ b/src/odb_backend.rs @@ -0,0 +1,290 @@ +//! Custom backends for [`Odb`]s. +use crate::util::Binding; +use crate::{raw, Error, ErrorClass, ErrorCode, ObjectType, Oid}; +use libc::{c_int, size_t}; +use std::ffi::c_void; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::{ptr, slice}; + +pub trait OdbBackend { + fn supported_operations(&self) -> SupportedOperations; + + fn read( + &mut self, + ctx: &OdbBackendContext, + oid: Oid, + out: &mut OdbBackendAllocation, + ) -> Result { + (ctx, oid, out); + unimplemented!("OdbBackend::read") + } + // TODO: fn read_prefix(&mut self, ctx: &OdbBackendContext); + fn read_header(&mut self, ctx: &OdbBackendContext, oid: Oid) -> Result { + (ctx, oid); + unimplemented!("OdbBackend::read") + } + // TODO: fn write() + // TODO: fn writestream() + // TODO: fn readstream() + fn exists(&mut self, ctx: &OdbBackendContext, oid: Oid) -> Result; + // TODO: fn exists_prefix() + // TODO: fn refresh() + // TODO: fn foreach() + // TODO: fn writepack() + // TODO: fn writemidx() + // TODO: fn freshen() +} + +bitflags::bitflags! { + pub struct SupportedOperations: u32 { + /// The backend supports the [`OdbBackend::read`] method. + const READ = 1; + /// The backend supports the [`OdbBackend::read_prefix`] method. + const READ_PREFIX = 1 << 1; + /// The backend supports the [`OdbBackend::read_header`] method. + const READ_HEADER = 1 << 2; + /// The backend supports the [`OdbBackend::write`] method. + const WRITE = 1 << 3; + /// The backend supports the [`OdbBackend::writestream`] method. + const WRITESTREAM = 1 << 4; + /// The backend supports the [`OdbBackend::readstream`] method. + const READSTREAM = 1 << 5; + /// The backend supports the [`OdbBackend::exists`] method. + const EXISTS = 1 << 6; + /// The backend supports the [`OdbBackend::exists_prefix`] method. + const EXISTS_PREFIX = 1 << 7; + /// The backend supports the [`OdbBackend::refresh`] method. + const REFRESH = 1 << 7; + /// The backend supports the [`OdbBackend::foreach`] method. + const FOREACH = 1 << 8; + /// The backend supports the [`OdbBackend::writepack`] method. + const WRITEPACK = 1 << 9; + /// The backend supports the [`OdbBackend::writemidx`] method. + const WRITEMIDX = 1 << 10; + /// The backend supports the [`OdbBackend::freshen`] method. + const FRESHEN = 1 << 11; + } +} + +pub struct OdbHeader { + pub size: usize, + pub object_type: ObjectType, +} + +pub struct OdbBackendAllocation { + backend_ptr: *mut raw::git_odb_backend, + raw: *mut c_void, + size: usize, +} +impl OdbBackendAllocation { + pub fn as_mut(&mut self) -> &mut [u8] { + unsafe { slice::from_raw_parts_mut(self.raw.cast(), self.size) } + } +} +impl Drop for OdbBackendAllocation { + fn drop(&mut self) { + unsafe { + raw::git_odb_backend_data_free(self.backend_ptr, self.raw); + } + } +} + +pub struct OdbBackendRead {} + +pub struct OdbBackendContext { + backend_ptr: *mut raw::git_odb_backend, +} +impl OdbBackendContext { + /// Creates an instance of `OdbBackendAllocation` that points to `null`. + /// Its size will be 0. + pub fn null_alloc(&self) -> OdbBackendAllocation { + OdbBackendAllocation { + backend_ptr: self.backend_ptr, + raw: ptr::null_mut(), + size: 0, + } + } + + /// Attempts to allocate a buffer of size `size`. + /// + /// # Return value + /// `Some(allocation)` if the allocation succeeded. + /// `None` otherwise. This usually indicates that there is not enough memory. + pub fn alloc(&self, size: usize) -> Option { + let data = unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as size_t) }; + if data.is_null() { + return None; + } + Some(OdbBackendAllocation { + backend_ptr: self.backend_ptr, + raw: data, + size, + }) + } + + /// Attempts to allocate a buffer of size `size`, returning an error when that fails. + /// Essentially the same as [`alloc`], but returns a [`Result`]. + /// + /// # Return value + /// `Ok(allocation)` if the allocation succeeded. + /// `Err(error)` otherwise. The error is always a `GenericError` of class `NoMemory`. + pub fn try_alloc(&self, size: usize) -> Result { + self.alloc(size).ok_or_else(|| { + Error::new( + ErrorCode::GenericError, + ErrorClass::NoMemory, + "out of memory", + ) + }) + } +} + +/// A handle to an [`OdbBackend`] that has been added to an [`Odb`](crate::Odb). +pub struct CustomOdbBackend<'a, B: OdbBackend> { + // NOTE: Any pointer in this field must be both non-null and properly aligned. + raw: NonNull>, + phantom: PhantomData &'a ()>, +} + +impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { + pub(crate) fn new_inner(backend: B) -> Box> { + let mut parent = raw::git_odb_backend { + version: raw::GIT_ODB_BACKEND_VERSION, + odb: ptr::null_mut(), + read: None, + read_prefix: None, + read_header: None, + write: None, + writestream: None, + readstream: None, + exists: None, + exists_prefix: None, + refresh: None, + foreach: None, + writepack: None, + writemidx: None, + freshen: None, + free: None, + }; + Self::set_operations(backend.supported_operations(), &mut parent); + + Box::new(Backend { + parent, + inner: backend, + }) + } + pub(crate) fn new(backend: Box>) -> Self { + // SAFETY: Box::into_raw guarantees that the pointer is properly aligned and non-null + let backend = Box::into_raw(backend); + let backend = unsafe { NonNull::new_unchecked(backend) }; + Self { + raw: backend, + phantom: PhantomData, + } + } + fn set_operations( + supported_operations: SupportedOperations, + backend: &mut raw::git_odb_backend, + ) { + macro_rules! op_if { + ($name:ident if $flag:ident) => { + backend.$name = supported_operations + .contains(SupportedOperations::$flag) + .then_some(Backend::::$name) + }; + } + op_if!(read if READ); + op_if!(read_header if READ_HEADER); + op_if!(exists if EXISTS); + + backend.free = Some(Backend::::free); + } + + pub(crate) fn as_git_odb_backend(&self) -> *mut raw::git_odb_backend { + self.raw.cast() + } +} + +#[repr(C)] +pub(crate) struct Backend { + parent: raw::git_odb_backend, + inner: B, +} +impl Backend { + extern "C" fn read( + data_ptr: *mut *mut c_void, + size_ptr: *mut size_t, + otype_ptr: *mut raw::git_object_t, + backend_ptr: *mut raw::git_odb_backend, + oid_ptr: *const raw::git_oid, + ) -> raw::git_error_code { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let data = unsafe { data_ptr.as_mut().unwrap() }; + let size = unsafe { size_ptr.as_mut().unwrap() }; + let object_type = unsafe { otype_ptr.as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + + let context = OdbBackendContext { backend_ptr }; + + let mut allocation = ManuallyDrop::new(context.null_alloc()); + + let output = match backend.inner.read(&context, oid, &mut allocation) { + Err(e) => { + ManuallyDrop::into_inner(allocation); + return e.raw_code(); + } + Ok(o) => o, + }; + + *size = allocation.size; + *data = allocation.raw; + *object_type = output.raw(); + + raw::GIT_OK + } + extern "C" fn read_header( + size_ptr: *mut size_t, + otype_ptr: *mut raw::git_object_t, + backend_ptr: *mut raw::git_odb_backend, + oid_ptr: *const raw::git_oid, + ) -> raw::git_error_code { + let size = unsafe { size_ptr.as_mut().unwrap() }; + let otype = unsafe { otype_ptr.as_mut().unwrap() }; + let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + + let context = OdbBackendContext { backend_ptr }; + + let header = match backend.inner.read_header(&context, oid) { + Err(e) => unsafe { return e.raw_set_git_error() }, + Ok(header) => header, + }; + *size = header.size; + *otype = header.object_type.raw(); + raw::GIT_OK + } + + extern "C" fn free(backend: *mut raw::git_odb_backend) { + let inner = unsafe { Box::from_raw(backend.cast::()) }; + drop(inner); + } + + extern "C" fn exists( + backend_ptr: *mut raw::git_odb_backend, + oid_ptr: *const raw::git_oid, + ) -> c_int { + let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + let context = OdbBackendContext { backend_ptr }; + let exists = match backend.inner.exists(&context, oid) { + Err(e) => return unsafe { e.raw_set_git_error() }, + Ok(x) => x, + }; + if exists { + 1 + } else { + 0 + } + } +} From 5244f6b70d4b9b4dc0a3dd62f24b72f20431ff41 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 11:21:35 +0200 Subject: [PATCH 11/28] feat: Complete rough structure of `odb_backend` module --- src/odb_backend.rs | 300 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 249 insertions(+), 51 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 18ca79cf5a..6235fa0a1d 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -1,33 +1,163 @@ //! Custom backends for [`Odb`]s. +//! +//! [`Odb`]: crate::Odb use crate::util::Binding; use crate::{raw, Error, ErrorClass, ErrorCode, ObjectType, Oid}; +use bitflags::bitflags; use libc::{c_int, size_t}; use std::ffi::c_void; use std::marker::PhantomData; use std::mem::ManuallyDrop; +use std::ptr::NonNull; use std::{ptr, slice}; +/// A custom implementation of an [`Odb`] backend. +/// +/// # Errors +/// +/// If the backend does not have enough memory, the error code should be +/// [`ErrorCode::GenericError`] and the class should be [`ErrorClass::NoMemory`]. +#[allow(unused_variables)] pub trait OdbBackend { + /// Returns the supported operations of this backend. + /// The return value is used to determine what functions to provide to libgit2. + /// + /// This method is only called once in [`Odb::add_custom_backend`] and once in every call to + /// [`CustomOdbBackend::refresh_operations`]; in general, it is called very rarely. + /// Very few implementations should change their available operations after being added to an + /// [`Odb`]. + /// + /// [`Odb`]: crate::Odb + /// [`Odb::add_custom_backend`]: crate::Odb::add_custom_backend fn supported_operations(&self) -> SupportedOperations; + /// Read an object. + /// + /// Corresponds to the `read` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::READ`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// If an implementation returns `Ok(())`, `object_type` and `data` SHOULD be + /// set to the object type and the contents of the object respectively. + /// + /// [`OdbBackendAllocation`]s SHOULD be created using `ctx` (see + /// [`OdbBackendContext::try_alloc`]). + /// + /// # Errors + /// + /// If an object does not exist, the error code should be [`ErrorCode::NotFound`]. + /// + /// See [`OdbBackend`] for more recommendations. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations fn read( &mut self, ctx: &OdbBackendContext, oid: Oid, - out: &mut OdbBackendAllocation, - ) -> Result { - (ctx, oid, out); + object_type: &mut ObjectType, + data: &mut OdbBackendAllocation, + ) -> Result<(), Error> { unimplemented!("OdbBackend::read") } - // TODO: fn read_prefix(&mut self, ctx: &OdbBackendContext); - fn read_header(&mut self, ctx: &OdbBackendContext, oid: Oid) -> Result { - (ctx, oid); - unimplemented!("OdbBackend::read") + + /// Find and read an object based on a prefix of its [`Oid`]. + /// + /// Corresponds to the `read_prefix` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::READ_PREFIX`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// Only the first `oid_prefix_len * 4` bits of `oid_prefix` are set. + /// The remaining `(GIT_OID_SHA1_HEXSIZE - oid_prefix_len) * 4` bits are set to 0. + /// + /// If an implementation returns `Ok(())`, `oid`, `data`, and `object_type` SHOULD be set to the + /// full object ID, the object type, and the contents of the object respectively. + /// + /// [`OdbBackendAllocation`]s SHOULD be created using `ctx` (see + /// [`OdbBackendContext::try_alloc`]). + /// + /// # Errors + /// + /// See [`OdbBackend::read`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn read_prefix( + &mut self, + ctx: &OdbBackendContext, + oid_prefix: Oid, + oid_prefix_length: usize, + oid: &mut Oid, + object_type: &mut ObjectType, + data: &mut OdbBackendAllocation, + ) -> Result<(), Error> { + unimplemented!("OdbBackend::read_prefix") } + + /// Read an object's length and object type but not its contents. + /// + /// Corresponds to the `read_header` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::READ_HEADER`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// If an implementation returns `Ok(())`, `length` and `object_type` SHOULD be set to the + /// length of the object's contents and the object type respectively. + /// + /// # Errors + /// + /// See [`OdbBackend::read`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn read_header( + &mut self, + ctx: &OdbBackendContext, + oid: Oid, + length: &mut usize, + object_type: &mut ObjectType, + ) -> Result<(), Error> { + unimplemented!("OdbBackend::read_header") + } + + /// Check if an object exists. + /// + /// Corresponds to the `exists` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::EXISTS`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// # Implementation notes + /// + /// An implementation SHOULD return `Ok(true)` if the object exists, and `Ok(false)` if the + /// object does not exist. + /// + /// # Errors + /// + /// Errors SHOULD NOT be indicated through returning `Ok(false)`, but SHOULD through the use + /// of [`Error`]. + /// + /// See [`OdbBackend`] for more recommendations. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn exists(&mut self, ctx: &OdbBackendContext, oid: Oid) -> Result { + unimplemented!("OdbBackend::exists") + } + // TODO: fn write() // TODO: fn writestream() // TODO: fn readstream() - fn exists(&mut self, ctx: &OdbBackendContext, oid: Oid) -> Result; // TODO: fn exists_prefix() // TODO: fn refresh() // TODO: fn foreach() @@ -36,8 +166,11 @@ pub trait OdbBackend { // TODO: fn freshen() } -bitflags::bitflags! { +bitflags! { + /// Supported operations for a backend. pub struct SupportedOperations: u32 { + // NOTE: The names are taken from the trait method names, but the order is taken from the + // fields of git_odb_backend. /// The backend supports the [`OdbBackend::read`] method. const READ = 1; /// The backend supports the [`OdbBackend::read_prefix`] method. @@ -67,41 +200,45 @@ bitflags::bitflags! { } } -pub struct OdbHeader { - pub size: usize, - pub object_type: ObjectType, -} - +/// An allocation that can be passed to libgit2. +/// +/// In addition to managing the pointer, this struct also keeps track of the size of an allocation. +/// +/// Internally, allocations are made using [`git_odb_backend_data_malloc`] and freed using +/// [`git_odb_backend_data_free`]. +/// +/// [`git_odb_backend_data_malloc`]: raw::git_odb_backend_data_alloc +/// [`git_odb_backend_data_free`]: raw::git_odb_backend_data_free pub struct OdbBackendAllocation { backend_ptr: *mut raw::git_odb_backend, - raw: *mut c_void, + raw: NonNull, size: usize, } impl OdbBackendAllocation { + /// Returns this allocation as a byte slice. pub fn as_mut(&mut self) -> &mut [u8] { - unsafe { slice::from_raw_parts_mut(self.raw.cast(), self.size) } + unsafe { slice::from_raw_parts_mut(self.raw.cast().as_ptr(), self.size) } } } impl Drop for OdbBackendAllocation { fn drop(&mut self) { unsafe { - raw::git_odb_backend_data_free(self.backend_ptr, self.raw); + raw::git_odb_backend_data_free(self.backend_ptr, self.raw.as_ptr()); } } } -pub struct OdbBackendRead {} - +/// Information passed to most of [`OdbBackend`]'s methods. pub struct OdbBackendContext { backend_ptr: *mut raw::git_odb_backend, } impl OdbBackendContext { - /// Creates an instance of `OdbBackendAllocation` that points to `null`. - /// Its size will be 0. - pub fn null_alloc(&self) -> OdbBackendAllocation { + /// Creates an instance of `OdbBackendAllocation` that is zero-sized. + /// This is useful for representing non-allocations. + pub const fn alloc_0(&self) -> OdbBackendAllocation { OdbBackendAllocation { backend_ptr: self.backend_ptr, - raw: ptr::null_mut(), + raw: NonNull::dangling(), size: 0, } } @@ -113,9 +250,7 @@ impl OdbBackendContext { /// `None` otherwise. This usually indicates that there is not enough memory. pub fn alloc(&self, size: usize) -> Option { let data = unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as size_t) }; - if data.is_null() { - return None; - } + let data = NonNull::new(data)?; Some(OdbBackendAllocation { backend_ptr: self.backend_ptr, raw: data, @@ -124,9 +259,10 @@ impl OdbBackendContext { } /// Attempts to allocate a buffer of size `size`, returning an error when that fails. - /// Essentially the same as [`alloc`], but returns a [`Result`]. + /// Essentially the same as [`OdbBackendContext::alloc`], but returns a [`Result`] instead. /// /// # Return value + /// /// `Ok(allocation)` if the allocation succeeded. /// `Err(error)` otherwise. The error is always a `GenericError` of class `NoMemory`. pub fn try_alloc(&self, size: usize) -> Result { @@ -183,6 +319,23 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { phantom: PhantomData, } } + + /// Refreshes the available operations that libgit2 can see. + pub fn refresh_operations(&mut self) { + Self::set_operations(self.as_inner().supported_operations(), unsafe { + &mut self.raw.as_mut().parent + }); + } + + /// Returns a reference to the inner implementation of `OdbBackend`. + pub fn as_inner(&self) -> &'a B { + unsafe { &self.raw.as_ref().inner } + } + /// Returns a mutable reference to the inner implementation of `OdbBackend`. + pub fn as_inner_mut(&mut self) -> &'a mut B { + unsafe { &mut self.raw.as_mut().inner } + } + fn set_operations( supported_operations: SupportedOperations, backend: &mut raw::git_odb_backend, @@ -195,15 +348,12 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { }; } op_if!(read if READ); + op_if!(read_prefix if READ_PREFIX); op_if!(read_header if READ_HEADER); op_if!(exists if EXISTS); backend.free = Some(Backend::::free); } - - pub(crate) fn as_git_odb_backend(&self) -> *mut raw::git_odb_backend { - self.raw.cast() - } } #[repr(C)] @@ -227,19 +377,64 @@ impl Backend { let context = OdbBackendContext { backend_ptr }; - let mut allocation = ManuallyDrop::new(context.null_alloc()); + let mut allocation = ManuallyDrop::new(context.alloc_0()); - let output = match backend.inner.read(&context, oid, &mut allocation) { - Err(e) => { - ManuallyDrop::into_inner(allocation); - return e.raw_code(); - } - Ok(o) => o, - }; + let mut object_type2 = ObjectType::Any; + + if let Err(e) = backend + .inner + .read(&context, oid, &mut object_type2, &mut allocation) + { + ManuallyDrop::into_inner(allocation); + return e.raw_code(); + } + + *size = allocation.size; + *data = allocation.raw.as_ptr(); + *object_type = object_type2.raw(); + + raw::GIT_OK + } + extern "C" fn read_prefix( + oid_ptr: *mut raw::git_oid, + data_ptr: *mut *mut c_void, + size_ptr: *mut size_t, + otype_ptr: *mut raw::git_object_t, + backend_ptr: *mut raw::git_odb_backend, + oid_prefix_ptr: *const raw::git_oid, + oid_prefix_len: size_t, + ) -> raw::git_error_code { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let data = unsafe { data_ptr.as_mut().unwrap() }; + let size = unsafe { size_ptr.as_mut().unwrap() }; + let object_type = unsafe { otype_ptr.as_mut().unwrap() }; + let oid_prefix = unsafe { Oid::from_raw(oid_prefix_ptr) }; + // This is a small hack because Oid doesn't expose the raw data which we need + let oid = unsafe { oid_ptr.cast::().as_mut().unwrap() }; + + let context = OdbBackendContext { backend_ptr }; + + let mut allocation = ManuallyDrop::new(context.alloc_0()); + + let mut object_type2 = ObjectType::Any; + let mut oid2 = Oid::zero(); + + if let Err(e) = backend.inner.read_prefix( + &context, + oid_prefix, + oid_prefix_len as usize, + &mut oid2, + &mut object_type2, + &mut allocation, + ) { + ManuallyDrop::into_inner(allocation); + return e.raw_code(); + } + *oid = oid2; *size = allocation.size; - *data = allocation.raw; - *object_type = output.raw(); + *data = allocation.raw.as_ptr(); + *object_type = object_type2.raw(); raw::GIT_OK } @@ -256,18 +451,16 @@ impl Backend { let context = OdbBackendContext { backend_ptr }; - let header = match backend.inner.read_header(&context, oid) { - Err(e) => unsafe { return e.raw_set_git_error() }, - Ok(header) => header, + let mut object_type = ObjectType::Any; + if let Err(e) = backend + .inner + .read_header(&context, oid, size, &mut object_type) + { + unsafe { return e.raw_set_git_error() } }; - *size = header.size; - *otype = header.object_type.raw(); - raw::GIT_OK - } + *otype = object_type.raw(); - extern "C" fn free(backend: *mut raw::git_odb_backend) { - let inner = unsafe { Box::from_raw(backend.cast::()) }; - drop(inner); + raw::GIT_OK } extern "C" fn exists( @@ -287,4 +480,9 @@ impl Backend { 0 } } + + extern "C" fn free(backend: *mut raw::git_odb_backend) { + let inner = unsafe { Box::from_raw(backend.cast::()) }; + drop(inner); + } } From 7e8e15a2417d809d8763c3236ac71e5c50d4d48e Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 11:55:10 +0200 Subject: [PATCH 12/28] feat: Add `OdbBackend::exists_prefix` --- src/odb_backend.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 6235fa0a1d..da666cc5d1 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -78,7 +78,7 @@ pub trait OdbBackend { /// Only the first `oid_prefix_len * 4` bits of `oid_prefix` are set. /// The remaining `(GIT_OID_SHA1_HEXSIZE - oid_prefix_len) * 4` bits are set to 0. /// - /// If an implementation returns `Ok(())`, `oid`, `data`, and `object_type` SHOULD be set to the + /// If an implementation returns `Ok(())`, `oid`, `data`, and `object_type` MUST be set to the /// full object ID, the object type, and the contents of the object respectively. /// /// [`OdbBackendAllocation`]s SHOULD be created using `ctx` (see @@ -112,7 +112,7 @@ pub trait OdbBackend { /// /// # Implementation notes /// - /// If an implementation returns `Ok(())`, `length` and `object_type` SHOULD be set to the + /// If an implementation returns `Ok(())`, `length` and `object_type` MUST be set to the /// length of the object's contents and the object type respectively. /// /// # Errors @@ -155,6 +155,36 @@ pub trait OdbBackend { unimplemented!("OdbBackend::exists") } + /// Check if an object exists based on a prefix of its [`Oid`]. + /// + /// Corresponds to the `exists_prefix` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::EXISTS_PREFIX`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// Only the first `oid_prefix_len * 4` bits of `oid_prefix` are set. + /// The remaining `(GIT_OID_SHA1_HEXSIZE - oid_prefix_len) * 4` bits are set to 0. + /// + /// If an implementation returns `Ok(oid)`, `oid` SHOULD be the full Oid of the object. + /// + /// # Errors + /// + /// See [`OdbBackend::exists`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn exists_prefix( + &mut self, + ctx: &OdbBackendContext, + oid_prefix: Oid, + oid_prefix_length: usize, + ) -> Result { + unimplemented!("OdbBackend::exists_prefix") + } + // TODO: fn write() // TODO: fn writestream() // TODO: fn readstream() @@ -351,6 +381,7 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(read_prefix if READ_PREFIX); op_if!(read_header if READ_HEADER); op_if!(exists if EXISTS); + op_if!(exists_prefix if EXISTS_PREFIX); backend.free = Some(Backend::::free); } @@ -480,6 +511,24 @@ impl Backend { 0 } } + + extern "C" fn exists_prefix( + oid_ptr: *mut raw::git_oid, + backend_ptr: *mut raw::git_odb_backend, + oid_prefix_ptr: *const raw::git_oid, + oid_prefix_len: size_t + ) -> c_int { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let oid_prefix = unsafe { Oid::from_raw(oid_prefix_ptr) }; + let oid = unsafe { oid_ptr.cast::().as_mut().unwrap() }; + + let context = OdbBackendContext { backend_ptr }; + *oid = match backend.inner.exists_prefix(&context, oid_prefix, oid_prefix_len) { + Err(e) => return unsafe { e.raw_set_git_error() }, + Ok(x) => x, + }; + raw::GIT_OK + } extern "C" fn free(backend: *mut raw::git_odb_backend) { let inner = unsafe { Box::from_raw(backend.cast::()) }; From 8526f9ba415de933d2ead8af50572ad3c416f9c3 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 11:56:10 +0200 Subject: [PATCH 13/28] change: Replace `git_error_code` return types with `c_int` Some methods don't necessarily return just `git_error_code`. --- src/odb_backend.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index da666cc5d1..151f914185 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -399,7 +399,7 @@ impl Backend { otype_ptr: *mut raw::git_object_t, backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, - ) -> raw::git_error_code { + ) -> c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let data = unsafe { data_ptr.as_mut().unwrap() }; let size = unsafe { size_ptr.as_mut().unwrap() }; @@ -434,7 +434,7 @@ impl Backend { backend_ptr: *mut raw::git_odb_backend, oid_prefix_ptr: *const raw::git_oid, oid_prefix_len: size_t, - ) -> raw::git_error_code { + ) -> c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let data = unsafe { data_ptr.as_mut().unwrap() }; let size = unsafe { size_ptr.as_mut().unwrap() }; @@ -474,7 +474,7 @@ impl Backend { otype_ptr: *mut raw::git_object_t, backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, - ) -> raw::git_error_code { + ) -> c_int { let size = unsafe { size_ptr.as_mut().unwrap() }; let otype = unsafe { otype_ptr.as_mut().unwrap() }; let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; From 04dfe76e4029ccc9b584f90ce01591708952c0d6 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 11:59:05 +0200 Subject: [PATCH 14/28] change: Direct usages of `libc::{size_t, c_int}` and `std::ffi::c_void` --- src/odb_backend.rs | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 151f914185..2b8f15c85f 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -4,8 +4,6 @@ use crate::util::Binding; use crate::{raw, Error, ErrorClass, ErrorCode, ObjectType, Oid}; use bitflags::bitflags; -use libc::{c_int, size_t}; -use std::ffi::c_void; use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ptr::NonNull; @@ -241,7 +239,7 @@ bitflags! { /// [`git_odb_backend_data_free`]: raw::git_odb_backend_data_free pub struct OdbBackendAllocation { backend_ptr: *mut raw::git_odb_backend, - raw: NonNull, + raw: NonNull, size: usize, } impl OdbBackendAllocation { @@ -279,7 +277,7 @@ impl OdbBackendContext { /// `Some(allocation)` if the allocation succeeded. /// `None` otherwise. This usually indicates that there is not enough memory. pub fn alloc(&self, size: usize) -> Option { - let data = unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as size_t) }; + let data = unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as libc::size_t) }; let data = NonNull::new(data)?; Some(OdbBackendAllocation { backend_ptr: self.backend_ptr, @@ -394,12 +392,12 @@ pub(crate) struct Backend { } impl Backend { extern "C" fn read( - data_ptr: *mut *mut c_void, - size_ptr: *mut size_t, + data_ptr: *mut *mut libc::c_void, + size_ptr: *mut libc::size_t, otype_ptr: *mut raw::git_object_t, backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, - ) -> c_int { + ) -> libc::c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let data = unsafe { data_ptr.as_mut().unwrap() }; let size = unsafe { size_ptr.as_mut().unwrap() }; @@ -428,13 +426,13 @@ impl Backend { } extern "C" fn read_prefix( oid_ptr: *mut raw::git_oid, - data_ptr: *mut *mut c_void, - size_ptr: *mut size_t, + data_ptr: *mut *mut libc::c_void, + size_ptr: *mut libc::size_t, otype_ptr: *mut raw::git_object_t, backend_ptr: *mut raw::git_odb_backend, oid_prefix_ptr: *const raw::git_oid, - oid_prefix_len: size_t, - ) -> c_int { + oid_prefix_len: libc::size_t, + ) -> libc::c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let data = unsafe { data_ptr.as_mut().unwrap() }; let size = unsafe { size_ptr.as_mut().unwrap() }; @@ -470,11 +468,11 @@ impl Backend { raw::GIT_OK } extern "C" fn read_header( - size_ptr: *mut size_t, + size_ptr: *mut libc::size_t, otype_ptr: *mut raw::git_object_t, backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, - ) -> c_int { + ) -> libc::c_int { let size = unsafe { size_ptr.as_mut().unwrap() }; let otype = unsafe { otype_ptr.as_mut().unwrap() }; let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; @@ -497,7 +495,7 @@ impl Backend { extern "C" fn exists( backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, - ) -> c_int { + ) -> libc::c_int { let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; let oid = unsafe { Oid::from_raw(oid_ptr) }; let context = OdbBackendContext { backend_ptr }; @@ -511,13 +509,13 @@ impl Backend { 0 } } - + extern "C" fn exists_prefix( oid_ptr: *mut raw::git_oid, backend_ptr: *mut raw::git_odb_backend, oid_prefix_ptr: *const raw::git_oid, - oid_prefix_len: size_t - ) -> c_int { + oid_prefix_len: libc::size_t + ) -> libc::c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let oid_prefix = unsafe { Oid::from_raw(oid_prefix_ptr) }; let oid = unsafe { oid_ptr.cast::().as_mut().unwrap() }; From d094829c56654740c0b3f30c759ff25117fa9dfe Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 12:34:17 +0200 Subject: [PATCH 15/28] chore: Run `cargo fmt` --- src/odb.rs | 14 +++++++++++--- src/odb_backend.rs | 10 +++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/odb.rs b/src/odb.rs index 81393991e3..2b367e517a 100644 --- a/src/odb.rs +++ b/src/odb.rs @@ -7,12 +7,12 @@ use std::ffi::CString; use libc::{c_char, c_int, c_uint, c_void, size_t}; +use crate::odb_backend::{CustomOdbBackend, OdbBackend}; use crate::panic; use crate::util::Binding; use crate::{ raw, Error, IndexerProgress, Mempack, Object, ObjectType, OdbLookupFlags, Oid, Progress, }; -use crate::odb_backend::{CustomOdbBackend, OdbBackend}; /// A structure to represent a git object database pub struct Odb<'repo> { @@ -275,10 +275,18 @@ impl<'repo> Odb<'repo> { /// Returns a handle to the backend. /// /// `backend` will be dropped when this Odb is dropped. - pub fn add_custom_backend<'odb, B: OdbBackend + 'odb>(&'odb self, backend: B, priority: i32) -> Result, Error> { + pub fn add_custom_backend<'odb, B: OdbBackend + 'odb>( + &'odb self, + backend: B, + priority: i32, + ) -> Result, Error> { let mut inner = CustomOdbBackend::new_inner(backend); unsafe { - try_call!(raw::git_odb_add_backend(self.raw, ptr::from_mut(inner.as_mut()).cast(), priority as c_int)); + try_call!(raw::git_odb_add_backend( + self.raw, + ptr::from_mut(inner.as_mut()).cast(), + priority as c_int + )); Ok(CustomOdbBackend::new(inner)) } } diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 2b8f15c85f..2faffb66cf 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -277,7 +277,8 @@ impl OdbBackendContext { /// `Some(allocation)` if the allocation succeeded. /// `None` otherwise. This usually indicates that there is not enough memory. pub fn alloc(&self, size: usize) -> Option { - let data = unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as libc::size_t) }; + let data = + unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as libc::size_t) }; let data = NonNull::new(data)?; Some(OdbBackendAllocation { backend_ptr: self.backend_ptr, @@ -514,14 +515,17 @@ impl Backend { oid_ptr: *mut raw::git_oid, backend_ptr: *mut raw::git_odb_backend, oid_prefix_ptr: *const raw::git_oid, - oid_prefix_len: libc::size_t + oid_prefix_len: libc::size_t, ) -> libc::c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let oid_prefix = unsafe { Oid::from_raw(oid_prefix_ptr) }; let oid = unsafe { oid_ptr.cast::().as_mut().unwrap() }; let context = OdbBackendContext { backend_ptr }; - *oid = match backend.inner.exists_prefix(&context, oid_prefix, oid_prefix_len) { + *oid = match backend + .inner + .exists_prefix(&context, oid_prefix, oid_prefix_len) + { Err(e) => return unsafe { e.raw_set_git_error() }, Ok(x) => x, }; From d6ac58a4ceaef3cfa23231039b7bb627b3e60d31 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 12:49:30 +0200 Subject: [PATCH 16/28] feat: Add `OdbBackend::write` --- src/odb_backend.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 2faffb66cf..5567a53111 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -129,6 +129,34 @@ pub trait OdbBackend { unimplemented!("OdbBackend::read_header") } + /// Write an object. + /// + /// Corresponds to the `write` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::WRITE`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// `oid` is calculated by libgit2 prior to this method being called. + /// + /// # Errors + /// + /// See [`OdbBackend`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn write( + &mut self, + ctx: &OdbBackendContext, + oid: Oid, + object_type: ObjectType, + data: &[u8], + ) -> Result<(), Error> { + unimplemented!("OdbBackend::write") + } + /// Check if an object exists. /// /// Corresponds to the `exists` function of [`git_odb_backend`]. @@ -183,7 +211,6 @@ pub trait OdbBackend { unimplemented!("OdbBackend::exists_prefix") } - // TODO: fn write() // TODO: fn writestream() // TODO: fn readstream() // TODO: fn exists_prefix() @@ -379,6 +406,7 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(read if READ); op_if!(read_prefix if READ_PREFIX); op_if!(read_header if READ_HEADER); + op_if!(write if WRITE); op_if!(exists if EXISTS); op_if!(exists_prefix if EXISTS_PREFIX); @@ -493,6 +521,24 @@ impl Backend { raw::GIT_OK } + extern "C" fn write( + backend_ptr: *mut raw::git_odb_backend, + oid_ptr: *const raw::git_oid, + data_ptr: *const libc::c_void, + len: usize, + otype: raw::git_object_t, + ) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + let data = unsafe { slice::from_raw_parts(data_ptr.cast::(), len) }; + let object_type = ObjectType::from_raw(otype).unwrap(); + let context = OdbBackendContext { backend_ptr }; + if let Err(e) = backend.inner.write(&context, oid, object_type, data) { + return unsafe { e.raw_set_git_error() }; + } + raw::GIT_OK + } + extern "C" fn exists( backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, From c69030ccdf96ac179474941dd41f2132b364f52d Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 13:21:22 +0200 Subject: [PATCH 17/28] feat: Add `OdbBackend::refresh` and `OdbBackend::freshen` --- src/odb_backend.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 5567a53111..86b74d4b13 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -211,14 +211,63 @@ pub trait OdbBackend { unimplemented!("OdbBackend::exists_prefix") } + /// Refreshes the backend. + /// + /// Corresponds to the `refresh` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::REFRESH`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method returns `Ok(())`. + /// + /// # Implementation notes + /// + /// This method is called automatically when a lookup fails (e.g. through + /// [`OdbBackend::exists`], [`OdbBackend::read`], or [`OdbBackend::read_header`]), + /// or when [`Odb::refresh`](crate::Odb::refresh) is invoked. + /// + /// # Errors + /// + /// See [`OdbBackend`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn refresh(&mut self, ctx: &OdbBackendContext) -> Result<(), Error> { + Ok(()) + } + + /// "Freshens" an already existing object, updating its last-used time. + /// + /// Corresponds to the `freshen` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::REFRESH`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// This method is called when [`Odb::write`](crate::Odb::write) is called, but the object + /// already exists and will not be rewritten. + /// + /// Implementations may want to update last-used timestamps. + /// + /// Implementations SHOULD return `Ok(())` if the object exists and was freshened; otherwise, + /// they SHOULD return an error. + /// + /// # Errors + /// + /// See [`OdbBackend`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn freshen(&mut self, ctx: &OdbBackendContext, oid: Oid) -> Result<(), Error> { + unimplemented!("OdbBackend::freshen") + } + // TODO: fn writestream() // TODO: fn readstream() - // TODO: fn exists_prefix() - // TODO: fn refresh() // TODO: fn foreach() // TODO: fn writepack() // TODO: fn writemidx() - // TODO: fn freshen() } bitflags! { @@ -409,6 +458,8 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(write if WRITE); op_if!(exists if EXISTS); op_if!(exists_prefix if EXISTS_PREFIX); + op_if!(refresh if REFRESH); + op_if!(freshen if FRESHEN); backend.free = Some(Backend::::free); } @@ -578,6 +629,29 @@ impl Backend { raw::GIT_OK } + extern "C" fn refresh(backend_ptr: *mut raw::git_odb_backend) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let context = OdbBackendContext { backend_ptr }; + if let Err(e) = backend.inner.refresh(&context) { + return unsafe { e.raw_set_git_error() }; + } + raw::GIT_OK + } + + extern "C" fn freshen( + backend_ptr: *mut raw::git_odb_backend, + oid_ptr: *const raw::git_oid, + ) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + let context = OdbBackendContext { backend_ptr }; + if let Err(e) = backend.inner.freshen(&context, oid) { + return unsafe { e.raw_set_git_error() }; + } + + raw::GIT_OK + } + extern "C" fn free(backend: *mut raw::git_odb_backend) { let inner = unsafe { Box::from_raw(backend.cast::()) }; drop(inner); From 28e10f79037e3fa63dd52673d3abab47b2f5b1dc Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 13:24:26 +0200 Subject: [PATCH 18/28] docs: Add more documentation to `OdbBackend` Added more documentation to clarify how to do error handling properly. Clarified the default behaviour of `OdbBackend::exists`. --- src/odb_backend.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 86b74d4b13..a20450299c 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -11,10 +11,18 @@ use std::{ptr, slice}; /// A custom implementation of an [`Odb`] backend. /// -/// # Errors +/// Most of the default implementations of this trait's methods panic when called as they are +/// intended to be overridden. /// -/// If the backend does not have enough memory, the error code should be -/// [`ErrorCode::GenericError`] and the class should be [`ErrorClass::NoMemory`]. +/// # Error recommendations +/// +/// Errors are generally left at the implementation's discretion; some recommendations are +/// made regarding error codes and classes to ease implementation and usage of custom backends. +/// +/// Read the individual methods' documentation for more specific recommendations. +/// +/// If the backend does not have enough memory, the error SHOULD be code +/// [`ErrorCode::GenericError`] and the class SHOULD be [`ErrorClass::NoMemory`]. #[allow(unused_variables)] pub trait OdbBackend { /// Returns the supported operations of this backend. @@ -163,6 +171,8 @@ pub trait OdbBackend { /// Requires that [`SupportedOperations::EXISTS`] is present in the value returned from /// [`supported_operations`] to expose it to libgit2. /// + /// The default implementation of this method panics. + /// /// # Implementation notes /// /// An implementation SHOULD return `Ok(true)` if the object exists, and `Ok(false)` if the From 0f4c37a6c3ecb597202813e7b244007f3e92cf93 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 13:30:03 +0200 Subject: [PATCH 19/28] docs: Fix `OdbBackend::freshen` operation flag --- src/odb_backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index a20450299c..4ab7ac61f7 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -248,7 +248,7 @@ pub trait OdbBackend { /// "Freshens" an already existing object, updating its last-used time. /// /// Corresponds to the `freshen` function of [`git_odb_backend`]. - /// Requires that [`SupportedOperations::REFRESH`] is present in the value returned from + /// Requires that [`SupportedOperations::FRESHEN`] is present in the value returned from /// [`supported_operations`] to expose it to libgit2. /// /// The default implementation of this method panics. From 314813a0684a6306924410a56065b8745d418995 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 13:38:08 +0200 Subject: [PATCH 20/28] feat: Add `OdbBackend::write_multipack_index` (`writemidx`) Added `OdbBackend::write_multipack_index`, corresponding to the `writemidx` function of `git_odb_backend`. Fix reference to `Odb` in `OdbBackend` documentation. --- src/odb_backend.rs | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 4ab7ac61f7..ca266defb6 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -9,7 +9,7 @@ use std::mem::ManuallyDrop; use std::ptr::NonNull; use std::{ptr, slice}; -/// A custom implementation of an [`Odb`] backend. +/// A custom implementation of an [`Odb`](crate::Odb) backend. /// /// Most of the default implementations of this trait's methods panic when called as they are /// intended to be overridden. @@ -273,11 +273,33 @@ pub trait OdbBackend { unimplemented!("OdbBackend::freshen") } + /// Creates a `multi-pack-index` file containing an index of all objects across all `.pack` + /// files. + /// + /// Corresponds to the `writemidx` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::WRITE_MULTIPACK_INDEX`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// TODO: Implementation notes for `write_multipack_index` + /// + /// # Errors + /// + /// See [`OdbBackend`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn write_multipack_index(&mut self, ctx: &OdbBackendContext) -> Result<(), Error> { + unimplemented!("OdbBackend::write_multipack_index") + } + // TODO: fn writestream() // TODO: fn readstream() // TODO: fn foreach() // TODO: fn writepack() - // TODO: fn writemidx() } bitflags! { @@ -305,10 +327,10 @@ bitflags! { const REFRESH = 1 << 7; /// The backend supports the [`OdbBackend::foreach`] method. const FOREACH = 1 << 8; - /// The backend supports the [`OdbBackend::writepack`] method. - const WRITEPACK = 1 << 9; - /// The backend supports the [`OdbBackend::writemidx`] method. - const WRITEMIDX = 1 << 10; + /// The backend supports the [`OdbBackend::write_pack`] method. + const WRITE_PACK = 1 << 9; + /// The backend supports the [`OdbBackend::write_multipack_index`] method. + const WRITE_MULTIPACK_INDEX = 1 << 10; /// The backend supports the [`OdbBackend::freshen`] method. const FRESHEN = 1 << 11; } @@ -469,6 +491,7 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(exists if EXISTS); op_if!(exists_prefix if EXISTS_PREFIX); op_if!(refresh if REFRESH); + op_if!(writemidx if WRITE_MULTIPACK_INDEX); op_if!(freshen if FRESHEN); backend.free = Some(Backend::::free); @@ -648,6 +671,15 @@ impl Backend { raw::GIT_OK } + extern "C" fn writemidx(backend_ptr: *mut raw::git_odb_backend) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let context = OdbBackendContext { backend_ptr }; + if let Err(e) = backend.inner.write_multipack_index(&context) { + return unsafe { e.raw_set_git_error() }; + } + raw::GIT_OK + } + extern "C" fn freshen( backend_ptr: *mut raw::git_odb_backend, oid_ptr: *const raw::git_oid, From 79c0267f9a88f11f51041c4172adda1960af91b0 Mon Sep 17 00:00:00 2001 From: tecc Date: Sat, 16 Aug 2025 19:32:52 +0200 Subject: [PATCH 21/28] refactor: Use `ptr::NonNull` and `marker::PhantomData` This is mostly to bring it inline with the other modules' code style. --- src/odb_backend.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index ca266defb6..37bd28bd20 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -6,8 +6,7 @@ use crate::{raw, Error, ErrorClass, ErrorCode, ObjectType, Oid}; use bitflags::bitflags; use std::marker::PhantomData; use std::mem::ManuallyDrop; -use std::ptr::NonNull; -use std::{ptr, slice}; +use std::{marker, ptr, slice}; /// A custom implementation of an [`Odb`](crate::Odb) backend. /// @@ -347,7 +346,7 @@ bitflags! { /// [`git_odb_backend_data_free`]: raw::git_odb_backend_data_free pub struct OdbBackendAllocation { backend_ptr: *mut raw::git_odb_backend, - raw: NonNull, + raw: ptr::NonNull, size: usize, } impl OdbBackendAllocation { @@ -374,7 +373,7 @@ impl OdbBackendContext { pub const fn alloc_0(&self) -> OdbBackendAllocation { OdbBackendAllocation { backend_ptr: self.backend_ptr, - raw: NonNull::dangling(), + raw: ptr::NonNull::dangling(), size: 0, } } @@ -387,7 +386,7 @@ impl OdbBackendContext { pub fn alloc(&self, size: usize) -> Option { let data = unsafe { raw::git_odb_backend_data_alloc(self.backend_ptr, size as libc::size_t) }; - let data = NonNull::new(data)?; + let data = ptr::NonNull::new(data)?; Some(OdbBackendAllocation { backend_ptr: self.backend_ptr, raw: data, @@ -416,8 +415,8 @@ impl OdbBackendContext { /// A handle to an [`OdbBackend`] that has been added to an [`Odb`](crate::Odb). pub struct CustomOdbBackend<'a, B: OdbBackend> { // NOTE: Any pointer in this field must be both non-null and properly aligned. - raw: NonNull>, - phantom: PhantomData &'a ()>, + raw: ptr::NonNull>, + phantom: marker::PhantomData &'a ()>, } impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { @@ -450,10 +449,10 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { pub(crate) fn new(backend: Box>) -> Self { // SAFETY: Box::into_raw guarantees that the pointer is properly aligned and non-null let backend = Box::into_raw(backend); - let backend = unsafe { NonNull::new_unchecked(backend) }; + let backend = unsafe { ptr::NonNull::new_unchecked(backend) }; Self { raw: backend, - phantom: PhantomData, + phantom: marker::PhantomData, } } @@ -598,7 +597,7 @@ impl Backend { .inner .read_header(&context, oid, size, &mut object_type) { - unsafe { return e.raw_set_git_error() } + return unsafe { e.raw_set_git_error() }; }; *otype = object_type.raw(); From a95868b7b191514138d52d3871f8a5a6ba0ac2e9 Mon Sep 17 00:00:00 2001 From: tecc Date: Sun, 17 Aug 2025 10:54:06 +0200 Subject: [PATCH 22/28] feat: Add `OdbBackend::open_writepack` interface Added `OdbBackend::open_writepack`, for opening streams that write packfiles. Added `OdbWritepack` trait for custom Writepack implementations. Added an associated type `OdbBackend::Writepack: OdbWritepack` that tells git2 what data to include for writepack implementations. Implemented `OdbWritepack` for `Infallible` so that people can opt out of implementing the Writepack trait. The implementation panics when any method is called, though that would be undefined behaviour since it would require `Infallible` to have been instantiated. A lot of types were duplicated in the process of this commit. I want other people's thoughts on how to merge them before I make any changes to their code, so I'll open a draft PR soon. --- src/odb_backend.rs | 399 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 387 insertions(+), 12 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 37bd28bd20..7f5b2f639b 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -1,14 +1,16 @@ //! Custom backends for [`Odb`]s. //! -//! [`Odb`]: crate::Odb +//! TODO: Merge a lot of these APIs with existing ones. +//! Currently use crate::util::Binding; -use crate::{raw, Error, ErrorClass, ErrorCode, ObjectType, Oid}; +use crate::{raw, Error, ErrorClass, ErrorCode, IntoCString, ObjectType, Odb, Oid}; use bitflags::bitflags; -use std::marker::PhantomData; +use std::convert::Infallible; use std::mem::ManuallyDrop; -use std::{marker, ptr, slice}; +use std::path::Path; +use std::{marker, mem, ptr, slice}; -/// A custom implementation of an [`Odb`](crate::Odb) backend. +/// A custom implementation of an [`Odb`] backend. /// /// Most of the default implementations of this trait's methods panic when called as they are /// intended to be overridden. @@ -23,7 +25,14 @@ use std::{marker, ptr, slice}; /// If the backend does not have enough memory, the error SHOULD be code /// [`ErrorCode::GenericError`] and the class SHOULD be [`ErrorClass::NoMemory`]. #[allow(unused_variables)] -pub trait OdbBackend { +pub trait OdbBackend: Sized { + /// Backend-specific writepack implementation. + /// + /// If the backend doesn't support writepack, this type should be [`Infallible`]. + /// + /// [`Infallible`]: std::convert::Infallible + type Writepack: OdbWritepack; + /// Returns the supported operations of this backend. /// The return value is used to determine what functions to provide to libgit2. /// @@ -31,9 +40,6 @@ pub trait OdbBackend { /// [`CustomOdbBackend::refresh_operations`]; in general, it is called very rarely. /// Very few implementations should change their available operations after being added to an /// [`Odb`]. - /// - /// [`Odb`]: crate::Odb - /// [`Odb::add_custom_backend`]: crate::Odb::add_custom_backend fn supported_operations(&self) -> SupportedOperations; /// Read an object. @@ -254,7 +260,7 @@ pub trait OdbBackend { /// /// # Implementation notes /// - /// This method is called when [`Odb::write`](crate::Odb::write) is called, but the object + /// This method is called when [`Odb::write`](Odb::write) is called, but the object /// already exists and will not be rewritten. /// /// Implementations may want to update last-used timestamps. @@ -272,6 +278,33 @@ pub trait OdbBackend { unimplemented!("OdbBackend::freshen") } + /// Opens a stream to write a packfile to this backend. + /// + /// Corresponds to the `writepack` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::WRITE_PACK`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// TODO: More information on what this even is. + /// + /// # Errors + /// + /// See [`OdbBackend`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn open_writepack( + &mut self, + ctx: &OdbBackendContext, + odb: &Odb<'_>, + callback: IndexerProgressCallback, + ) -> Result { + unimplemented!("OdbBackend::open_writepack") + } + /// Creates a `multi-pack-index` file containing an index of all objects across all `.pack` /// files. /// @@ -298,7 +331,6 @@ pub trait OdbBackend { // TODO: fn writestream() // TODO: fn readstream() // TODO: fn foreach() - // TODO: fn writepack() } bitflags! { @@ -411,8 +443,255 @@ impl OdbBackendContext { }) } } +/// Indexer progress callback. +pub struct IndexerProgressCallback { + callback: raw::git_indexer_progress_cb, + payload: *mut libc::c_void, +} +impl IndexerProgressCallback { + /// Invokes this callback. + pub fn invoke(&mut self, progress: &IndexerProgress) -> Result<(), Error> { + let Some(callback) = self.callback else { + return Ok(()); + }; + let value = callback(unsafe { progress.raw.as_ref() }, self.payload); + if value != raw::GIT_OK { + return Err(Error::last_error(value)); + } + Ok(()) + } + /// Creates an [`Indexer`] using this callback. Compare with [`crate::indexer::Indexer`]. + pub fn into_indexer(self, odb: &Odb<'_>, path: &Path, verify: bool) -> Result { + let mut opts = unsafe { mem::zeroed::() }; + unsafe { + try_call!(raw::git_indexer_options_init( + &mut opts, + raw::GIT_INDEXER_OPTIONS_VERSION + )); + } + opts.progress_cb = self.callback; + opts.progress_cb_payload = self.payload; + opts.verify = verify.into(); + let mut indexer: *mut raw::git_indexer = ptr::null_mut(); + let path = path.into_c_string()?; + unsafe { + try_call!(raw::git_indexer_new( + &mut indexer, + path.as_ptr(), + 0, + odb.raw(), + &mut opts + )); + } + + Ok(Indexer { + raw: ptr::NonNull::new(indexer).unwrap(), + }) + } +} + +/// Indexer that stores packfiles at an arbitrary path. See [`crate::indexer::Indexer`]. +/// +/// TODO: Merge this type with aforementioned [`crate::indexer::Indexer`]? +/// They do essentially the same thing, except that the older one allows setting a callback. +/// It's probably better to merge the two into one base type and then have the elder become a +/// wrapper around the base type similar to [`CustomOdbBackend`] that allows setting the +/// callback. +pub struct Indexer { + raw: ptr::NonNull, +} +impl Indexer { + /// Appends data to this indexer. + pub fn append(&mut self, data: &[u8], stats: &IndexerProgress) -> Result { + let result = unsafe { + raw::git_indexer_append( + self.raw.as_ptr(), + data.as_ptr().cast(), + data.len(), + stats.raw().as_ptr(), + ) + }; + + if result < 0 { + Err(Error::last_error(result)) + } else { + Ok(result as usize) + } + } + /// Commit the packfile to disk. + pub fn commit(&mut self, stats: &IndexerProgress) -> Result<(), Error> { + unsafe { + try_call!(raw::git_indexer_commit( + self.raw.as_ptr(), + stats.raw().as_ptr() + )); + } + Ok(()) + } +} +impl Drop for Indexer { + fn drop(&mut self) { + unsafe { raw::git_indexer_free(self.raw.as_ptr()) } + } +} + +/// Implementation of the [`git_odb_writepack`] interface. See [`OdbBackend`]. +/// +/// This is the backend equivalent of [`OdbPackwriter`]. +/// +/// TODO: More documentation regarding what Writepack even is. +/// libgit2 is an enigma slowly peeled back in lines. +/// +/// [`git_odb_writepack`]: raw::git_odb_writepack +/// [`OdbPackwriter`]: crate::odb::OdbPackwriter +pub trait OdbWritepack> { + /// Append data to this stream. + /// + /// Corresponds to the `append` function of [`git_odb_writepack`]. + /// See [`OdbBackend::open_writepack`] for more information. + /// See also [`OdbPackwriter`] (this method corresponds to its [`io::Write`] implementation). + /// + /// # Implementation notes + /// + /// TODO: Implementation notes + /// + /// [`git_odb_writepack`]: raw::git_odb_writepack + /// [`OdbPackwriter`]: crate::odb::OdbPackwriter + /// [`io::Write`]: std::io::Write + fn append( + &mut self, + context: &mut OdbWritepackContext, + data: &[u8], + stats: &mut IndexerProgress, + ) -> Result<(), Error>; + /// Finish writing this packfile. + /// + /// Corresponds to the `commit` function of [`git_odb_writepack`]. + /// See [`OdbBackend::open_writepack`] for more information. + /// See also [`OdbPackwriter`] (this method corresponds to [`OdbPackwriter::commit`]). + /// + /// [`git_odb_writepack`]: raw::git_odb_writepack + /// [`OdbPackwriter`]: crate::odb::OdbPackwriter + /// [`OdbPackwriter::commit`]: crate::odb::OdbPackwriter::commit + fn commit( + &mut self, + context: &mut OdbWritepackContext, + stats: &mut IndexerProgress, + ) -> Result<(), Error>; +} + +impl> OdbWritepack for Infallible { + fn append( + &mut self, + _context: &mut OdbWritepackContext, + _data: &[u8], + _stats: &mut IndexerProgress, + ) -> Result<(), Error> { + unreachable!() + } + + fn commit( + &mut self, + _context: &mut OdbWritepackContext, + _stats: &mut IndexerProgress, + ) -> Result<(), Error> { + unreachable!() + } +} +/// Context struct passed to [`OdbWritepack`]'s methods. +/// +/// This type allows access to the associated [`OdbBackend`]. +pub struct OdbWritepackContext { + backend_ptr: ptr::NonNull>, +} +impl OdbWritepackContext { + /// Get a reference to the associated [`OdbBackend`]. + pub fn backend(&self) -> &B { + unsafe { &self.backend_ptr.as_ref().inner } + } + /// Get a mutable reference to the associated [`OdbBackend`]. + pub fn backend_mut(&mut self) -> &mut B { + unsafe { &mut self.backend_ptr.as_mut().inner } + } +} + +/// For tracking statistics in [`OdbWritepack`] implementations. +/// +/// This type is essentially just a mutable version of the [`Progress`] type; in fact, their only +/// difference is that [`IndexerProgress`] contains a [`NonNull`] whereas +/// [`Progress`] is either an owned value or a pointer to [`git_indexer_progress`]. +/// +/// [`Progress`]: crate::Progress +/// [`git_indexer_progress`]: raw::git_indexer_progress +/// [`NonNull`]: ptr::NonNull +pub struct IndexerProgress { + raw: ptr::NonNull, +} -/// A handle to an [`OdbBackend`] that has been added to an [`Odb`](crate::Odb). +macro_rules! define_stats { + ( + $( + $field_name:ident: $set_fn:ident, $as_mut_fn:ident, $preferred_type:ident, $native_type:ident + ),* + ) => { + impl IndexerProgress { + $( + #[doc = "Gets the `"] + #[doc = stringify!($field_name)] + #[doc = "` field's value from the underlying"] + #[doc = "[`git_indexer_progress`](raw::git_indexer_progress) struct"] + pub fn $field_name(&self) -> $preferred_type { + unsafe { self.raw.as_ref() }.$field_name + } + #[doc = "Sets the `"] + #[doc = stringify!($field_name)] + #[doc = "` field's value from the underlying"] + #[doc = "[`git_indexer_progress`](raw::git_indexer_progress) struct"] + #[doc = ""] + #[doc = "# Panics"] + #[doc = ""] + #[doc = "This method may panic if the value cannot be casted to the native type, which"] + #[doc = "is only the case for platforms where [`libc::"] + #[doc = stringify!($native_type)] + #[doc = "`] is smaller than [`"] + #[doc = stringify!($preferred_type)] + #[doc = "`]."] + pub fn $set_fn(&mut self, value: $preferred_type) { + unsafe { self.raw.as_mut() }.$field_name = value as libc::$native_type; + } + #[doc = "Return a mutable reference to the underlying `"] + #[doc = stringify!($field_name)] + #[doc = "` field of the [`git_indexer_progress`](raw::git_indexer_progress) struct"] + pub fn $as_mut_fn(&mut self) -> &mut libc::$native_type { + &mut unsafe { self.raw.as_mut() }.$field_name + } + )* + } + }; +} +define_stats!( + total_objects: set_total_objects, total_objects_mut, u32, c_uint, + indexed_objects: set_indexed_objects, indexed_objects_mut, u32, c_uint, + received_objects: set_received_objects, received_objects_mut, u32, c_uint, + local_objects: set_local_objects, local_objects_mut, u32, c_uint, + total_deltas: set_total_deltas, total_deltas_mut, u32, c_uint, + indexed_deltas: set_indexed_deltas, indexed_deltas_mut, u32, c_uint, + received_bytes: set_received_bytes, received_bytes_mut, usize, size_t +); + +impl Binding for IndexerProgress { + type Raw = ptr::NonNull; + + unsafe fn from_raw(raw: Self::Raw) -> Self { + Self { raw } + } + + fn raw(&self) -> Self::Raw { + self.raw + } +} + +/// A handle to an [`OdbBackend`] that has been added to an [`Odb`]. pub struct CustomOdbBackend<'a, B: OdbBackend> { // NOTE: Any pointer in this field must be both non-null and properly aligned. raw: ptr::NonNull>, @@ -490,6 +769,7 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(exists if EXISTS); op_if!(exists_prefix if EXISTS_PREFIX); op_if!(refresh if REFRESH); + op_if!(writepack if WRITE_PACK); op_if!(writemidx if WRITE_MULTIPACK_INDEX); op_if!(freshen if FRESHEN); @@ -670,6 +950,44 @@ impl Backend { raw::GIT_OK } + extern "C" fn writepack( + out_writepack_ptr: *mut *mut raw::git_odb_writepack, + backend_ptr: *mut raw::git_odb_backend, + odb_ptr: *mut raw::git_odb, + progress_cb: raw::git_indexer_progress_cb, + progress_payload: *mut libc::c_void, + ) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let context = OdbBackendContext { backend_ptr }; + + let odb = unsafe { Odb::from_raw(odb_ptr) }; + let callback = IndexerProgressCallback { + callback: progress_cb, + payload: progress_payload, + }; + + let writepack = match backend.inner.open_writepack(&context, &odb, callback) { + Err(e) => return unsafe { e.raw_set_git_error() }, + Ok(x) => x, + }; + + let writepack = Writepack:: { + writepack: raw::git_odb_writepack { + backend: backend_ptr, + append: Some(Writepack::::append), + commit: Some(Writepack::::commit), + free: Some(Writepack::::free), + }, + inner: writepack, + }; + let writepack = Box::into_raw(Box::new(writepack)); + + let out_writepack = unsafe { out_writepack_ptr.as_mut().unwrap() }; + *out_writepack = writepack.cast(); + + raw::GIT_OK + } + extern "C" fn writemidx(backend_ptr: *mut raw::git_odb_backend) -> libc::c_int { let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; let context = OdbBackendContext { backend_ptr }; @@ -698,3 +1016,60 @@ impl Backend { drop(inner); } } + +#[repr(C)] +struct Writepack +where + B: OdbBackend, +{ + writepack: raw::git_odb_writepack, + inner: B::Writepack, +} + +impl Writepack +where + B: OdbBackend, +{ + extern "C" fn append( + writepack_ptr: *mut raw::git_odb_writepack, + data_ptr: *const libc::c_void, + data_len: libc::size_t, + progress_ptr: *mut raw::git_indexer_progress, + ) -> libc::c_int { + let writepack_ptr = unsafe { ptr::NonNull::new_unchecked(writepack_ptr) }; + let data = unsafe { slice::from_raw_parts(data_ptr.cast::(), data_len) }; + let mut progress = + unsafe { IndexerProgress::from_raw(ptr::NonNull::new_unchecked(progress_ptr)) }; + let writepack = unsafe { writepack_ptr.cast::().as_mut() }; + + let mut context = OdbWritepackContext { + backend_ptr: unsafe { ptr::NonNull::new_unchecked(writepack.writepack.backend) }.cast(), + }; + + if let Err(e) = writepack.inner.append(&mut context, data, &mut progress) { + return unsafe { e.raw_set_git_error() }; + } + + raw::GIT_OK + } + extern "C" fn commit( + writepack_ptr: *mut raw::git_odb_writepack, + progress_ptr: *mut raw::git_indexer_progress, + ) -> libc::c_int { + let writepack_ptr = unsafe { ptr::NonNull::new_unchecked(writepack_ptr) }; + let writepack = unsafe { writepack_ptr.cast::().as_mut() }; + let mut progress = + unsafe { IndexerProgress::from_raw(ptr::NonNull::new_unchecked(progress_ptr)) }; + let mut context = OdbWritepackContext { + backend_ptr: unsafe { ptr::NonNull::new_unchecked(writepack.writepack.backend) }.cast(), + }; + if let Err(e) = writepack.inner.commit(&mut context, &mut progress) { + return unsafe { e.raw_set_git_error() }; + } + raw::GIT_OK + } + extern "C" fn free(writepack_ptr: *mut raw::git_odb_writepack) { + let inner = unsafe { Box::from_raw(writepack_ptr.cast::()) }; + drop(inner); + } +} From 5e4fb91d9b828f7375ff4473d8af238574fdd1d3 Mon Sep 17 00:00:00 2001 From: tecc Date: Sun, 17 Aug 2025 11:28:42 +0200 Subject: [PATCH 23/28] docs: Fix documentation of `SupportedOperations::WRITE_PACK` --- src/odb_backend.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 7f5b2f639b..6e7ad39c34 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -336,8 +336,9 @@ pub trait OdbBackend: Sized { bitflags! { /// Supported operations for a backend. pub struct SupportedOperations: u32 { - // NOTE: The names are taken from the trait method names, but the order is taken from the - // fields of git_odb_backend. + // NOTE: The names are mostly taken from the trait method names, but the order of the flags + // is taken from the fields of git_odb_backend. + // Essentially, choose a name that is tasteful. /// The backend supports the [`OdbBackend::read`] method. const READ = 1; /// The backend supports the [`OdbBackend::read_prefix`] method. @@ -358,7 +359,7 @@ bitflags! { const REFRESH = 1 << 7; /// The backend supports the [`OdbBackend::foreach`] method. const FOREACH = 1 << 8; - /// The backend supports the [`OdbBackend::write_pack`] method. + /// The backend supports the [`OdbBackend::open_writepack`] method. const WRITE_PACK = 1 << 9; /// The backend supports the [`OdbBackend::write_multipack_index`] method. const WRITE_MULTIPACK_INDEX = 1 << 10; From 203f81e3865b926b3a5f99d44e0e64b62161da95 Mon Sep 17 00:00:00 2001 From: tecc Date: Tue, 19 Aug 2025 18:19:46 +0200 Subject: [PATCH 24/28] feat: Streaming read and writes (`readstream` and `writestream`) Added method `OdbBackend::open_read_stream`, corresponding to the `readstream` function of `git_odb_backend`. Added trait `OdbReadStream` as a subset of `git_odb_stream`'s interface; in particular, the `read` function of `git_odb_stream`. Added associated type `OdbBackend::ReadStream: OdbReadStream`. Added `OdbBackend::open_write_stream`, corresponding to the `writestream` function of `git_odb_backend`. Added trait `OdbWriteStream` as a subset of `git_odb_stream`'s interface; in particular, the `write` and `finalize_write` functions of `git_odb_stream`. Added associated type `OdbBackend::WriteStream: OdbWriteStream`. Added `allocate` and `free` functions to reduce code repetition. --- src/odb_backend.rs | 325 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 321 insertions(+), 4 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 6e7ad39c34..709d17d032 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -32,6 +32,14 @@ pub trait OdbBackend: Sized { /// /// [`Infallible`]: std::convert::Infallible type Writepack: OdbWritepack; + /// Backend-specific readable stream. + /// + /// If the backend doesn't support reading through streams, this type should be [`Infallible`]. + type ReadStream: OdbReadStream; + /// Backend-specific writable stream. + /// + /// If the backend doesn't support writing through streams, this type should be [`Infallible`]. + type WriteStream: OdbWriteStream; /// Returns the supported operations of this backend. /// The return value is used to determine what functions to provide to libgit2. @@ -328,8 +336,62 @@ pub trait OdbBackend: Sized { unimplemented!("OdbBackend::write_multipack_index") } - // TODO: fn writestream() - // TODO: fn readstream() + /// Opens a stream to read an object. + /// + /// Corresponds to the `readstream` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::READSTREAM`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// If an implementation returns `Ok(stream)`, `length` and `object_type` MUST be set to the + /// length of the object's contents and the object type respectively; see + /// [`OdbBackend::read_header`]. + /// + /// # Errors + /// + /// See [`OdbBackend::read`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn open_read_stream( + &mut self, + ctx: &OdbBackendContext, + oid: Oid, + length: &mut usize, + object_type: &mut ObjectType, + ) -> Result { + unimplemented!("OdbBackend::open_read_stream") + } + /// Opens a stream to write an object. + /// + /// Corresponds to the `writestream` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::WRITESTREAM`] is present in the value returned from + /// [`supported_operations`] to expose it to libgit2. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// The Oid of the object is calculated by libgit2 *after* all the data has been written. + /// + /// # Errors + /// + /// See [`OdbBackend::write`]. + /// + /// [`git_odb_backend`]: raw::git_odb_backend + /// [`supported_operations`]: Self::supported_operations + fn open_write_stream( + &mut self, + ctx: &OdbBackendContext, + length: usize, + object_type: ObjectType, + ) -> Result { + unimplemented!("OdbBackend::open_write_stream") + } + // TODO: fn foreach() } @@ -347,9 +409,9 @@ bitflags! { const READ_HEADER = 1 << 2; /// The backend supports the [`OdbBackend::write`] method. const WRITE = 1 << 3; - /// The backend supports the [`OdbBackend::writestream`] method. + /// The backend supports the [`OdbBackend::open_write_stream`] method. const WRITESTREAM = 1 << 4; - /// The backend supports the [`OdbBackend::readstream`] method. + /// The backend supports the [`OdbBackend::open_read_stream`] method. const READSTREAM = 1 << 5; /// The backend supports the [`OdbBackend::exists`] method. const EXISTS = 1 << 6; @@ -692,6 +754,94 @@ impl Binding for IndexerProgress { } } +/// A stream that can be read from. +pub trait OdbReadStream { + /// Read as many bytes as possible from this stream, returning how many bytes were read. + /// + /// Corresponds to the `read` function of [`git_odb_stream`]. + /// + /// # Implementation notes + /// + /// If `Ok(read_bytes)` is returned, `read_bytes` should be how many bytes were read from this + /// stream. This number must not exceed the length of `out`. + /// + /// `out` will never have a length greater than [`libc::c_int::MAX`]. + /// + /// > Whilst a caller may be able to pass buffers longer than that, `read_bytes` (from the `Ok` + /// > return value) must be convertible to a [`libc::c_int`] for git2 to be able to return the + /// > value back to libgit2. + /// > For that reason, git2 will automatically limit the buffer length to [`libc::c_int::MAX`]. + /// + /// # Errors + /// + /// See [`OdbBackend`]. + /// + /// [`git_odb_stream`]: raw::git_odb_stream + fn read(&mut self, ctx: &mut OdbStreamContext, out: &mut [u8]) -> Result; +} + +impl OdbReadStream for Infallible { + fn read(&mut self, _ctx: &mut OdbStreamContext, _out: &mut [u8]) -> Result { + unreachable!() + } +} + +/// A stream that can be written to. +pub trait OdbWriteStream { + /// Write bytes to this stream. + /// + /// Corresponds to the `write` function of [`git_odb_stream`]. + /// + /// # Implementation notes + /// + /// All calls to `write` will be "finalized" by a single call to [`finalize_write`], after which + /// no more calls to this stream will occur. + /// + /// [`git_odb_stream`]: raw::git_odb_stream + /// [`finalize_write`]: OdbWriteStream::finalize_write + fn write(&mut self, ctx: &mut OdbStreamContext, data: &[u8]) -> Result<(), Error>; + /// Store the contents of the stream as an object with the specified [`Oid`]. + /// + /// Corresponds to the `finalize_write` function of [`git_odb_stream`]. + /// + /// # Implementation notes + /// + /// This method might not be invoked if: + /// - an error occurs in the [`write`] implementation, + /// - `oid` refers to an already existing object in another backend, or + /// - the final number of received bytes differs from the size declared when the stream was opened. + /// + /// + /// [`git_odb_stream`]: raw::git_odb_stream + /// [`write`]: OdbWriteStream::write + fn finalize_write(&mut self, ctx: &mut OdbStreamContext, oid: Oid) -> Result<(), Error>; +} + +impl OdbWriteStream for Infallible { + fn write(&mut self, _ctx: &mut OdbStreamContext, _data: &[u8]) -> Result<(), Error> { + unreachable!() + } + + fn finalize_write(&mut self, _ctx: &mut OdbStreamContext, _oid: Oid) -> Result<(), Error> { + unreachable!() + } +} + +/// Context struct passed to [`OdbReadStream`] and [`OdbWriteStream`]'s methods. +pub struct OdbStreamContext { + backend_ptr: ptr::NonNull>, +} +impl OdbStreamContext { + /// Get a reference to the associated [`OdbBackend`]. + pub fn backend(&self) -> &B { + unsafe { &self.backend_ptr.as_ref().inner } + } + /// Get a mutable reference to the associated [`OdbBackend`]. + pub fn backend_mut(&mut self) -> &mut B { + unsafe { &mut self.backend_ptr.as_mut().inner } + } +} + /// A handle to an [`OdbBackend`] that has been added to an [`Odb`]. pub struct CustomOdbBackend<'a, B: OdbBackend> { // NOTE: Any pointer in this field must be both non-null and properly aligned. @@ -767,6 +917,8 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(read_prefix if READ_PREFIX); op_if!(read_header if READ_HEADER); op_if!(write if WRITE); + op_if!(writestream if WRITESTREAM); + op_if!(readstream if READSTREAM); op_if!(exists if EXISTS); op_if!(exists_prefix if EXISTS_PREFIX); op_if!(refresh if REFRESH); @@ -902,6 +1054,90 @@ impl Backend { } raw::GIT_OK } + extern "C" fn writestream( + stream_out: *mut *mut raw::git_odb_stream, + backend_ptr: *mut raw::git_odb_backend, + length: raw::git_object_size_t, + object_type: raw::git_object_t, + ) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; + let object_type = ObjectType::from_raw(object_type).unwrap(); + let context = OdbBackendContext { backend_ptr }; + let stream_out = unsafe { stream_out.as_mut().unwrap() }; + let stream = match backend + .inner + .open_write_stream(&context, length as usize, object_type) + { + Err(e) => return unsafe { e.raw_set_git_error() }, + Ok(x) => x, + }; + + let stream = WriteStream:: { + parent: raw::git_odb_stream { + backend: backend_ptr, + mode: raw::GIT_STREAM_WRONLY, + hash_ctx: ptr::null_mut(), + declared_size: 0, + received_bytes: 0, + read: None, + write: Some(WriteStream::::write), + finalize_write: Some(WriteStream::::finalize_write), + free: Some(WriteStream::::free), + }, + _marker: marker::PhantomData, + inner: stream, + }; + + *stream_out = unsafe { allocate(stream).cast() }; + + raw::GIT_OK + } + extern "C" fn readstream( + stream_out: *mut *mut raw::git_odb_stream, + length_ptr: *mut libc::size_t, + otype_ptr: *mut raw::git_object_t, + backend_ptr: *mut raw::git_odb_backend, + oid_ptr: *const raw::git_oid, + ) -> libc::c_int { + let size = unsafe { length_ptr.as_mut().unwrap() }; + let otype = unsafe { otype_ptr.as_mut().unwrap() }; + let backend = unsafe { backend_ptr.cast::>().as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + let stream_out = unsafe { stream_out.as_mut().unwrap() }; + + let context = OdbBackendContext { backend_ptr }; + + let mut object_type = ObjectType::Any; + let stream = match backend + .inner + .open_read_stream(&context, oid, size, &mut object_type) + { + Err(e) => return unsafe { e.raw_set_git_error() }, + Ok(x) => x, + }; + + *otype = object_type.raw(); + + let stream = ReadStream:: { + parent: raw::git_odb_stream { + backend: backend_ptr, + mode: raw::GIT_STREAM_RDONLY, + hash_ctx: ptr::null_mut(), + declared_size: 0, + received_bytes: 0, + read: Some(ReadStream::::read), + write: None, + finalize_write: None, + free: Some(ReadStream::::free), + }, + _marker: marker::PhantomData, + inner: stream, + }; + + *stream_out = unsafe { allocate(stream).cast() }; + + raw::GIT_OK + } extern "C" fn exists( backend_ptr: *mut raw::git_odb_backend, @@ -1074,3 +1310,84 @@ where drop(inner); } } + +struct Stream { + parent: raw::git_odb_stream, + _marker: marker::PhantomData, + inner: T, +} +impl Stream { + extern "C" fn read( + stream_ptr: *mut raw::git_odb_stream, + out_ptr: *mut libc::c_char, + out_len: libc::size_t, + ) -> libc::c_int + where + B: OdbBackend, + T: OdbReadStream, + { + let stream = unsafe { stream_ptr.cast::().as_mut().unwrap() }; + let buf_len = (out_len as usize).max(libc::c_int::MAX as usize); + let buf = unsafe { slice::from_raw_parts_mut(out_ptr.cast::(), buf_len) }; + let mut context = OdbStreamContext { + backend_ptr: ptr::NonNull::new(stream.parent.backend).unwrap().cast(), + }; + let read_bytes = match stream.inner.read(&mut context, buf) { + Err(e) => return unsafe { e.raw_set_git_error() }, + Ok(x) => x, + }; + read_bytes as libc::c_int + } + + extern "C" fn write( + stream_ptr: *mut raw::git_odb_stream, + data: *const libc::c_char, + len: libc::size_t, + ) -> libc::c_int + where + B: OdbBackend, + T: OdbWriteStream, + { + let stream = unsafe { stream_ptr.cast::().as_mut().unwrap() }; + let data = unsafe { slice::from_raw_parts(data.cast::(), len) }; + let mut context = OdbStreamContext { + backend_ptr: ptr::NonNull::new(stream.parent.backend).unwrap().cast(), + }; + if let Err(e) = stream.inner.write(&mut context, data) { + return unsafe { e.raw_set_git_error() }; + } + + raw::GIT_OK + } + extern "C" fn finalize_write( + stream_ptr: *mut raw::git_odb_stream, + oid_ptr: *const raw::git_oid, + ) -> libc::c_int + where + B: OdbBackend, + T: OdbWriteStream, + { + let stream = unsafe { stream_ptr.cast::().as_mut().unwrap() }; + let oid = unsafe { Oid::from_raw(oid_ptr) }; + let mut context = OdbStreamContext { + backend_ptr: ptr::NonNull::new(stream.parent.backend).unwrap().cast(), + }; + if let Err(e) = stream.inner.finalize_write(&mut context, oid) { + return unsafe { e.raw_set_git_error() }; + } + raw::GIT_OK + } + + extern "C" fn free(stream_ptr: *mut raw::git_odb_stream) { + unsafe { free(stream_ptr.cast::()) } + } +} +type WriteStream = Stream::WriteStream>; +type ReadStream = Stream::ReadStream>; + +unsafe fn allocate(value: T) -> *mut T { + Box::into_raw(Box::new(value)) +} +unsafe fn free(ptr: *mut T) { + drop(Box::from_raw(ptr)) +} From 9d875cf29c90e6a24c274b5403db54a5010b014f Mon Sep 17 00:00:00 2001 From: tecc Date: Tue, 19 Aug 2025 18:32:44 +0200 Subject: [PATCH 25/28] refactor: Use shorthand for allocating/freeing --- src/odb_backend.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 709d17d032..2be06525df 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -1088,7 +1088,7 @@ impl Backend { inner: stream, }; - *stream_out = unsafe { allocate(stream).cast() }; + *stream_out = unsafe { box_allocate(stream).cast() }; raw::GIT_OK } @@ -1134,7 +1134,7 @@ impl Backend { inner: stream, }; - *stream_out = unsafe { allocate(stream).cast() }; + *stream_out = unsafe { box_allocate(stream).cast() }; raw::GIT_OK } @@ -1217,10 +1217,9 @@ impl Backend { }, inner: writepack, }; - let writepack = Box::into_raw(Box::new(writepack)); let out_writepack = unsafe { out_writepack_ptr.as_mut().unwrap() }; - *out_writepack = writepack.cast(); + *out_writepack = unsafe { box_allocate(writepack).cast() }; raw::GIT_OK } @@ -1249,8 +1248,7 @@ impl Backend { } extern "C" fn free(backend: *mut raw::git_odb_backend) { - let inner = unsafe { Box::from_raw(backend.cast::()) }; - drop(inner); + unsafe { box_free(backend.cast::()) } } } @@ -1306,8 +1304,7 @@ where raw::GIT_OK } extern "C" fn free(writepack_ptr: *mut raw::git_odb_writepack) { - let inner = unsafe { Box::from_raw(writepack_ptr.cast::()) }; - drop(inner); + unsafe { box_free(writepack_ptr.cast::()) } } } @@ -1379,15 +1376,15 @@ impl Stream { } extern "C" fn free(stream_ptr: *mut raw::git_odb_stream) { - unsafe { free(stream_ptr.cast::()) } + unsafe { box_free(stream_ptr.cast::()) } } } type WriteStream = Stream::WriteStream>; type ReadStream = Stream::ReadStream>; -unsafe fn allocate(value: T) -> *mut T { +unsafe fn box_allocate(value: T) -> *mut T { Box::into_raw(Box::new(value)) } -unsafe fn free(ptr: *mut T) { +unsafe fn box_free(ptr: *mut T) { drop(Box::from_raw(ptr)) } From 04b79b054203612e35ee5ba9271a2d8b53206d62 Mon Sep 17 00:00:00 2001 From: tecc Date: Tue, 19 Aug 2025 18:37:41 +0200 Subject: [PATCH 26/28] fix: Add cast to `_` for `git_odb_stream.mode` --- src/odb_backend.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 2be06525df..66139d4592 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -1075,7 +1075,7 @@ impl Backend { let stream = WriteStream:: { parent: raw::git_odb_stream { backend: backend_ptr, - mode: raw::GIT_STREAM_WRONLY, + mode: raw::GIT_STREAM_WRONLY as _, hash_ctx: ptr::null_mut(), declared_size: 0, received_bytes: 0, @@ -1121,7 +1121,7 @@ impl Backend { let stream = ReadStream:: { parent: raw::git_odb_stream { backend: backend_ptr, - mode: raw::GIT_STREAM_RDONLY, + mode: raw::GIT_STREAM_RDONLY as _, hash_ctx: ptr::null_mut(), declared_size: 0, received_bytes: 0, From 88a891902c194362faedc16586e36974e154cabe Mon Sep 17 00:00:00 2001 From: tecc Date: Tue, 19 Aug 2025 19:41:58 +0200 Subject: [PATCH 27/28] feat: Add SQLite ODB backend example using rusqlite 0.37 --- Cargo.lock | 62 +++++++++++++++ Cargo.toml | 1 + examples/odb_backend_sqlite.rs | 137 +++++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 examples/odb_backend_sqlite.rs diff --git a/Cargo.lock b/Cargo.lock index d302b33c3b..5dc3d10fb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,12 +234,30 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -320,6 +338,7 @@ dependencies = [ "log", "openssl-probe", "openssl-sys", + "rusqlite", "tempfile", "time", "url", @@ -336,6 +355,24 @@ dependencies = [ "url", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown", +] + [[package]] name = "heck" version = "0.5.0" @@ -499,6 +536,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libssh2-sys" version = "0.3.1" @@ -658,6 +706,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags 2.9.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc_version" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index 4f53a90c03..3840edc4fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ openssl-probe = { version = "0.1", optional = true } [dev-dependencies] clap = { version = "4.4.13", features = ["derive"] } time = { version = "0.3.37", features = ["formatting"] } +rusqlite = { version = "0.37.0", features = ["bundled"] } tempfile = "3.1.0" url = "2.5.4" diff --git a/examples/odb_backend_sqlite.rs b/examples/odb_backend_sqlite.rs new file mode 100644 index 0000000000..d089734046 --- /dev/null +++ b/examples/odb_backend_sqlite.rs @@ -0,0 +1,137 @@ +//! # ODB backend implementation: SQLite +//! The following is a port of libgit2-backends' `sqlite/sqlite.c` file. + +use git2::odb_backend::{OdbBackend, OdbBackendAllocation, OdbBackendContext, SupportedOperations}; +use git2::{Error, ErrorClass, ErrorCode, ObjectType, Oid}; +use libgit2_sys as raw; +use rusqlite::Error as RusqliteError; +use rusqlite::{params, OptionalExtension}; +use std::convert::Infallible; + +pub struct SqliteBackend { + conn: rusqlite::Connection, +} + +const ST_READ: &str = "SELECT type, size, data FROM 'git2_odb' WHERE oid = ?;"; +const ST_READ_HEADER: &str = "SELECT type, size FROM 'git2_odb' WHERE `oid` = ?;"; +const ST_WRITE: &str = "INSERT OR IGNORE INTO 'git2_odb' VALUES (?, ?, ?, ?)"; + +impl SqliteBackend { + pub fn new(conn: rusqlite::Connection) -> rusqlite::Result { + // Check if we need to create the git2_odb table + if conn.table_exists(None, "git2_odb")? { + // Table exists, do nothing + } else { + conn.execute("CREATE TABLE 'git2_odb' ('oid' CHARACTER(20) PRIMARY KET NOT NULL, 'type' INTEGER NOT NULL, 'size' INTEGER NOT NULL, 'data' BLOB)", params![])?; + } + + conn.prepare_cached(ST_READ)?; + conn.prepare_cached(ST_READ_HEADER)?; + conn.prepare_cached(ST_WRITE)?; + + Ok(Self { conn }) + } +} + +impl OdbBackend for SqliteBackend { + type Writepack = Infallible; + type ReadStream = Infallible; + type WriteStream = Infallible; + + fn supported_operations(&self) -> SupportedOperations { + SupportedOperations::READ + | SupportedOperations::READ_HEADER + | SupportedOperations::WRITE + | SupportedOperations::EXISTS + } + + fn read( + &mut self, + ctx: &OdbBackendContext, + oid: Oid, + object_type: &mut ObjectType, + data: &mut OdbBackendAllocation, + ) -> Result<(), Error> { + let row = self + .conn + .prepare_cached(ST_READ) + .map_err(map_sqlite_err)? + .query_one(params![oid.as_bytes()], |row| { + let object_type: raw::git_object_t = row.get(0)?; + let size: usize = row.get(1)?; + let data: Box<[u8]> = row.get(2)?; + Ok((ObjectType::from_raw(object_type).unwrap(), size, data)) + }) + .map_err(map_sqlite_err)?; + *object_type = row.0; + *data = ctx.try_alloc(row.1)?; + data.as_mut().copy_from_slice(&row.2); + Ok(()) + } + + fn read_header( + &mut self, + _ctx: &OdbBackendContext, + oid: Oid, + length: &mut usize, + object_type: &mut ObjectType, + ) -> Result<(), Error> { + let row = self + .conn + .prepare_cached(ST_READ_HEADER) + .map_err(map_sqlite_err)? + .query_one(params![oid.as_bytes()], |row| { + let object_type: raw::git_object_t = row.get(0)?; + let size: usize = row.get(1)?; + Ok((ObjectType::from_raw(object_type).unwrap(), size)) + }) + .map_err(map_sqlite_err)?; + *object_type = row.0; + *length = row.1; + Ok(()) + } + + fn write( + &mut self, + _ctx: &OdbBackendContext, + oid: Oid, + object_type: ObjectType, + data: &[u8], + ) -> Result<(), Error> { + self.conn + .prepare_cached(ST_WRITE) + .map_err(map_sqlite_err)? + .execute(params![ + oid.as_bytes(), + object_type.raw(), + oid.as_bytes().len(), + data + ]) + .map_err(map_sqlite_err)?; + Ok(()) + } + + fn exists(&mut self, _ctx: &OdbBackendContext, oid: Oid) -> Result { + let row = self + .conn + .prepare_cached(ST_READ_HEADER) + .map_err(map_sqlite_err)? + .query_one(params![oid.as_bytes()], |_| Ok(())) + .optional() + .map_err(map_sqlite_err)?; + Ok(row.is_some()) + } +} + +fn map_sqlite_err(err: RusqliteError) -> Error { + match err { + RusqliteError::QueryReturnedNoRows => { + Error::new(ErrorCode::NotFound, ErrorClass::None, "not found") + } + _ => Error::new(ErrorCode::GenericError, ErrorClass::Object, err.to_string()), + } +} + +fn main() { + todo!("Demonstrate how to use SqliteBackend") +} From 6a6b9ee6184b5ecb32a1a7d059c86549480d007e Mon Sep 17 00:00:00 2001 From: tecc Date: Fri, 22 Aug 2025 22:58:03 +0200 Subject: [PATCH 28/28] feat: Add `OdbBackend::foreach` --- src/odb_backend.rs | 72 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/src/odb_backend.rs b/src/odb_backend.rs index 66139d4592..d89afd6dcc 100644 --- a/src/odb_backend.rs +++ b/src/odb_backend.rs @@ -392,7 +392,28 @@ pub trait OdbBackend: Sized { unimplemented!("OdbBackend::open_write_stream") } - // TODO: fn foreach() + /// Iterates through all objects this backend knows of. + /// + /// Corresponds to the `foreach` function of [`git_odb_backend`]. + /// Requires that [`SupportedOperations::FOREACH`] is present in the value returned from + /// [`supported_operations`]. + /// + /// The default implementation of this method panics. + /// + /// # Implementation notes + /// + /// For each object in this backend, `callback` should be invoked with the object's Oid once. + /// See [`ForeachCallback::invoke`]. + /// + /// # Errors + /// + /// If the callback returns an error, the backend SHOULD propagate that error instead of + /// continuing iteration. + /// + /// See [`OdbBackend`] for more recommendations. + fn foreach(&mut self, ctx: &OdbBackendContext, callback: ForeachCallback) -> Result<(), Error> { + unimplemented!("OdbBackend::foreach") + } } bitflags! { @@ -827,6 +848,37 @@ impl OdbWriteStream for Infallible { } } +/// Callback used in calls to [`OdbBackend::foreach`]. +/// +/// A wrapper for [`git_odb_foreach_cb`](raw::git_odb_foreach_cb). +pub struct ForeachCallback { + callback: raw::git_odb_foreach_cb, + payload: *mut libc::c_void, +} +impl ForeachCallback { + /// Invokes this callback. + /// + /// # Arguments + /// + /// `oid` should be the ID of the object that was iterated over. + /// + /// # Return value + /// + /// Returns `Ok(())` if the callback returns 0. + /// Returns an error if the callback returns a non-zero integer. + pub fn invoke(&mut self, oid: &Oid) -> Result<(), Error> { + let code = match self.callback { + Some(callback) => callback(oid.raw(), self.payload), + None => return Ok(()), // maybe this should be an error + }; + if code != 0 { + Err(Error::last_error(code)) + } else { + Ok(()) + } + } +} + /// Context struct passed to [`OdbReadStream`] and [`OdbWriteStream`]'s methods. pub struct OdbStreamContext { backend_ptr: ptr::NonNull>, @@ -922,6 +974,7 @@ impl<'a, B: OdbBackend> CustomOdbBackend<'a, B> { op_if!(exists if EXISTS); op_if!(exists_prefix if EXISTS_PREFIX); op_if!(refresh if REFRESH); + op_if!(foreach if FOREACH); op_if!(writepack if WRITE_PACK); op_if!(writemidx if WRITE_MULTIPACK_INDEX); op_if!(freshen if FRESHEN); @@ -1187,6 +1240,23 @@ impl Backend { raw::GIT_OK } + extern "C" fn foreach( + backend_ptr: *mut raw::git_odb_backend, + foreach_cb: raw::git_odb_foreach_cb, + foreach_cb_payload: *mut libc::c_void, + ) -> libc::c_int { + let backend = unsafe { backend_ptr.cast::().as_mut().unwrap() }; + let context = OdbBackendContext { backend_ptr }; + let callback = ForeachCallback { + callback: foreach_cb, + payload: foreach_cb_payload, + }; + if let Err(e) = backend.inner.foreach(&context, callback) { + return unsafe { e.raw_set_git_error() }; + } + raw::GIT_OK + } + extern "C" fn writepack( out_writepack_ptr: *mut *mut raw::git_odb_writepack, backend_ptr: *mut raw::git_odb_backend,