From 362f623b4d027f8ce137731e7ac948749f74494c Mon Sep 17 00:00:00 2001 From: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> Date: Fri, 3 Oct 2025 13:35:30 -0400 Subject: [PATCH 1/5] Refactor storage options handling in open_table Refactor open_table_with_storage_options to remove 'aws.' prefixes from options. Signed-off-by: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> --- crates/core/src/delta_datafusion/mod.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index a656b235e2..b616e6e91b 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -807,7 +807,21 @@ impl TableProviderFactory for DeltaTableFactory { open_table(table_url).await? } else { let table_url = ensure_table_uri(&cmd.to_owned().location)?; - open_table_with_storage_options(table_url, cmd.to_owned().options).await? + open_table_with_storage_options( + table_url, + // remove "aws." prefixes from options + cmd.to_owned() + .options + .iter() + .filter_map(|(k, v)| { + if k.starts_with("aws.") { + Some((k.trim_start_matches("aws.").to_string(), v.to_string())) + } else { + Some((k.to_string(), v.to_string())) + } + }) + .collect::>(), + ).await? }; Ok(Arc::new(provider)) } From 4021d8aa97cc64f4bb66957dcbaeba5cc2ee5a89 Mon Sep 17 00:00:00 2001 From: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> Date: Sat, 4 Oct 2025 12:07:03 -0400 Subject: [PATCH 2/5] add: de/serializer to Signed-off-by: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> --- .../src/delta_datafusion/table_provider.rs | 2 +- crates/core/src/logstore/mod.rs | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/core/src/delta_datafusion/table_provider.rs b/crates/core/src/delta_datafusion/table_provider.rs index df62050704..7708f4b178 100644 --- a/crates/core/src/delta_datafusion/table_provider.rs +++ b/crates/core/src/delta_datafusion/table_provider.rs @@ -773,7 +773,7 @@ impl TableProvider for DeltaTable { } /// A Delta table provider that enables additional metadata columns to be included during the scan -#[derive(Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct DeltaTableProvider { snapshot: EagerSnapshot, log_store: LogStoreRef, diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index d505f0dfaa..2481cee21e 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -159,6 +159,27 @@ pub fn default_logstore( /// Sharable reference to [`LogStore`] pub type LogStoreRef = Arc; +impl Serialize for dyn LogStore + '_ { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.config().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Box { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let config = LogStoreConfig::deserialize(deserializer)?; + logstore_for(config.location.clone(), config.options.clone()) + .map_err(D::Error::custom) + .map(|ls| Box::new(ls) as Box) + } +} + static DELTA_LOG_PATH: LazyLock = LazyLock::new(|| Path::from("_delta_log")); pub(crate) static DELTA_LOG_REGEX: LazyLock = From ae8cb6d57785287251292e78cc8084ede042a2ae Mon Sep 17 00:00:00 2001 From: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> Date: Sat, 4 Oct 2025 12:27:17 -0400 Subject: [PATCH 3/5] add: DeltaSinkWire for de/encode Signed-off-by: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> --- crates/core/src/delta_datafusion/table_provider.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/core/src/delta_datafusion/table_provider.rs b/crates/core/src/delta_datafusion/table_provider.rs index 7708f4b178..10df7953d7 100644 --- a/crates/core/src/delta_datafusion/table_provider.rs +++ b/crates/core/src/delta_datafusion/table_provider.rs @@ -96,6 +96,14 @@ pub struct DeltaDataSink { metrics: ExecutionPlanMetricsSet, } +#[derive(Serialize, Deserialize)] +pub struct DeltaDataSinkWire { + log_store: LogStoreRef, + snapshot: EagerSnapshot, + save_mode: SaveMode, + schema: SchemaRef, +} + /// A [`DataSink`] implementation for writing to Delta Lake. /// /// `DeltaDataSink` is used by [`DataSinkExec`] during query execution to From 13c80b6368e0d4450000b13980a316261956c8dd Mon Sep 17 00:00:00 2001 From: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> Date: Sat, 4 Oct 2025 16:19:03 -0400 Subject: [PATCH 4/5] Revert "add: DeltaSinkWire for de/encode" This reverts commit 422916fab573c4580c0935f04d93395c3753fab3. Signed-off-by: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> --- crates/core/src/delta_datafusion/table_provider.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/core/src/delta_datafusion/table_provider.rs b/crates/core/src/delta_datafusion/table_provider.rs index 10df7953d7..7708f4b178 100644 --- a/crates/core/src/delta_datafusion/table_provider.rs +++ b/crates/core/src/delta_datafusion/table_provider.rs @@ -96,14 +96,6 @@ pub struct DeltaDataSink { metrics: ExecutionPlanMetricsSet, } -#[derive(Serialize, Deserialize)] -pub struct DeltaDataSinkWire { - log_store: LogStoreRef, - snapshot: EagerSnapshot, - save_mode: SaveMode, - schema: SchemaRef, -} - /// A [`DataSink`] implementation for writing to Delta Lake. /// /// `DeltaDataSink` is used by [`DataSinkExec`] during query execution to From eb0bf65b788411f56c7382238eb9c3b647d7d2b6 Mon Sep 17 00:00:00 2001 From: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> Date: Sat, 4 Oct 2025 16:20:06 -0400 Subject: [PATCH 5/5] Revert "add: de/serializer to" This reverts commit 4794f749f43e8ef10541c487c6d916c5d79a9080. Signed-off-by: Dan-J-D <52302961+Dan-J-D@users.noreply.github.com> --- .../src/delta_datafusion/table_provider.rs | 2 +- crates/core/src/logstore/mod.rs | 21 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/crates/core/src/delta_datafusion/table_provider.rs b/crates/core/src/delta_datafusion/table_provider.rs index 7708f4b178..df62050704 100644 --- a/crates/core/src/delta_datafusion/table_provider.rs +++ b/crates/core/src/delta_datafusion/table_provider.rs @@ -773,7 +773,7 @@ impl TableProvider for DeltaTable { } /// A Delta table provider that enables additional metadata columns to be included during the scan -#[derive(Serialize, Deserialize, Debug)] +#[derive(Debug)] pub struct DeltaTableProvider { snapshot: EagerSnapshot, log_store: LogStoreRef, diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 2481cee21e..d505f0dfaa 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -159,27 +159,6 @@ pub fn default_logstore( /// Sharable reference to [`LogStore`] pub type LogStoreRef = Arc; -impl Serialize for dyn LogStore + '_ { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.config().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for Box { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let config = LogStoreConfig::deserialize(deserializer)?; - logstore_for(config.location.clone(), config.options.clone()) - .map_err(D::Error::custom) - .map(|ls| Box::new(ls) as Box) - } -} - static DELTA_LOG_PATH: LazyLock = LazyLock::new(|| Path::from("_delta_log")); pub(crate) static DELTA_LOG_REGEX: LazyLock =