Skip to content

feat: Make duplicate check optional for adding parquet files #1034

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
liurenjie1024 merged 6 commits into from
Mar 12, 2025
Merged

feat: Make duplicate check optional for adding parquet files #1034

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
012f111
feat: Make duplicate check optional for adding parquet files
jonathanc-n Mar 3, 2025
f505488
Merge branch 'main' into check-duplicate
jonathanc-n Mar 3, 2025
9fea7c9
Merge branch 'main' into check-duplicate
jonathanc-n Mar 6, 2025
076b4dd
Merge branch 'main' into check-duplicate
jonathanc-n Mar 12, 2025
74f64e6
fix
jonathanc-n Mar 12, 2025
c5f4b13
Merge branch 'check-duplicate' of https://github.com/jonathanc-n/iceb…
jonathanc-n Mar 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 53 additions & 47 deletions crates/iceberg/src/transaction.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,11 @@ impl<'a> FastAppendAction<'a> {

/// Adds existing parquet files
#[allow(dead_code)]
async fn add_parquet_files(mut self, file_path: Vec<String>) -> Result<Transaction<'a>> {
async fn add_parquet_files(
mut self,
file_path: Vec<String>,
check_duplicate: bool,
) -> Result<Transaction<'a>> {
if !self
.snapshot_produce_action
.tx
Expand All @@ -236,57 +240,59 @@ impl<'a> FastAppendAction<'a> {

self.add_data_files(data_files)?;

self.apply().await
self.apply(check_duplicate).await
}

/// Finished building the action and apply it to the transaction.
pub async fn apply(self) -> Result<Transaction<'a>> {
pub async fn apply(self, check_duplicate: bool) -> Result<Transaction<'a>> {
Copy link
Copy Markdown
Contributor

@liurenjie1024 liurenjie1024 Mar 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should add an option in FileAppendAction rather in apply method, since this approach can't be extended to more options.

struct FileAppendAction {
   check_duplicate: bool
}

impl FileAppendAction {
   pub fn check_duplicate(v: bool) -> Self {
      this.check_duplicate = v;
      self
   }
}

Copy link
Copy Markdown
Contributor Author

@jonathanc-n jonathanc-n Mar 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@liurenjie1024 Thanks, should be good now 👍

// Checks duplicate files
let new_files: HashSet<&str> = self
.snapshot_produce_action
.added_data_files
.iter()
.map(|df| df.file_path.as_str())
.collect();

let mut manifest_stream = self
.snapshot_produce_action
.tx
.table
.inspect()
.manifests()
.scan()
.await?;
let mut referenced_files = Vec::new();

while let Some(batch) = manifest_stream.try_next().await? {
let file_path_array = batch
.column(1)
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"Failed to downcast file_path column to StringArray",
)
})?;

for i in 0..batch.num_rows() {
let file_path = file_path_array.value(i);
if new_files.contains(file_path) {
referenced_files.push(file_path.to_string());
if check_duplicate {
let new_files: HashSet<&str> = self
.snapshot_produce_action
.added_data_files
.iter()
.map(|df| df.file_path.as_str())
.collect();

let mut manifest_stream = self
.snapshot_produce_action
.tx
.table
.inspect()
.manifests()
.scan()
.await?;
let mut referenced_files = Vec::new();

while let Some(batch) = manifest_stream.try_next().await? {
let file_path_array = batch
.column(1)
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"Failed to downcast file_path column to StringArray",
)
})?;

for i in 0..batch.num_rows() {
let file_path = file_path_array.value(i);
if new_files.contains(file_path) {
referenced_files.push(file_path.to_string());
}
}
}
}

if !referenced_files.is_empty() {
return Err(Error::new(
ErrorKind::DataInvalid,
format!(
"Cannot add files that are already referenced by table, files: {}",
referenced_files.join(", ")
),
));
if !referenced_files.is_empty() {
return Err(Error::new(
ErrorKind::DataInvalid,
format!(
"Cannot add files that are already referenced by table, files: {}",
referenced_files.join(", ")
),
));
}
}

self.snapshot_produce_action
Expand Down Expand Up @@ -884,7 +890,7 @@ mod tests {
.build()
.unwrap();
action.add_data_files(vec![data_file.clone()]).unwrap();
let tx = action.apply().await.unwrap();
let tx = action.apply(true).await.unwrap();

// check updates and requirements
assert!(
Expand Down Expand Up @@ -971,7 +977,7 @@ mod tests {

// Attempt to add the existing Parquet files with fast append.
let new_tx = fast_append_action
.add_parquet_files(file_paths.clone())
.add_parquet_files(file_paths.clone(), true)
.await
.expect("Adding existing Parquet files should succeed");

Expand Down
4 changes: 2 additions & 2 deletions crates/integration_tests/tests/shared_tests/append_data_file_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ async fn test_append_data_file() {
let tx = Transaction::new(&table);
let mut append_action = tx.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result
Expand All @@ -134,7 +134,7 @@ async fn test_append_data_file() {
let tx = Transaction::new(&table);
let mut append_action = tx.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result again
Expand Down
2 changes: 1 addition & 1 deletion crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ async fn test_append_partition_data_file() {
append_action
.add_data_files(data_file_valid.clone())
.unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result
Expand Down
4 changes: 2 additions & 2 deletions crates/integration_tests/tests/shared_tests/conflict_commit_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,12 @@ async fn test_append_data_file_conflict() {
let tx1 = Transaction::new(&table);
let mut append_action = tx1.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx1 = append_action.apply().await.unwrap();
let tx1 = append_action.apply(true).await.unwrap();

let tx2 = Transaction::new(&table);
let mut append_action = tx2.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx2 = append_action.apply().await.unwrap();
let tx2 = append_action.apply(true).await.unwrap();
let table = tx2
.commit(&rest_catalog)
.await
Expand Down
2 changes: 1 addition & 1 deletion crates/integration_tests/tests/shared_tests/scan_all_type.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ async fn test_scan_all_type() {
let tx = Transaction::new(&table);
let mut append_action = tx.fast_append(None, vec![]).unwrap();
append_action.add_data_files(data_file.clone()).unwrap();
let tx = append_action.apply().await.unwrap();
let tx = append_action.apply(true).await.unwrap();
let table = tx.commit(&rest_catalog).await.unwrap();

// check result
Expand Down
Loading