Skip to content

Commit 1a0b444

Browse files
committed
feat: apache iceberg file added
1 parent 12be47e commit 1a0b444

File tree

3 files changed

+110
-2
lines changed

3 files changed

+110
-2
lines changed
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
description: Apache Iceberg Rust Package
2+
3+
# Rust package
4+
5+
!!! bug
6+
7+
The code below doesn't work. My feeling is that curently (19.12.2024) the `iceberg_rust` package isn't mature yet. For this reason is switched to [Delta Lake](../delta_lake/delta_rs.md).
8+
9+
Dependencies:
10+
11+
```toml
12+
[dependencies]
13+
iceberg-rust = "0.6.1"
14+
iceberg-sql-catalog = "0.6.1"
15+
object_store = "0.11.1"
16+
tokio = "1.42.0"
17+
```
18+
19+
Code:
20+
21+
```rust
22+
use std::{collections::HashMap, error::Error, sync::Arc};
23+
24+
use iceberg_rust::{
25+
catalog::Catalog,
26+
object_store::ObjectStoreBuilder,
27+
spec::{
28+
schema::Schema as IcebergSchema,
29+
types::{PrimitiveType, StructField as IcebergStructField, StructType, Type},
30+
},
31+
table::Table as IcebergTable,
32+
};
33+
use iceberg_sql_catalog::SqlCatalog;
34+
use object_store::aws::AmazonS3Builder;
35+
36+
struct Table {
37+
name: String,
38+
properties: Option<HashMap<String, String>>,
39+
struct_fields: Vec<IcebergStructField>,
40+
}
41+
42+
impl Table {
43+
pub async fn to_iceberg_table(self, catalog: Arc<dyn Catalog>, namespace: &[String]) -> Result<IcebergTable, Box<dyn Error>> {
44+
let schema = IcebergSchema::builder()
45+
.with_fields(StructType::builder().fields(self.struct_fields).build().unwrap())
46+
.build()?;
47+
let properties = self.properties.unwrap_or(HashMap::new());
48+
let table = IcebergTable::builder()
49+
.with_name(self.name)
50+
.with_schema(schema)
51+
.with_properties(properties)
52+
.build(namespace, catalog)
53+
.await?;
54+
55+
Ok(table)
56+
}
57+
}
58+
59+
#[tokio::main]
60+
async fn main() {
61+
let object_store = ObjectStoreBuilder::S3(
62+
AmazonS3Builder::new()
63+
.with_endpoint("http://localhost:5561")
64+
.with_region("us-east-1")
65+
.with_bucket_name("data-lakehouse")
66+
.with_access_key_id("admin")
67+
.with_secret_access_key("password")
68+
.with_virtual_hosted_style_request(true),
69+
);
70+
let catalog: Arc<dyn Catalog> = Arc::new(
71+
SqlCatalog::new(
72+
"postgresql://postgres:postgres@localhost:5500/data_lakehouse_catalog",
73+
"data_lakehouse_catalog",
74+
object_store,
75+
)
76+
.await
77+
.expect("Sql catalog creation failed"),
78+
);
79+
80+
let table = Table {
81+
name: "table_01".to_owned(),
82+
properties: None,
83+
struct_fields: vec![
84+
IcebergStructField {
85+
id: 0,
86+
name: "id".to_owned(),
87+
required: true,
88+
field_type: Type::Primitive(PrimitiveType::Int),
89+
doc: None,
90+
},
91+
IcebergStructField {
92+
id: 1,
93+
name: "age".to_owned(),
94+
required: false,
95+
field_type: Type::Primitive(PrimitiveType::Int),
96+
doc: None,
97+
},
98+
],
99+
};
100+
101+
let namespace = &["namespace_01".to_owned()];
102+
table.to_iceberg_table(catalog.clone(), namespace).await.expect("Table creation failed");
103+
104+
println!("{:?}", catalog);
105+
}
106+
```

docs/data_engineering/data_lakehouse/delta_lake/delta_rs.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# delta-rs
1+
# Rust package: delta-rs
22

33
## S3 configurations
44

mkdocs.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,6 @@ markdown_extensions:
1818
- pymdownx.superfences
1919
# permalinks
2020
- toc:
21-
permalink: true
21+
permalink: true
22+
# call outs
23+
- admonition

0 commit comments

Comments
 (0)