Skip to content

Commit c732736

Browse files
shyikoshyiko
committed
Support S3 Table buckets
1 parent 3b55c5b commit c732736

File tree

10 files changed

+326
-2
lines changed

10 files changed

+326
-2
lines changed

‎examples/s3tables/.ice-rest-catalog.envsubst.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
uri: https://s3tables.${AWS_REGION}.amazonaws.com/iceberg
2+
warehouse: ${CATALOG_BUCKET_ARN}
3+
4+
bearerTokens:
5+
- value: foo
6+
7+
anonymousAccess:
8+
enabled: true

examples/scratch/.ice.yaml renamed to examples/s3tables/.ice.yaml

File renamed without changes.

‎examples/s3tables/README.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# examples/s3tables
2+
3+
In the example below, we:
4+
5+
- create S3 Table bucket
6+
- insert & query data via `ice`
7+
8+
```shell
9+
# optional: open shell containing `aws` (awscliv2), `envsubst` & `clickhouse`
10+
devbox shell
11+
12+
export CATALOG_BUCKET="$USER-ice-rest-catalog-s3tables-demo"
13+
export AWS_REGION=us-west-1
14+
15+
source aws.credentials
16+
17+
# create S3 Table bucket
18+
aws s3tables create-table-bucket --name "$CATALOG_BUCKET"
19+
export CATALOG_BUCKET_ARN=$(aws s3tables list-table-buckets --query "tableBuckets[?name==\`$CATALOG_BUCKET\`].arn" --output=text)
20+
21+
# start Iceberg REST Catalog
22+
cat .ice-rest-catalog.envsubst.yaml | envsubst -no-unset -no-empty > .ice-rest-catalog.yaml
23+
ice-rest-catalog
24+
25+
# insert data into catalog
26+
ice insert ns1.table1 -p file://iris.parquet
27+
28+
# check the data
29+
ice scan ns1.table1
30+
31+
# clean up
32+
ice delete-table ns1.table1
33+
ice delete-namespace ns1
34+
35+
# delete S3 Table bucket
36+
aws s3tables delete-table-bucket --table-bucket-arn "$CATALOG_BUCKET_ARN"
37+
```

‎examples/s3tables/devbox.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.10.7/.schema/devbox.schema.json",
3+
"packages": [
4+
"envsubst@latest",
5+
"awscli2@latest",
6+
"[email protected]+6"
7+
],
8+
"env": {
9+
"AT": "ice:examples/scratch"
10+
},
11+
"shell": {
12+
"init_hook": [
13+
"export PATH=$(pwd):$(pwd)/.devbox/bin:$PATH",
14+
"[ -f .devbox/bin/clickhouse ] || (curl https://clickhouse.com/ | sh && mv clickhouse .devbox/bin/)"
15+
]
16+
}
17+
}

‎examples/s3tables/devbox.lock‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
{
2+
"lockfile_version": "1",
3+
"packages": {
4+
"awscli2@latest": {
5+
"last_modified": "2025-05-19T23:16:24Z",
6+
"resolved": "github:NixOS/nixpkgs/359c442b7d1f6229c1dc978116d32d6c07fe8440#awscli2",
7+
"source": "devbox-search",
8+
"version": "2.27.2",
9+
"systems": {
10+
"aarch64-darwin": {
11+
"outputs": [
12+
{
13+
"name": "out",
14+
"path": "/nix/store/b5fa0fplfpiyx8zwc9y7d08a57lsk229-awscli2-2.27.2",
15+
"default": true
16+
},
17+
{
18+
"name": "dist",
19+
"path": "/nix/store/sc270lyggbhz78lcxh9khqmggayqpp6i-awscli2-2.27.2-dist"
20+
}
21+
],
22+
"store_path": "/nix/store/b5fa0fplfpiyx8zwc9y7d08a57lsk229-awscli2-2.27.2"
23+
},
24+
"aarch64-linux": {
25+
"outputs": [
26+
{
27+
"name": "out",
28+
"path": "/nix/store/161s22pbddllc4mabgr5rvrpmhiffsp4-awscli2-2.27.2",
29+
"default": true
30+
},
31+
{
32+
"name": "dist",
33+
"path": "/nix/store/ss8xqzwn1l8qdbwlml439bx1ik13vxlk-awscli2-2.27.2-dist"
34+
}
35+
],
36+
"store_path": "/nix/store/161s22pbddllc4mabgr5rvrpmhiffsp4-awscli2-2.27.2"
37+
},
38+
"x86_64-darwin": {
39+
"outputs": [
40+
{
41+
"name": "out",
42+
"path": "/nix/store/l18j8wgidf145m0zw6ydg8blim44hrkq-awscli2-2.27.2",
43+
"default": true
44+
},
45+
{
46+
"name": "dist",
47+
"path": "/nix/store/q9rzj2ikpl9vn4sns3ydsy7i8sachnha-awscli2-2.27.2-dist"
48+
}
49+
],
50+
"store_path": "/nix/store/l18j8wgidf145m0zw6ydg8blim44hrkq-awscli2-2.27.2"
51+
},
52+
"x86_64-linux": {
53+
"outputs": [
54+
{
55+
"name": "out",
56+
"path": "/nix/store/gx3hx0b3lbm11sishxl4k7a7ha4w405q-awscli2-2.27.2",
57+
"default": true
58+
},
59+
{
60+
"name": "dist",
61+
"path": "/nix/store/r5cw3141myaf5lnzra9c0nc8prb2zv6y-awscli2-2.27.2-dist"
62+
}
63+
],
64+
"store_path": "/nix/store/gx3hx0b3lbm11sishxl4k7a7ha4w405q-awscli2-2.27.2"
65+
}
66+
}
67+
},
68+
"envsubst@latest": {
69+
"last_modified": "2025-05-16T20:19:48Z",
70+
"resolved": "github:NixOS/nixpkgs/12a55407652e04dcf2309436eb06fef0d3713ef3#envsubst",
71+
"source": "devbox-search",
72+
"version": "1.4.3",
73+
"systems": {
74+
"aarch64-darwin": {
75+
"outputs": [
76+
{
77+
"name": "out",
78+
"path": "/nix/store/x53syh6dn8wp33var0nmmphnhz0vh269-envsubst-1.4.3",
79+
"default": true
80+
}
81+
],
82+
"store_path": "/nix/store/x53syh6dn8wp33var0nmmphnhz0vh269-envsubst-1.4.3"
83+
},
84+
"aarch64-linux": {
85+
"outputs": [
86+
{
87+
"name": "out",
88+
"path": "/nix/store/0z9przi3raij5jc31yrin797bsbmxnyq-envsubst-1.4.3",
89+
"default": true
90+
}
91+
],
92+
"store_path": "/nix/store/0z9przi3raij5jc31yrin797bsbmxnyq-envsubst-1.4.3"
93+
},
94+
"x86_64-darwin": {
95+
"outputs": [
96+
{
97+
"name": "out",
98+
"path": "/nix/store/7055nw59099678fcib94bnv1flybq50k-envsubst-1.4.3",
99+
"default": true
100+
}
101+
],
102+
"store_path": "/nix/store/7055nw59099678fcib94bnv1flybq50k-envsubst-1.4.3"
103+
},
104+
"x86_64-linux": {
105+
"outputs": [
106+
{
107+
"name": "out",
108+
"path": "/nix/store/9012qqpjaa0j66lxv94j51z4xhnmqq5a-envsubst-1.4.3",
109+
"default": true
110+
}
111+
],
112+
"store_path": "/nix/store/9012qqpjaa0j66lxv94j51z4xhnmqq5a-envsubst-1.4.3"
113+
}
114+
}
115+
},
116+
"github:NixOS/nixpkgs/nixpkgs-unstable": {
117+
"resolved": "github:NixOS/nixpkgs/e4b09e47ace7d87de083786b404bf232eb6c89d8?lastModified=1748856973&narHash=sha256-RlTsJUvvr8ErjPBsiwrGbbHYW8XbB%2Foek0Gi78XdWKg%3D"
118+
},
119+
"[email protected]+6": {
120+
"last_modified": "2025-05-16T20:19:48Z",
121+
"resolved": "github:NixOS/nixpkgs/12a55407652e04dcf2309436eb06fef0d3713ef3#jdk21_headless",
122+
"source": "devbox-search",
123+
"version": "21.0.7+6",
124+
"systems": {
125+
"aarch64-linux": {
126+
"outputs": [
127+
{
128+
"name": "out",
129+
"path": "/nix/store/r4isdsjpz1i2zi93y6cjl53m0ngvbww3-openjdk-headless-21.0.7+6",
130+
"default": true
131+
},
132+
{
133+
"name": "debug",
134+
"path": "/nix/store/s7h859cnwfbr0ss7dlssphbhhivdc63p-openjdk-headless-21.0.7+6-debug"
135+
}
136+
],
137+
"store_path": "/nix/store/r4isdsjpz1i2zi93y6cjl53m0ngvbww3-openjdk-headless-21.0.7+6"
138+
},
139+
"x86_64-linux": {
140+
"outputs": [
141+
{
142+
"name": "out",
143+
"path": "/nix/store/jfxsvqhkxza9vrmpjw2nfxyz32883q1b-openjdk-headless-21.0.7+6",
144+
"default": true
145+
},
146+
{
147+
"name": "debug",
148+
"path": "/nix/store/ffknhap8n6nh733c33z714a9whmqz392-openjdk-headless-21.0.7+6-debug"
149+
}
150+
],
151+
"store_path": "/nix/store/jfxsvqhkxza9vrmpjw2nfxyz32883q1b-openjdk-headless-21.0.7+6"
152+
}
153+
}
154+
}
155+
}
156+
}

‎examples/s3tables/iris.parquet‎

2.39 KB
Binary file not shown.

‎examples/s3tables/iris.parquet.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Downloaded from https://www.tablab.app/parquet/sample.

‎ice-rest-catalog/pom.xml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,65 @@
155155
<artifactId>commons-codec</artifactId>
156156
<version>1.17.1</version>
157157
</dependency>
158+
<!-- s3tables -->
159+
<dependency>
160+
<groupId>software.amazon.awssdk</groupId>
161+
<artifactId>s3tables</artifactId>
162+
<version>2.29.26</version>
163+
<exclusions>
164+
<exclusion>
165+
<groupId>org.slf4j</groupId>
166+
<artifactId>slf4j-api</artifactId>
167+
</exclusion>
168+
</exclusions>
169+
</dependency>
170+
<dependency>
171+
<groupId>software.amazon.s3tables</groupId>
172+
<artifactId>s3-tables-catalog-for-iceberg</artifactId>
173+
<version>0.1.5</version>
174+
<exclusions>
175+
<exclusion>
176+
<groupId>org.apache.iceberg</groupId>
177+
<artifactId>iceberg-bundled-guava</artifactId>
178+
</exclusion>
179+
<exclusion>
180+
<groupId>org.apache.iceberg</groupId>
181+
<artifactId>iceberg-aws</artifactId>
182+
</exclusion>
183+
<exclusion>
184+
<groupId>commons-logging</groupId>
185+
<artifactId>commons-logging</artifactId>
186+
</exclusion>
187+
<exclusion>
188+
<groupId>software.amazon.awssdk</groupId>
189+
<artifactId>s3tables</artifactId>
190+
</exclusion>
191+
<exclusion>
192+
<groupId>org.apache.iceberg</groupId>
193+
<artifactId>iceberg-common</artifactId>
194+
</exclusion>
195+
<exclusion>
196+
<groupId>org.apache.commons</groupId>
197+
<artifactId>commons-text</artifactId>
198+
</exclusion>
199+
<exclusion>
200+
<groupId>com.github.ben-manes.caffeine</groupId>
201+
<artifactId>caffeine</artifactId>
202+
</exclusion>
203+
<exclusion>
204+
<groupId>org.slf4j</groupId>
205+
<artifactId>slf4j-api</artifactId>
206+
</exclusion>
207+
<exclusion>
208+
<groupId>org.apache.iceberg</groupId>
209+
<artifactId>iceberg-api</artifactId>
210+
</exclusion>
211+
<exclusion>
212+
<groupId>org.apache.iceberg</groupId>
213+
<artifactId>iceberg-core</artifactId>
214+
</exclusion>
215+
</exclusions>
216+
</dependency>
158217
<!-- jetty -->
159218
<dependency>
160219
<groupId>org.eclipse.jetty</groupId>

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/CustomS3TablesCatalog.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*/
10+
package com.altinity.ice.rest.catalog.internal.aws;
11+
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
import org.apache.iceberg.catalog.Namespace;
15+
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
16+
import software.amazon.s3tables.iceberg.S3TablesCatalog;
17+
18+
public class CustomS3TablesCatalog extends S3TablesCatalog {
19+
20+
@Override
21+
public List<Namespace> listNamespaces(Namespace namespace) throws NoSuchNamespaceException {
22+
if (!namespace.isEmpty()) {
23+
// Do not fail if client sends GET ?parent=$ns request (even if S3 Table buckets don't support
24+
// nested namespaces).
25+
return new ArrayList<>();
26+
}
27+
return super.listNamespaces(namespace);
28+
}
29+
}

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.altinity.ice.internal.iceberg.io.LocalFileIO;
1313
import com.altinity.ice.internal.iceberg.io.SchemeFileIO;
1414
import com.altinity.ice.internal.strings.Strings;
15+
import com.altinity.ice.rest.catalog.internal.aws.CustomS3TablesCatalog;
1516
import com.altinity.ice.rest.catalog.internal.etcd.EtcdCatalog;
1617
import com.fasterxml.jackson.annotation.JsonCreator;
1718
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -30,6 +31,7 @@
3031
import java.util.stream.Collectors;
3132
import org.apache.iceberg.CatalogProperties;
3233
import org.apache.iceberg.aws.AwsClientProperties;
34+
import org.apache.iceberg.aws.AwsProperties;
3335
import org.apache.iceberg.aws.s3.S3FileIOProperties;
3436
import org.apache.iceberg.jdbc.JdbcCatalog;
3537
import org.apache.iceberg.relocated.com.google.common.io.Files;
@@ -174,8 +176,8 @@ public static Config load(String configFile) throws IOException {
174176

175177
public Map<String, String> toIcebergLoadTableConfig() {
176178
var m = new HashMap<String, String>();
179+
String iceIODefault = "ice.io.default.";
177180
if (s3 != null) {
178-
String iceIODefault = "ice.io.default.";
179181
if (!Strings.isNullOrEmpty(s3.endpoint)) {
180182
m.put(iceIODefault + S3FileIOProperties.ENDPOINT, s3.endpoint);
181183
}
@@ -186,6 +188,10 @@ public Map<String, String> toIcebergLoadTableConfig() {
186188
m.put(iceIODefault + AwsClientProperties.CLIENT_REGION, s3.region);
187189
}
188190
}
191+
if (warehouse.startsWith("arn:aws:s3tables:")) {
192+
String region = warehouse.split(":")[3];
193+
m.putIfAbsent(iceIODefault + AwsClientProperties.CLIENT_REGION, region);
194+
}
189195
for (Map.Entry<String, String> e : loadTableProperties.entrySet()) {
190196
if (e.getValue() != null) {
191197
m.put(e.getKey(), e.getValue());
@@ -258,7 +264,18 @@ public void putNotNullOrEmpty(String key, String value) {
258264
m.putIfAbsent(CatalogProperties.CATALOG_IMPL, EtcdCatalog.class.getName());
259265
}
260266

261-
if (m.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "").startsWith("file://")) {
267+
String warehouse = m.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "");
268+
269+
if (warehouse.startsWith("arn:aws:s3tables:")) {
270+
m.putIfAbsent(CatalogProperties.CATALOG_IMPL, CustomS3TablesCatalog.class.getName());
271+
m.putIfAbsent("rest.sigv4-enabled", "true");
272+
m.putIfAbsent(AwsProperties.REST_SIGNING_NAME, "s3tables");
273+
String region = warehouse.split(":")[3];
274+
m.putIfAbsent(AwsProperties.REST_SIGNER_REGION, region);
275+
m.putIfAbsent(AwsClientProperties.CLIENT_REGION, region);
276+
}
277+
278+
if (warehouse.startsWith("file://")) {
262279
if (!m.containsKey(LocalFileIO.LOCALFILEIO_PROP_BASEDIR)) {
263280
// FIXME: wrong thing to do if warehouse is absolute
264281
m.put(LocalFileIO.LOCALFILEIO_PROP_BASEDIR, new File(".").getAbsolutePath());

0 commit comments

Comments
 (0)