Skip to content

Commit 6e47271

Browse files
shyikoshyiko
committed
Add 0.9.0 release changes
1 parent 58b6b6e commit 6e47271

File tree

19 files changed

+1489
-150
lines changed

19 files changed

+1489
-150
lines changed

‎CHANGELOG.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased](https://github.com/altinity/ice/compare/v0.8.1...master)
99

10+
### Added
11+
- `ice alter-table` command.
12+
- `ice insert --force-duplicates` CLI flag.
13+
- `ice describe` output to include nested metrics.
14+
15+
### Changed
16+
- `ice insert` to accept files with compatible schema: subset of table schema and/or with primitive type promotions listed in
17+
[Schema Evolution](https://iceberg.apache.org/spec/#schema-evolution) (previously file schema had to match exactly).
18+
19+
### Fixed
20+
- Calculation of nested metrics.
21+
1022
## [0.8.1](https://github.com/altinity/ice/compare/v0.8.0...v0.8.1)
1123

1224
### Fixed

‎examples/schema-evolution/.ice-rest-catalog.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
uri: jdbc:sqlite:file:data/ice-rest-catalog/catalog.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500
2+
warehouse: file://warehouse
3+
localFileIOBaseDir: data/ice-rest-catalog
4+
anonymousAccess:
5+
enabled: true
6+
accessConfig:
7+
readOnly: false

‎examples/schema-evolution/.ice.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
uri: http://localhost:5000
2+
localFileIOBaseDir: data/ice-rest-catalog
3+
httpCacheDir: data/ice/http/cache

‎examples/schema-evolution/README.md‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# examples/schema-evolution
2+
3+
1. Generate parquet files with slightly different schema
4+
(feel free to skip this step if you don't intend to modify included t.v1.parquet & t.v2.parquet)
5+
6+
```shell
7+
# optional: open shell containing `clickhouse`
8+
devbox shell
9+
10+
clickhouse local -q $"
11+
CREATE TABLE t (id Int32, toq String, il Int32, d Int32) ORDER BY id;
12+
INSERT INTO t VALUES
13+
(1, 'foo1', 4, 7),
14+
(2, 'foo2', 5, 8),
15+
(3, 'foo3', 6, 9);
16+
17+
SELECT * FROM t INTO OUTFILE 't.v1.parquet' TRUNCATE FORMAT Parquet;
18+
19+
ALTER TABLE t RENAME COLUMN toq TO q;
20+
ALTER TABLE t ALTER COLUMN il TYPE Int64;
21+
ALTER TABLE t DROP COLUMN d;
22+
ALTER TABLE t ADD COLUMN a Nullable(String);
23+
24+
SELECT * FROM t INTO OUTFILE 't.v2.parquet' TRUNCATE FORMAT Parquet;
25+
26+
SELECT * FROM 't.v1.parquet';
27+
SELECT * FROM 't.v2.parquet';
28+
"
29+
30+
2. Start Iceberg REST Catalog
31+
32+
```
33+
ice-rest-catalog
34+
```
35+
36+
3. Create Iceberg table from `t.v1.parquet`
37+
38+
```shell
39+
ice insert examples.schema_evolution -p file://t.v1.parquet
40+
ice describe -sp examples.schema_evolution
41+
ice scan examples.schema_evolution
42+
```
43+
44+
4. Update table schema to match `t.v2.parquet`.
45+
46+
```shell
47+
ice alter-table examples.schema_evolution $'[
48+
{"op":"rename_column","name":"toq","new_name":"q"},
49+
{"op":"alter_column","name":"il","type":"long"},
50+
{"op":"drop_column","name":"d"},
51+
{"op":"add_column","name":"a","type":"string"}
52+
]'
53+
ice describe -sp examples.schema_evolution
54+
ice scan examples.schema_evolution
55+
````
56+
57+
5. Append `t.v2.parquet` to the table. Confirm data can be read.
58+
59+
```shell
60+
ice insert examples.schema_evolution -p file://t.v2.parquet
61+
ice scan examples.schema_evolution
62+
```

‎examples/schema-evolution/devbox.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.10.7/.schema/devbox.schema.json",
3+
"env": {
4+
"AT": "ice:examples/schema-evolution"
5+
},
6+
"packages": {
7+
"sqlite": "latest",
8+
"jdk-headless": "21.0.7+6"
9+
},
10+
"shell": {
11+
"init_hook": [
12+
"export PATH=$(pwd):$(pwd)/.devbox/bin:$PATH",
13+
"[ -f .devbox/bin/clickhouse ] || (curl https://clickhouse.com/ | sh && mv clickhouse .devbox/bin/)"
14+
]
15+
}
16+
}

‎examples/schema-evolution/devbox.lock‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
{
2+
"lockfile_version": "1",
3+
"packages": {
4+
"github:NixOS/nixpkgs/nixpkgs-unstable": {
5+
"resolved": "github:NixOS/nixpkgs/e4b09e47ace7d87de083786b404bf232eb6c89d8?lastModified=1748856973&narHash=sha256-RlTsJUvvr8ErjPBsiwrGbbHYW8XbB%2Foek0Gi78XdWKg%3D"
6+
},
7+
"jdk-headless@21.0.7+6": {
8+
"last_modified": "2025-05-16T20:19:48Z",
9+
"resolved": "github:NixOS/nixpkgs/12a55407652e04dcf2309436eb06fef0d3713ef3#jdk21_headless",
10+
"source": "devbox-search",
11+
"version": "21.0.7+6",
12+
"systems": {
13+
"aarch64-linux": {
14+
"outputs": [
15+
{
16+
"name": "out",
17+
"path": "/nix/store/r4isdsjpz1i2zi93y6cjl53m0ngvbww3-openjdk-headless-21.0.7+6",
18+
"default": true
19+
},
20+
{
21+
"name": "debug",
22+
"path": "/nix/store/s7h859cnwfbr0ss7dlssphbhhivdc63p-openjdk-headless-21.0.7+6-debug"
23+
}
24+
],
25+
"store_path": "/nix/store/r4isdsjpz1i2zi93y6cjl53m0ngvbww3-openjdk-headless-21.0.7+6"
26+
},
27+
"x86_64-linux": {
28+
"outputs": [
29+
{
30+
"name": "out",
31+
"path": "/nix/store/jfxsvqhkxza9vrmpjw2nfxyz32883q1b-openjdk-headless-21.0.7+6",
32+
"default": true
33+
},
34+
{
35+
"name": "debug",
36+
"path": "/nix/store/ffknhap8n6nh733c33z714a9whmqz392-openjdk-headless-21.0.7+6-debug"
37+
}
38+
],
39+
"store_path": "/nix/store/jfxsvqhkxza9vrmpjw2nfxyz32883q1b-openjdk-headless-21.0.7+6"
40+
}
41+
}
42+
},
43+
"sqlite@latest": {
44+
"last_modified": "2025-05-29T08:12:58Z",
45+
"resolved": "github:NixOS/nixpkgs/3866ad91cfc172f08a6839def503d8fc2923c603#sqlite",
46+
"source": "devbox-search",
47+
"version": "3.48.0",
48+
"systems": {
49+
"aarch64-darwin": {
50+
"outputs": [
51+
{
52+
"name": "bin",
53+
"path": "/nix/store/cq20rd1ci51s72zv7k29h63clrj6rl0l-sqlite-3.48.0-bin",
54+
"default": true
55+
},
56+
{
57+
"name": "man",
58+
"path": "/nix/store/sal36w50bf6hlv076n4bypsyk51rn3ki-sqlite-3.48.0-man",
59+
"default": true
60+
},
61+
{
62+
"name": "dev",
63+
"path": "/nix/store/i30kh3xb6miyckm5nq554lmwfgciyxb7-sqlite-3.48.0-dev"
64+
},
65+
{
66+
"name": "doc",
67+
"path": "/nix/store/hxvmkk8icrgshsdg39qlsn9bia3dz41z-sqlite-3.48.0-doc"
68+
},
69+
{
70+
"name": "out",
71+
"path": "/nix/store/3q2bsgj3rpj8xksh6zgscwv3g1mzcjxy-sqlite-3.48.0"
72+
}
73+
],
74+
"store_path": "/nix/store/cq20rd1ci51s72zv7k29h63clrj6rl0l-sqlite-3.48.0-bin"
75+
},
76+
"aarch64-linux": {
77+
"outputs": [
78+
{
79+
"name": "bin",
80+
"path": "/nix/store/j9gsxv41ravm46zxkq6bz0kih7vikf2q-sqlite-3.48.0-bin",
81+
"default": true
82+
},
83+
{
84+
"name": "man",
85+
"path": "/nix/store/53xmz12n4qch61r792wvlm8aaw88r2mv-sqlite-3.48.0-man",
86+
"default": true
87+
},
88+
{
89+
"name": "debug",
90+
"path": "/nix/store/cchzb2prmi3b93r0n252nj9kmi8sq15h-sqlite-3.48.0-debug"
91+
},
92+
{
93+
"name": "dev",
94+
"path": "/nix/store/qq5r0v7w2knd099ih1yi705sgqb6xrq4-sqlite-3.48.0-dev"
95+
},
96+
{
97+
"name": "doc",
98+
"path": "/nix/store/dy5bgi74fdcyrvsqljcm71ys8vln6jwq-sqlite-3.48.0-doc"
99+
},
100+
{
101+
"name": "out",
102+
"path": "/nix/store/nzzrpcj9r48kq73fwibjxbwpnykrj5rf-sqlite-3.48.0"
103+
}
104+
],
105+
"store_path": "/nix/store/j9gsxv41ravm46zxkq6bz0kih7vikf2q-sqlite-3.48.0-bin"
106+
},
107+
"x86_64-darwin": {
108+
"outputs": [
109+
{
110+
"name": "bin",
111+
"path": "/nix/store/whxzqmryz1s807f9difm7sbky86p0mpl-sqlite-3.48.0-bin",
112+
"default": true
113+
},
114+
{
115+
"name": "man",
116+
"path": "/nix/store/c1q9an2k734218s47hsb3w7ppa5pdpgz-sqlite-3.48.0-man",
117+
"default": true
118+
},
119+
{
120+
"name": "out",
121+
"path": "/nix/store/9wjh0cydjvncz80x3l3flw67rvq6jwdq-sqlite-3.48.0"
122+
},
123+
{
124+
"name": "dev",
125+
"path": "/nix/store/2p883ilw7k58sq8hwcnl519di738wxp0-sqlite-3.48.0-dev"
126+
},
127+
{
128+
"name": "doc",
129+
"path": "/nix/store/dz22l0c3h2gxvzx45vkwcwd3sr0mvjs5-sqlite-3.48.0-doc"
130+
}
131+
],
132+
"store_path": "/nix/store/whxzqmryz1s807f9difm7sbky86p0mpl-sqlite-3.48.0-bin"
133+
},
134+
"x86_64-linux": {
135+
"outputs": [
136+
{
137+
"name": "bin",
138+
"path": "/nix/store/b83kagl3d98zf8dbvh52lw4xg881bhkf-sqlite-3.48.0-bin",
139+
"default": true
140+
},
141+
{
142+
"name": "man",
143+
"path": "/nix/store/xrjy0a756pb41qq9a38lqvy2m7jd78p5-sqlite-3.48.0-man",
144+
"default": true
145+
},
146+
{
147+
"name": "out",
148+
"path": "/nix/store/yfjzkkkyxcalyj7l1n4d4y6s81i65hmy-sqlite-3.48.0"
149+
},
150+
{
151+
"name": "debug",
152+
"path": "/nix/store/719sjqlrhmrn02079dam88m0ch4gwm7h-sqlite-3.48.0-debug"
153+
},
154+
{
155+
"name": "dev",
156+
"path": "/nix/store/92gwa4j45skp8d096csmnj2a8jcn0q9w-sqlite-3.48.0-dev"
157+
},
158+
{
159+
"name": "doc",
160+
"path": "/nix/store/a0h09d196h7j564mrcy15bldd95blw7j-sqlite-3.48.0-doc"
161+
}
162+
],
163+
"store_path": "/nix/store/b83kagl3d98zf8dbvh52lw4xg881bhkf-sqlite-3.48.0-bin"
164+
}
165+
}
166+
}
167+
}
168+
}

‎examples/schema-evolution/t.v1.parquet‎

840 Bytes
Binary file not shown.

‎examples/schema-evolution/t.v2.parquet‎

743 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)