Draft etcd, localfileio, typed config

Author: shyiko

.bin/docker-build-ice

@@ -40,7 +40,7 @@ for q in "" "debug-"; do
   docker manifest create "$image_name:$q$IMAGE_TAG" \
     --amend "$image_name:$q$IMAGE_TAG-amd64" \
     --amend "$image_name:$q$IMAGE_TAG-arm64"
-  docker manifest push "$image_name:$q$IMAGE_TAG"
+  docker manifest push --purge "$image_name:$q$IMAGE_TAG"
 done
 
 regctl image copy "$image_name:$IMAGE_TAG" "$image_name:latest"

.bin/docker-build-ice-rest-catalog

@@ -43,7 +43,7 @@ for q in "" "debug-" "debug-with-ice-"; do
   docker manifest create "$image_name:$q$IMAGE_TAG" \
     --amend "$image_name:$q$IMAGE_TAG-amd64" \
     --amend "$image_name:$q$IMAGE_TAG-arm64"
-  docker manifest push "$image_name:$q$IMAGE_TAG"
+  docker manifest push --purge "$image_name:$q$IMAGE_TAG"
 done
 
 regctl image copy "$image_name:$IMAGE_TAG" "$image_name:latest"

.bin/local-ice

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
 DIR="$(dirname "${BASH_SOURCE[0]}")/.."
-exec "$DIR/mvnw" -f "$DIR/pom.xml" -q -Pno-check -pl ice compile exec:java -Dexec.mainClass=com.altinity.ice.Main -Dexec.args="$*" # FIXME: $*
+exec "$DIR/mvnw" -f "$DIR/pom.xml" -q -Pno-check -am -pl ice compile exec:java -Dexec.mainClass=com.altinity.ice.cli.Main -Dexec.args="$*" # FIXME: $*

.bin/local-ice-rest-catalog

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
 DIR="$(dirname "${BASH_SOURCE[0]}")/.."
-exec "$DIR/mvnw" -f "$DIR/pom.xml" -q -Pno-check -pl ice-rest-catalog compile exec:java -Dexec.mainClass=com.altinity.ice.rest.catalog.Main -Dexec.args="$*" # FIXME: $*
+exec "$DIR/mvnw" -f "$DIR/pom.xml" -q -P ice-rest-catalog-exec -Pno-check -am -pl ice-rest-catalog compile exec:java -Dexec.mainClass=com.altinity.ice.rest.catalog.Main -Dexec.args="$*" # FIXME: $*

.bin/local-pyiceberg-insert-ns1-table1-dataset

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
-cd "$(dirname "${BASH_SOURCE[0]}")/../ice-rest-catalog/pyiceberg-test"
+cd "$(dirname "${BASH_SOURCE[0]}")/../ice-rest-catalog/src/test/pyiceberg"
 
 uv run ice_create_table.py ns1.table1 --location=s3://bucket1/ns1/table1 --schema-from-parquet \
   https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet

.bin/local-pyiceberg-scan

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 set -euo pipefail
-cd "$(dirname "${BASH_SOURCE[0]}")/../ice-rest-catalog/pyiceberg-test"
+cd "$(dirname "${BASH_SOURCE[0]}")/../ice-rest-catalog/src/test/pyiceberg"
 
 uv run ice_scan.py "$@"

README.md

@@ -2,8 +2,8 @@
 
 A suite of tools aimed at making [Iceberg](https://iceberg.apache.org/) REST Catalogs more approachable. 
 
-- [ice-rest-catalog](ice-rest-catalog/) - A Kubernetes-ready Iceberg REST catalog backed by [etcd](https://etcd.io/) (WIP).  
-Starts with a one-liner: `ice-rest-catalog -c config.yaml`.
+- [ice-rest-catalog](ice-rest-catalog/) - A Kubernetes-ready Iceberg REST catalog backed by [etcd](https://etcd.io/).  
+Run locally with a one-liner: `ice-rest-catalog -c config.yaml`.
 
 - [ice](ice/) - A CLI for interacting with Iceberg REST catalogs.  
 Create/delete tables, insert data with `ice insert -p ns1.table1 file://example.parquet`, etc. 

examples/docker-compose/.ice.yaml

@@ -1,7 +1,4 @@
-catalog:
-  default:
-    uri: http://localhost:5000
-    header.authorization: bearer foo
-    s3.endpoint: http://localhost:8999
-    client.region: mino
-    ice.http.cache: var/cache/ice/http
+uri: http://localhost:5000
+bearerToken: foo
+s3:
+  endpoint: http://localhost:8999

examples/docker-compose/README.md

@@ -11,11 +11,17 @@ In the example below, we:
 # feel free to skip it if you have clickhouse client installed already
 devbox shell
 
-docker compose down -v && sudo rm -rf docker-compose.volumes/
-docker compose up # spins up ice-rest-catalog + clickhouse
+docker compose down -v && sudo rm -rf data/
+docker compose up # spin up minio, ice-rest-catalog & clickhouse
 
 ice insert nyc.taxis -p \
   https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet
 
 clickhouse client --query 'select count(*) from ice.`nyc.taxis`;'
 ```
+
+### Troubleshooting
+
+1. `docker compose up` fails with `ERROR: Invalid interpolation format for "content" option in config "clickhouse-init": "#!/bin/bash`
+
+Solution: Upgrade docker/docker compose to v2.

examples/docker-compose/docker-compose.yaml

@@ -26,7 +26,7 @@ services:
     depends_on:
       - minio
   ice-rest-catalog:
-    image: altinity/ice-rest-catalog:latest
+    image: altinity/ice-rest-catalog:${ICE_REST_CATALOG_TAG:-latest}
     restart: unless-stopped
     ports:
       - '5000:5000' # iceberg/http
@@ -35,7 +35,7 @@ services:
         target: /etc/ice/ice-rest-catalog.yaml
     volumes:
       # for access to /var/lib/ice-rest-catalog/db.sqlite
-      - ./docker-compose.volumes/ice-rest-catalog/var/lib/ice-rest-catalog:/var/lib/ice-rest-catalog
+      - ./data/docker-compose/ice-rest-catalog/var/lib/ice-rest-catalog:/var/lib/ice-rest-catalog
     depends_on:
       - minio-init
   clickhouse:
@@ -51,7 +51,7 @@ services:
         target: /docker-entrypoint-initdb.d/init-db.sh
     volumes:
       # for access to clickhouse-logs
-      - ./docker-compose.volumes/clickhouse/var/log/clickhouse-server:/var/log/clickhouse-server
+      - ./data/docker-compose/clickhouse/var/log/clickhouse-server:/var/log/clickhouse-server
     depends_on:
       - ice-rest-catalog
 configs:
@@ -68,16 +68,19 @@ configs:
           SETTINGS catalog_type = 'rest',
             auth_header = 'Authorization: Bearer foo',
             storage_endpoint = 'http://minio:8999',
-            warehouse = 's3://bucket1/';
+            warehouse = 's3://bucket1';
       "
   ice-rest-catalog-yaml:
     content: |
-      uri: "jdbc:sqlite:file:/var/lib/ice-rest-catalog/db.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500"
+      uri: jdbc:sqlite:file:/var/lib/ice-rest-catalog/db.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500
       warehouse: s3://bucket1
-      s3.endpoint: http://minio:8999
-      s3.access-key-id: "miniouser"
-      s3.secret-access-key: "miniopassword"
-      ice.s3.region: minio
-      ice.token: foo
+      s3:
+        endpoint: http://minio:8999
+        pathStyleAccess: true
+        accessKeyID: miniouser
+        secretAccessKey: miniopassword
+        region: minio
+      bearerTokens:
+        - value: foo
 volumes:
   minio: