feat: fully working demo lake

Author: MattOates

README.md

@@ -79,14 +79,61 @@ flowchart TD
 
 4. **Verify the lakehouse**
 
+Once you have a client up via 3 all setup should be automatic from the Docker container. Doing something like the following should get you some gene data loaded into your lake.
+
    ```sql
-   INSTALL ducklake;
-   INSTALL postgres;
-   ATTACH 'ducklake:postgres:dbname=ducklake_catalog host=postgres user=ducklake password=ducklake' AS the_ducklake (DATA_PATH 's3://ducklake/lake/');
-   USE the_ducklake;
-   SELECT * FROM ducklake.schema;
+   CREATE OR REPLACE TABLE gene AS
+    SELECT *
+    FROM read_csv_auto(
+    'https://storage.googleapis.com/public-download-files/hgnc/tsv/tsv/non_alt_loci_set.txt',
+    HEADER      => TRUE,
+    DELIM       => '\t',
+    SAMPLE_SIZE => 100000
+   );
    ```
 
+Then `select * from gene limit 10;`
+```
+┌────────────┬─────────┬──────────────────────┬───┬──────────────────────┬───────────┐
+│  hgnc_id   │ symbol  │         name         │ … │     mane_select      │   gencc   │
+│  varchar   │ varchar │       varchar        │   │       varchar        │  varchar  │
+├────────────┼─────────┼──────────────────────┼───┼──────────────────────┼───────────┤
+│ HGNC:2973  │ DNM1L   │ dynamin 1 like       │ … │ ENST00000549701.6|…  │ HGNC:2973 │
+│ HGNC:21122 │ DNM1P5  │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:21126 │ DNM1P9  │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:21134 │ DNM1P17 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:21135 │ DNM1P18 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:21136 │ DNM1P19 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:35171 │ DNM1P24 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:35172 │ DNM1P25 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:35173 │ DNM1P26 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+│ HGNC:35174 │ DNM1P27 │ dynamin 1 pseudoge…  │ … │ NULL                 │ NULL      │
+├────────────┴─────────┴──────────────────────┴───┴──────────────────────┴───────────┤
+│ 10 rows                                                       54 columns (5 shown) │
+└────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+With the above you should see some data landing in ./data of the repository in both minio and pgdata.
+
+You should already be dropped straight into a configured DuckLake using mino and pg against your local disk. But if you want to attach from the 
+outside rather than from the container helper you will need to do the following in your duckdb client/.duckdbrc:
+
+```sql
+INSTALL ducklake;
+INSTALL postgres;
+   
+SET s3_url_style           = 'path';
+SET s3_endpoint            = 'minio:9000';
+SET s3_access_key_id       = 'minioadmin';
+SET s3_secret_access_key   = 'minioadmin';
+SET s3_region              = 'us-east-1';
+SET s3_use_ssl             = false;
+
+-- Auto-attach your lakehouse
+ATTACH 'ducklake:postgres:dbname=ducklake_catalog host=postgres user=ducklake password=ducklake'
+AS the_ducklake  (DATA_PATH 's3://ducklake/lake/');
+```
+
 ## Configuration
 
 All credentials and endpoints are controlled via environment variables in `docker-compose.yml`:

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       POSTGRES_PASSWORD: ducklake
       POSTGRES_DB: ducklake_catalog
     volumes:
-      - pgdata:/var/lib/postgresql/data
+      - ./data/pgdata:/var/lib/postgresql/data
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ducklake"]
       interval: 5s
@@ -24,7 +24,7 @@ services:
     ports:
       - "9000:9000"
     volumes:
-      - miniodata:/data
+      - ./data/minio:/data
     healthcheck:
       test: ["CMD", "curl", "-I", "http://localhost:9000/minio/health/live"]
       interval: 5s

ducklake-init/Dockerfile

@@ -11,6 +11,11 @@ RUN apt-get update \
 
 # Install the latest duckdb CLI client
 RUN curl https://install.duckdb.org | sh
+ENV PATH="/root/.duckdb/cli/latest:$PATH"
+CMD duckdb --version
+
+# Copy DuckDB CLI init file so every duckdb session picks up MinIO settings
+COPY duckdbrc.tpl /duckdbrc.tpl
 
 COPY entrypoint.py /entrypoint.py
 RUN chmod +x /entrypoint.py

ducklake-init/duckdbrc.tpl

@@ -0,0 +1,14 @@
+-- ducklake-init/duckdbrc.tpl
+
+-- DuckDB + MinIO HTTPFS config
+SET s3_url_style           = 'path';
+SET s3_endpoint            = 'minio:9000';
+SET s3_access_key_id       = '$AWS_ACCESS_KEY_ID';
+SET s3_secret_access_key   = '$AWS_SECRET_ACCESS_KEY';
+SET s3_region              = '$AWS_REGION';
+SET s3_use_ssl             = false;
+
+-- Auto-attach your lakehouse
+ATTACH 'ducklake:postgres:dbname=$POSTGRES_DB host=postgres user=$POSTGRES_USER password=$POSTGRES_PASSWORD'
+AS the_ducklake  (DATA_PATH 's3://$BUCKET/lake/');
+USE the_ducklake;

ducklake-init/entrypoint.py

@@ -11,6 +11,7 @@
 # ///
 
 import os
+from string import Template
 import time
 
 import duckdb
@@ -20,6 +21,22 @@
 import psycopg
 import requests
 
+
+def dump_duckdbrc():
+    tpl = Template(open("/duckdbrc.tpl").read())
+    content = tpl.substitute(
+        AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"],
+        AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"],
+        AWS_REGION=os.environ.get("AWS_REGION", "us-east-1"),
+        POSTGRES_DB=os.environ["POSTGRES_DB"],
+        POSTGRES_USER=os.environ["POSTGRES_USER"],
+        POSTGRES_PASSWORD=os.environ["POSTGRES_PASSWORD"],
+        BUCKET=os.environ["BUCKET"],
+    )
+    with open("/root/.duckdbrc", "w") as f:
+        f.write(content)
+
+
 def wait_for_postgres(host="postgres", user=None, password=None, db=None, timeout=60):
     deadline = time.time() + timeout
     while time.time() < deadline:
@@ -56,7 +73,11 @@ def main():
     aws_ep     = os.environ["AWS_ENDPOINT_URL"]
     bucket     = os.environ["BUCKET"]
 
-    # Wait for dependencies
+    # 1) Render ~/.duckdbrc so both CLI and Python API sessions will
+    #    pick up the HTTPFS settings + auto-ATTACH.
+    dump_duckdbrc()
+
+    # 2) Wait on Postgres & MinIO, ensure bucket, then bootstrap/attach via Python API…
     wait_for_postgres(user=pg_user, password=pg_pass, db=pg_db)
     wait_for_minio(aws_ep)
 
@@ -76,17 +97,31 @@ def main():
 
     # Initialize or attach DuckLake
     con = duckdb.connect()
+
+    # 1) install extensions
     con.execute("INSTALL ducklake;")
     con.execute("INSTALL postgres;")
 
+    # 2) configure HTTPFS for MinIO
+    for key, val in {
+        "s3_url_style":    "path",
+        "s3_endpoint":     "minio:9000",
+        "s3_access_key_id":     os.environ["AWS_ACCESS_KEY_ID"],
+        "s3_secret_access_key": os.environ["AWS_SECRET_ACCESS_KEY"],
+        "s3_region":            os.environ.get("AWS_REGION", "us-east-1"),
+        "s3_use_ssl":           "false",
+    }.items():
+        con.execute(f"SET {key}='{val}';")
+
+    # 3) now attach/initialize DuckLake
     attach_sql = f"""
     ATTACH 'ducklake:postgres:dbname={pg_db} host=postgres user={pg_user} password={pg_pass}'
     AS the_ducklake (DATA_PATH 's3://{bucket}/lake/');
     """
     con.execute(attach_sql)
     con.execute("USE the_ducklake;")
 
-    # Keep the container alive
+    # 4)Keep the container alive
     print("DuckLake init complete; container is now running.")
     while True:
         time.sleep(3600)