Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 44 additions & 19 deletions extensions/onelake/description.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
extension:
name: onelake
description: This extension allows you to connect DuckDB to Microsoft Fabric OneLake workspaces and lakehouses, enabling you to query data stored in OneLake directly from DuckDB.
version: 1.0.2
version: 1.1.0
language: C++
build: cmake
license: MIT
maintainers:
- achrafcei
excluded_platforms: "windows_amd64_mingw"
requires_toolchains: "rust"
repo:
github: datumnova/duckdb_onelake
ref: 838c17038de3756b43afd6d2b4b1d04ea412dfe7
ref: 59c131bc6e81f2edc5d08f71a4cc521f1c51d39e
canonical_name: onelake

docs:
Expand Down Expand Up @@ -45,23 +46,14 @@ docs:
-- CHAIN 'cli'
-- );

-- Optional: use preissued tokens stored in env variables (defaults shown), if they are already available with the same name, no need to set them here (only update the SET commands if the names are different
-- Optional: use preissued tokens stored in env variables (defaults shown)
--SET onelake_env_fabric_token_variable = 'FABRIC_API_TOKEN';
--SET onelake_env_storage_token_variable = 'AZURE_STORAGE_TOKEN';
--CREATE SECRET onelake_env (
-- TYPE ONELAKE,
-- PROVIDER credential_chain,
-- CHAIN 'env'
--);
-- Combine chain steps if you want CLI fallback
--CREATE SECRET onelake_env_chain (
-- TYPE ONELAKE,
-- PROVIDER credential_chain,
-- CHAIN 'cli, env'
--);

-- Optionally keep the token in-session instead of touching the shell
--SET VARIABLE AZURE_STORAGE_TOKEN = '<preissued_onelake_access_token>';

-- Attach to your OneLake workspace and lakehouse
ATTACH '<your_workspace_name>/<your_lakehouse_name>.Lakehouse'
Expand All @@ -72,14 +64,47 @@ docs:

SHOW TABLES;

SELECT * FROM <your_table_name> LIMIT 10 ; -- USING ICEBERG;
-- Query tables (Delta or Iceberg)
SELECT * FROM <your_table_name> LIMIT 10;

-- Create a new Delta table
CREATE TABLE new_table (
id INT,
name VARCHAR
);

-- Create a new Delta table with partitioning
CREATE TABLE new_table_partitioned (
id INT,
name VARCHAR,
region VARCHAR
) PARTITION BY (region);

-- Write to Delta tables (INSERT)
INSERT INTO new_table VALUES (1, 'New Data');

-- SET write mode to overwrite (default is append)
SET onelake_delta_write_mode = 'overwrite';
INSERT INTO new_table VALUES (2, 'Overwritten Data');


-- Perform destructive operations (UPDATE, DELETE, DROP)
-- These require explicit opt-in per session
SET onelake_allow_destructive_operations = true;

UPDATE new_table
SET name = 'Updated Value'
WHERE id = 1;

DELETE FROM new_table WHERE id = 1;

DROP TABLE new_table;

extended_description: |
This extension enables DuckDB to connect to Microsoft Fabric OneLake workspaces and lakehouses, allowing users to query data stored in OneLake directly from DuckDB.
It supports authentication via service principals or credential chains (e.g., Azure CLI) and provides seamless integration with OneLake's data storage capabilities.
For detailed setup and usage instructions, visit the [extension repository](https://github.com/datumnova/duckdb_onelake).
This extension enables DuckDB to connect to Microsoft Fabric OneLake workspaces and lakehouses, allowing users to query and modify data stored in OneLake directly from DuckDB.
It supports authentication via service principals, credential chains (e.g., Azure CLI), or environment tokens and provides seamless integration with OneLake's data storage capabilities.

Current limitations:
- Only read access is supported; write operations are not implemented.
Key Features:
- Read support for both Delta and Iceberg tables.
- Full write support for Delta tables including CREATE TABLE, INSERT, UPDATE, DELETE, and DROP.
- Configurable write modes (append, overwrite) and schema evolution.
- ALTER TABLE support is planned for future releases.
9 changes: 9 additions & 0 deletions extensions/onelake/docs/functions_description.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"function" , "description" , "comment", "example"
"onelake_attach" , "Attach to a OneLake workspace and lakehouse." , "" , "ATTACH 'workspace/lakehouse.Lakehouse' AS my_lakehouse (TYPE ONELAKE);"
"onelake_scan" , "Scan a Delta table from the attached lakehouse." , "" , "SELECT * FROM my_lakehouse.my_schema.my_table;"
"onelake_iceberg_scan", "Scan an Iceberg table from the attached lakehouse using the ICEBERG syntax.", "" , "SELECT * FROM my_lakehouse.my_schema.my_table USING ICEBERG;"
"onelake_secret" , "Create a secret for OneLake authentication." , "" , "CREATE SECRET my_secret (TYPE ONELAKE, TENANT_ID '...', CLIENT_ID '...', CLIENT_SECRET '...');"
"onelake_secret_chain", "Create a secret using the credential chain provider." , "" , "CREATE SECRET my_chain (TYPE ONELAKE, PROVIDER credential_chain, CHAIN 'env');"
"onelake_write" , "Insert data into a OneLake table." , "" , "INSERT INTO my_lakehouse.my_schema.my_table SELECT * FROM source_table;"
"onelake_create_table", "Create a new Delta table in OneLake." , "" , "CREATE TABLE my_lakehouse.my_schema.new_table (id INT) PARTITION BY (id);"
"onelake_drop_table" , "Drop a table from OneLake." , "" , "DROP TABLE my_lakehouse.my_schema.my_table;"
Loading