Implement postgresql metadata provider

Author: Shefeek Jinnah

Cargo.toml

@@ -26,14 +26,22 @@ futures = "0.3"
 tracing = "0.1"
 thiserror = "2.0"
 
+# Multi-database metadata providers (optional)
+sqlx = { version = "0.8", features = ["runtime-tokio"], optional = true }
+
 [dev-dependencies]
 testcontainers = "0.23"
-testcontainers-modules = { version = "0.11", features = ["minio"] }
+testcontainers-modules = { version = "0.11", features = ["minio", "postgres", "mysql"] }
 tempfile = "3.14"
 anyhow = "1.0"
 aws-sdk-s3 = "1.75"
 aws-credential-types = "1.2"
 
 [features]
 # Allow skipping tests that use docker (in CI, macos doesn't support docker).
-skip-tests-with-docker = []
+skip-tests-with-docker = []
+
+# Metadata provider backends
+metadata-postgres = ["sqlx", "sqlx/postgres", "sqlx/chrono"]
+# Future: metadata-sqlite = ["sqlx", "sqlx/sqlite", "sqlx/chrono"]
+# Future: metadata-mysql = ["sqlx", "sqlx/mysql", "sqlx/chrono"]

examples/postgres_catalog.rs

@@ -0,0 +1,71 @@
+//! PostgreSQL catalog example
+//!
+//! ## Usage
+//!
+//! Default (connects to localhost:5432):
+//! ```bash
+//! cargo run --example postgres_catalog --features metadata-postgres
+//! ```
+//!
+//! Custom PostgreSQL connection:
+//! ```bash
+//! POSTGRES_URL="postgresql://user:pass@host:5432/dbname" \
+//!   cargo run --example postgres_catalog --features metadata-postgres
+//! ```
+
+use datafusion::prelude::*;
+use datafusion_ducklake::{DuckLakeCatalog, PostgresMetadataProvider};
+use std::sync::Arc;
+
+#[cfg(feature = "metadata-postgres")]
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    use std::env;
+
+    println!("==> PostgreSQL catalog example\n");
+
+    let conn_str = env::var("POSTGRES_URL").unwrap_or_else(|_| {
+        "postgresql://postgres:postgres@localhost:5432/ducklake_catalog".to_string()
+    });
+    println!("Connecting to: {}", conn_str);
+
+    let provider = PostgresMetadataProvider::new(&conn_str).await?;
+    println!("✓ Connected\n");
+
+    let catalog = DuckLakeCatalog::new(provider)?;
+    let ctx = SessionContext::new();
+    ctx.register_catalog("ducklake", Arc::new(catalog));
+
+    println!("==> Schemas:");
+    ctx.sql("SELECT * FROM ducklake.information_schema.schemata")
+        .await?
+        .show()
+        .await?;
+
+    println!("\n==> Tables:");
+    ctx.sql("SELECT * FROM ducklake.information_schema.tables")
+        .await?
+        .show()
+        .await?;
+
+    println!("\n==> Snapshots:");
+    ctx.sql("SELECT * FROM ducklake.information_schema.snapshots")
+        .await?
+        .show()
+        .await?;
+
+    println!("\n✓ Example completed successfully");
+    println!("\nNext steps:");
+    println!("  1. Populate the catalog with schemas, tables, and data files");
+    println!("  2. Query your DuckLake tables using SQL");
+    println!("  3. Enjoy PostgreSQL metadata catalog support!");
+
+    Ok(())
+}
+
+#[cfg(not(feature = "metadata-postgres"))]
+fn main() {
+    eprintln!("Error: This example requires the 'metadata-postgres' feature");
+    eprintln!("Run with: cargo run --example postgres_catalog --features metadata-postgres");
+    std::process::exit(1);
+}

src/error.rs

@@ -17,6 +17,11 @@ pub enum DuckLakeError {
     #[error("DuckDB error: {0}")]
     DuckDb(#[from] duckdb::Error),
 
+    /// sqlx database error (for PostgreSQL metadata provider)
+    #[cfg(feature = "metadata-postgres")]
+    #[error("Database error: {0}")]
+    Sqlx(#[from] sqlx::Error),
+
     /// Catalog not found
     #[error("Catalog not found: {0}")]
     CatalogNotFound(String),

src/lib.rs

@@ -47,6 +47,10 @@ pub mod table;
 pub mod table_functions;
 pub mod types;
 
+// Multi-database metadata providers (optional, feature-gated)
+#[cfg(feature = "metadata-postgres")]
+pub mod metadata_provider_postgres;
+
 // Result type for DuckLake operations
 pub type Result<T> = std::result::Result<T, DuckLakeError>;
 
@@ -58,3 +62,7 @@ pub use metadata_provider_duckdb::DuckdbMetadataProvider;
 pub use schema::DuckLakeSchema;
 pub use table::DuckLakeTable;
 pub use table_functions::register_ducklake_functions;
+
+// Re-export multi-database metadata providers (feature-gated)
+#[cfg(feature = "metadata-postgres")]
+pub use metadata_provider_postgres::PostgresMetadataProvider;

src/metadata_provider.rs

@@ -324,3 +324,12 @@ pub trait MetadataProvider: Send + Sync + std::fmt::Debug {
     /// List all files across all tables for a snapshot
     fn list_all_files(&self, snapshot_id: i64) -> Result<Vec<FileWithTable>>;
 }
+
+#[cfg(feature = "metadata-postgres")]
+/// Helper function to bridge async sqlx operations to sync MetadataProvider trait
+pub(crate) fn block_on<F, T>(f: F) -> T
+where
+    F: std::future::Future<Output = T>,
+{
+    tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(f))
+}