feat: add lazy loading of tables

Author: PengLiVectra

python/deltalake/table.py

@@ -229,6 +229,7 @@ def __init__(
         storage_options: Optional[Dict[str, str]] = None,
         without_files: bool = False,
         log_buffer_size: Optional[int] = None,
+        lazy_load: bool = False
     ):
         """
         Create the Delta Table from a path with an optional version.
@@ -247,9 +248,19 @@ def __init__(
                                 This can decrease latency if there are many files in the log since the last checkpoint,
                                 but will also increase memory usage. Possible rate limits of the storage backend should
                                 also be considered for optimal performance. Defaults to 4 * number of cpus.
-
+            lazy_load: when true the table metadata isn't loaded
         """
         self._storage_options = storage_options
+        if lazy_load:
+            self._table = RawDeltaTable.load_lazy(
+                str(table_uri),
+                version=version,
+                storage_options=storage_options,
+                without_files=without_files,
+                log_buffer_size=log_buffer_size,
+            )
+            self._metadata = None
+            return
         self._table = RawDeltaTable(
             str(table_uri),
             version=version,
@@ -425,6 +436,8 @@ def metadata(self) -> Metadata:
         Returns:
             the current Metadata registered in the transaction log
         """
+        if not self._metadata:
+            self._metadata = Metadata(self._table)
         return self._metadata
 
     def protocol(self) -> ProtocolVersions:

python/src/lib.rs

@@ -121,6 +121,14 @@ impl RawDeltaTable {
         })
     }
 
+    #[classmethod]
+    fn load_lazy(_cls: &PyType, table_uri: &str) -> PyResult<Self> {
+        let be = storage::get_backend_for_uri(table_uri).unwrap();
+        let table = DeltaTable::new(table_uri, be, DeltaTableConfig::default())
+            .map_err(PyDeltaTableError::from_raw)?;
+        Ok(RawDeltaTable { _table: table })
+    }
+
     #[classmethod]
     #[pyo3(signature = (data_catalog, database_name, table_name, data_catalog_id, catalog_options = None))]
     fn get_table_uri_from_data_catalog(