ENH: Add basic support for data layer metadata IO (#237)

Author: brendan-ward

CHANGES.md

@@ -20,6 +20,9 @@
 -   Add access to low-level pyarrow `RecordBatchReader` via
     `pyogrio.raw.open_arrow`, which allows iterating over batches of Arrow
     tables (#205).
+-   Add support for writing dataset and layer metadata (where supported by
+    driver) to `write` and `write_dataframe`, and add support for reading
+    dataset and layer metadata in `read_info` (#237).
 
 ### Packaging
 

pyogrio/_io.pyx

@@ -116,13 +116,17 @@ cdef char** dict_to_options(object values):
     Parameters
     ----------
     values: dict
+        all keys and values must be strings
 
     Returns
     -------
     char**
     """
     cdef char **options = NULL
 
+    if values is None:
+        return NULL
+
     for k, v in values.items():
         k = k.encode('UTF-8')
         v = v.encode('UTF-8')
@@ -305,6 +309,54 @@ cdef get_driver(OGRDataSourceH ogr_dataset):
     return driver
 
 
+cdef set_metadata(GDALMajorObjectH obj, object metadata):
+    """Set metadata on a dataset or layer
+
+    Parameters
+    ----------
+    obj : pointer to dataset or layer
+    metadata : dict, optional (default None)
+        keys and values must be strings
+    """
+
+    cdef char **metadata_items = NULL
+    cdef int err = 0
+
+    metadata_items = dict_to_options(metadata)
+    if metadata_items != NULL:
+        # only default namepace is currently supported
+        err = GDALSetMetadata(obj, metadata_items, NULL)
+
+        CSLDestroy(metadata_items)
+        metadata_items = NULL
+
+    if err:
+        raise RuntimeError("Could not set metadata") from None
+
+cdef get_metadata(GDALMajorObjectH obj):
+    """Get metadata for a dataset or layer
+
+    Parameters
+    ----------
+    obj : pointer to dataset or layer
+
+    Returns
+    -------
+    dict or None
+        metadata as key, value pairs
+    """
+    # only default namespace is currently supported
+    cdef char **metadata = GDALGetMetadata(obj, NULL)
+
+    if metadata != NULL:
+        return dict(
+            metadata[i].decode('UTF-8').split('=', 1)
+            for i in range(CSLCount(metadata))
+        )
+
+    return None
+
+
 cdef detect_encoding(OGRDataSourceH ogr_dataset, OGRLayerH ogr_layer):
     """Attempt to detect the encoding of the layer.
     If it supports UTF-8, use that.
@@ -1274,7 +1326,9 @@ def ogr_read_info(
                 "random_read": OGR_L_TestCapability(ogr_layer, OLCRandomRead),
                 "fast_set_next_by_index": OGR_L_TestCapability(ogr_layer, OLCFastSetNextByIndex),
                 "fast_spatial_filter": OGR_L_TestCapability(ogr_layer, OLCFastSpatialFilter),
-            }
+            },
+            'layer_metadata': get_metadata(ogr_layer),
+            'dataset_metadata': get_metadata(ogr_dataset),
         }
 
     finally:
@@ -1419,7 +1473,7 @@ def ogr_write(
     str path, str layer, str driver, geometry, fields, field_data, field_mask,
     str crs, str geometry_type, str encoding, object dataset_kwargs,
     object layer_kwargs, bint promote_to_multi=False, bint nan_as_null=True,
-    bint append=False
+    bint append=False, dataset_metadata=None, layer_metadata=None
 ):
     cdef const char *path_c = NULL
     cdef const char *layer_c = NULL
@@ -1564,6 +1618,10 @@ def ogr_write(
         else:
             ogr_layer = exc_wrap_pointer(get_ogr_layer(ogr_dataset, layer))
 
+        # Set dataset and layer metadata
+        set_metadata(ogr_dataset, dataset_metadata)
+        set_metadata(ogr_layer, layer_metadata)
+
     except Exception as exc:
         OGRReleaseDataSource(ogr_dataset)
         ogr_dataset = NULL

pyogrio/_ogr.pxd

@@ -38,6 +38,7 @@ cdef extern from "cpl_string.h":
     char**      CSLSetNameValue(char **list, const char *name, const char *value)
     void        CSLDestroy(char **list)
     char**      CSLAddString(char **list, const char *string)
+    int         CSLCount(char **list)
 
 
 cdef extern from "cpl_vsi.h" nogil:
@@ -372,7 +373,9 @@ cdef extern from "gdal.h":
     OGRErr          GDALDatasetStartTransaction(GDALDatasetH ds, int bForce)
     OGRErr          GDALDatasetCommitTransaction(GDALDatasetH ds)
     OGRErr          GDALDatasetRollbackTransaction(GDALDatasetH ds)
+    char**          GDALGetMetadata(GDALMajorObjectH obj, const char *pszDomain)
     const char*     GDALGetMetadataItem(GDALMajorObjectH obj, const char *pszName, const char *pszDomain)
+    OGRErr          GDALSetMetadata(GDALMajorObjectH obj, char **metadata, const char *pszDomain)
     const char*     GDALVersionInfo(const char *pszRequest)
 
 

pyogrio/core.py

@@ -180,6 +180,8 @@ def read_info(path_or_buffer, /, layer=None, encoding=None, **kwargs):
                 "geometry": "<geometry type>",
                 "features": <feature count>,
                 "driver": "<driver>",
+                "dataset_metadata" "<dict of dataset metadata or None>"
+                "layer_metadata" "<dict of layer metadata or None>"
             }
     """
     path, buffer = get_vsi_path(path_or_buffer)

pyogrio/geopandas.py

@@ -204,6 +204,9 @@ def write_dataframe(
     promote_to_multi=None,
     nan_as_null=True,
     append=False,
+    dataset_metadata=None,
+    layer_metadata=None,
+    metadata=None,
     dataset_options=None,
     layer_options=None,
     **kwargs,
@@ -261,6 +264,16 @@ def write_dataframe(
         driver supports appending to an existing data source, will cause the
         data to be appended to the existing records in the data source.
         NOTE: append support is limited to specific drivers and GDAL versions.
+    dataset_metadata : dict, optional (default: None)
+        Metadata to be stored at the dataset level in the output file; limited
+        to drivers that support writing metadata, such as GPKG, and silently
+        ignored otherwise. Keys and values must be strings.
+    layer_metadata : dict, optional (default: None)
+        Metadata to be stored at the layer level in the output file; limited to
+        drivers that support writing metadata, such as GPKG, and silently
+        ignored otherwise. Keys and values must be strings.
+    metadata : dict, optional (default: None)
+        alias of layer_metadata
     dataset_options : dict, optional
         Dataset creation option (format specific) passed to OGR. Specify as
         a key-value dictionary.
@@ -409,6 +422,9 @@ def write_dataframe(
         promote_to_multi=promote_to_multi,
         nan_as_null=nan_as_null,
         append=append,
+        dataset_metadata=dataset_metadata,
+        layer_metadata=layer_metadata,
+        metadata=metadata,
         dataset_options=dataset_options,
         layer_options=layer_options,
         **kwargs,

pyogrio/raw.py

@@ -380,6 +380,9 @@ def write(
     promote_to_multi=None,
     nan_as_null=True,
     append=False,
+    dataset_metadata=None,
+    layer_metadata=None,
+    metadata=None,
     dataset_options=None,
     layer_options=None,
     **kwargs,
@@ -403,6 +406,21 @@ def write(
             "append to FlatGeobuf is not supported for GDAL <= 3.5.0 due to segfault"
         )
 
+    if metadata is not None:
+        if layer_metadata is not None:
+            raise ValueError("Cannot pass both metadata and layer_metadata")
+        layer_metadata = metadata
+
+    # validate metadata types
+    for metadata in [dataset_metadata, layer_metadata]:
+        if metadata is not None:
+            for k, v in metadata.items():
+                if not isinstance(k, str):
+                    raise ValueError(f"metadata key {k} must be a string")
+
+                if not isinstance(v, str):
+                    raise ValueError(f"metadata value {v} must be a string")
+
     if promote_to_multi is None:
         promote_to_multi = (
             geometry_type.startswith("Multi")
@@ -449,6 +467,8 @@ def write(
         promote_to_multi=promote_to_multi,
         nan_as_null=nan_as_null,
         append=append,
+        dataset_metadata=dataset_metadata,
+        layer_metadata=layer_metadata,
         dataset_kwargs=dataset_kwargs,
         layer_kwargs=layer_kwargs,
     )

pyogrio/tests/test_geopandas_io.py

@@ -1059,3 +1059,52 @@ def test_write_nullable_dtypes(tmp_path):
     expected["col4"] = expected["col4"].astype("float64")
     expected["col5"] = expected["col5"].astype(object)
     assert_geodataframe_equal(output_gdf, expected)
+
+
+@pytest.mark.parametrize(
+    "metadata_type", ["dataset_metadata", "layer_metadata", "metadata"]
+)
+def test_metadata_io(tmpdir, naturalearth_lowres, metadata_type):
+    metadata = {"level": metadata_type}
+
+    df = read_dataframe(naturalearth_lowres)
+
+    filename = os.path.join(str(tmpdir), "test.gpkg")
+    write_dataframe(df, filename, **{metadata_type: metadata})
+
+    metadata_key = "layer_metadata" if metadata_type == "metadata" else metadata_type
+
+    assert read_info(filename)[metadata_key] == metadata
+
+
+@pytest.mark.parametrize("metadata_type", ["dataset_metadata", "layer_metadata"])
+@pytest.mark.parametrize(
+    "metadata",
+    [
+        {1: 2},
+        {"key": None},
+        {"key": 1},
+    ],
+)
+def test_invalid_metadata(tmpdir, naturalearth_lowres, metadata_type, metadata):
+    with pytest.raises(ValueError, match="must be a string"):
+        filename = os.path.join(str(tmpdir), "test.gpkg")
+        write_dataframe(
+            read_dataframe(naturalearth_lowres), filename, **{metadata_type: metadata}
+        )
+
+
+@pytest.mark.parametrize("metadata_type", ["dataset_metadata", "layer_metadata"])
+def test_metadata_unsupported(tmpdir, naturalearth_lowres, metadata_type):
+    """metadata is silently ignored"""
+
+    filename = os.path.join(str(tmpdir), "test.geojson")
+    write_dataframe(
+        read_dataframe(naturalearth_lowres),
+        filename,
+        **{metadata_type: {"key": "value"}},
+    )
+
+    metadata_key = "layer_metadata" if metadata_type == "metadata" else metadata_type
+
+    assert read_info(filename)[metadata_key] is None