From c0fcc0b043d840a26095acf0be200a3f22e89de5 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 2 Dec 2025 11:11:21 +0100 Subject: [PATCH 1/9] include: define NOTZARR and ZARRMETA errors --- include/netcdf.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/netcdf.h b/include/netcdf.h index 459e6b1be1..3d3e128295 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -531,8 +531,10 @@ by the desired type. */ #define NC_EOBJECT (-140) /**< Some object exists when it should not */ #define NC_ENOOBJECT (-141) /**< Some object not found */ #define NC_EPLUGIN (-142) /**< Unclassified failure in accessing a dynamically loaded plugin> */ +#define NC_ENOTZARR (-143) /**< Malformed (NC)Zarr file */ +#define NC_EZARRMETA (-144) /**< Invalid (NC)Zarr file consolidated metadata */ -#define NC4_LAST_ERROR (-142) /**< @internal All netCDF errors > this. */ +#define NC4_LAST_ERROR (-144) /**< @internal All netCDF errors > this. */ /* * Don't forget to update docs/all-error-codes.md if adding new error codes here! @@ -964,7 +966,7 @@ nc_inq_var_endian(int ncid, int varid, int *endianp); /* Define a filter for a variable */ EXTERNL int -nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params); +nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms); /* Learn about the first filter on a variable */ EXTERNL int From 1f35fd2d3c241a835131f8a2bc45d9d507559c11 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Wed, 3 Dec 2025 11:56:58 +0100 Subject: [PATCH 2/9] rename Z* directives to Z2* --- libnczarr/zinternal.h | 9 ++++----- libnczarr/zsync.c | 22 +++++++++++----------- nczarr_test/ut_map.c | 10 +++++----- nczarr_test/ut_mapapi.c | 14 +++++++------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 2548ad54ba..0592410ea6 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -38,11 +38,10 @@ # endif #endif -#define ZMETAROOT "/.zgroup" -#define ZMETAATTR "/.zattrs" -#define ZGROUP ".zgroup" -#define ZATTRS ".zattrs" -#define ZARRAY ".zarray" +#define Z2GROUPROOT "/.zgroup" +#define Z2GROUP ".zgroup" +#define Z2ATTRS ".zattrs" +#define Z2ARRAY ".zarray" /* V2 Reserved Attributes */ /* diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index b5dc00663a..69b231417f 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -182,13 +182,13 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose) if((stat = NCZ_grpkey(grp,&fullpath))) goto done; - /* build ZGROUP contents */ + /* build Z2GROUP contents */ NCJnew(NCJ_DICT,&jgroup); snprintf(version,sizeof(version),"%d",zinfo->zarr.zarr_version); if((stat = NCJaddstring(jgroup,NCJ_STRING,"zarr_format"))<0) {stat = NC_EINVAL; goto done;} if((stat = NCJaddstring(jgroup,NCJ_INT,version))<0) {stat = NC_EINVAL; goto done;} - /* build ZGROUP path */ - if((stat = nczm_concat(fullpath,ZGROUP,&key))) + /* build Z2GROUP path */ + if((stat = nczm_concat(fullpath,Z2GROUP,&key))) goto done; /* Write to map */ if((stat=NCZ_uploadjson(map,key,jgroup))) goto done; @@ -487,7 +487,7 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) } /* build .zarray path */ - if((stat = nczm_concat(fullpath,ZARRAY,&key))) + if((stat = nczm_concat(fullpath,Z2ARRAY,&key))) goto done; /* Write to map */ @@ -1145,7 +1145,7 @@ define_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) if((stat = NCZ_grpkey(grp,&fullpath))) goto done; /* Download .zgroup and .zattrs */ - if((stat = downloadzarrobj(file,&zgrp->zgroup,fullpath,ZGROUP))) goto done; + if((stat = downloadzarrobj(file,&zgrp->zgroup,fullpath,Z2GROUP))) goto done; jgroup = zgrp->zgroup.obj; jattrs = zgrp->zgroup.atts; @@ -1451,7 +1451,7 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) goto done; /* Download */ - if((stat = downloadzarrobj(file,&zvar->zarray,varpath,ZARRAY))) goto done; + if((stat = downloadzarrobj(file,&zvar->zarray,varpath,Z2ARRAY))) goto done; jvar = zvar->zarray.obj; jatts = zvar->zarray.atts; assert(jvar == NULL || NCJsort(jvar) == NCJ_DICT); @@ -1822,7 +1822,7 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp) if((stat = NCZ_grpkey(root,&fullpath))) goto done; /* Download the root group .zgroup and associated .zattrs */ - if((stat = downloadzarrobj(file, &zroot->zgroup, fullpath, ZGROUP))) goto done; + if((stat = downloadzarrobj(file, &zroot->zgroup, fullpath, Z2GROUP))) goto done; jzgroup = zroot->zgroup.obj; /* Look for superblock; first in .zattrs and then in .zgroup */ @@ -1988,7 +1988,7 @@ searchvars(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames) if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */ /* See if name/.zarray exists */ if((stat = nczm_concat(grpkey,name,&varkey))) goto done; - if((stat = nczm_concat(varkey,ZARRAY,&zarray))) goto done; + if((stat = nczm_concat(varkey,Z2ARRAY,&zarray))) goto done; if((stat = nczmap_exists(zfile->map,zarray)) == NC_NOERR) nclistpush(varnames,strdup(name)); stat = NC_NOERR; @@ -2023,7 +2023,7 @@ searchsubgrps(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* subgrpnames) if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */ /* See if name/.zgroup exists */ if((stat = nczm_concat(grpkey,name,&subkey))) goto done; - if((stat = nczm_concat(subkey,ZGROUP,&zgroup))) goto done; + if((stat = nczm_concat(subkey,Z2GROUP,&zgroup))) goto done; if((stat = nczmap_exists(zfile->map,zgroup)) == NC_NOERR) nclistpush(subgrpnames,strdup(name)); stat = NC_NOERR; @@ -2502,7 +2502,7 @@ upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts) if(stat) goto done; /* write .zattrs*/ - if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done; + if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done; if((stat=NCZ_uploadjson(map,key,jatts))) goto done; nullfree(key); key = NULL; @@ -2603,7 +2603,7 @@ downloadzarrobj(NC_FILE_INFO_T* file, struct ZARROBJ* zobj, const char* fullpath if((stat = nczm_concat(fullpath,objname,&key))) goto done; if((stat=NCZ_downloadjson(map,key,&zobj->obj))) goto done; nullfree(key); key = NULL; - if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done; + if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done; if((stat=NCZ_downloadjson(map,key,&zobj->atts))) goto done; done: nullfree(key); diff --git a/nczarr_test/ut_map.c b/nczarr_test/ut_map.c index b769ce643a..f3a6d4c4e3 100644 --- a/nczarr_test/ut_map.c +++ b/nczarr_test/ut_map.c @@ -95,7 +95,7 @@ simplecreate(void) if((stat = nczmap_create(impl,url,0,0,NULL,&map))) goto done; - if((stat=nczm_concat(NULL,ZMETAROOT,&path))) + if((stat=nczm_concat(NULL,Z2GROUPROOT,&path))) goto done; /* Write empty metadata content */ @@ -135,7 +135,7 @@ writemeta(void) if((stat = nczmap_open(impl,url,NC_WRITE,0,NULL,&map))) goto done; - if((stat=nczm_concat(META1,ZARRAY,&path))) + if((stat=nczm_concat(META1,Z2ARRAY,&path))) goto done; if((stat = nczmap_write(map, path, strlen(metadata1), metadata1))) goto done; @@ -158,7 +158,7 @@ writemeta2(void) if((stat = nczmap_open(impl,url,NC_WRITE,0,NULL,&map))) goto done; - if((stat=nczm_concat(META2,ZARRAY,&path))) + if((stat=nczm_concat(META2,Z2ARRAY,&path))) goto done; if((stat = nczmap_write(map, path, strlen(metadata2), metadata2))) goto done; @@ -212,7 +212,7 @@ readmeta(void) if((stat = nczmap_open(impl,url,0,0,NULL,&map))) goto done; - if((stat = readkey(map,META1,ZARRAY))) goto done; + if((stat = readkey(map,META1,Z2ARRAY))) goto done; done: (void)nczmap_close(map,0); @@ -228,7 +228,7 @@ readmeta2(void) if((stat = nczmap_open(impl,url,0,0,NULL,&map))) goto done; - if((stat = readkey(map,META2,ZARRAY))) + if((stat = readkey(map,META2,Z2ARRAY))) goto done; done: diff --git a/nczarr_test/ut_mapapi.c b/nczarr_test/ut_mapapi.c index 600225054a..67e0e8dc14 100644 --- a/nczarr_test/ut_mapapi.c +++ b/nczarr_test/ut_mapapi.c @@ -99,7 +99,7 @@ simplecreate(void) printf("Pass: create: create: %s\n",url); - truekey = makekey(ZMETAROOT); + truekey = makekey(Z2GROUPROOT); if((stat = nczmap_write(map, truekey, 0, NULL))) goto done; printf("Pass: create: defineobj: %s\n",truekey); @@ -184,13 +184,13 @@ simplemeta(void) report(PASS,"open",map); /* Make sure .nczarr exists (from simplecreate) */ - truekey = makekey(ZMETAROOT); + truekey = makekey(Z2GROUPROOT); if((stat = nczmap_exists(map,truekey))) goto done; report(PASS,".nczarr: exists",map); free(truekey); truekey = NULL; - if((stat=nczm_concat(META1,ZARRAY,&key))) + if((stat=nczm_concat(META1,Z2ARRAY,&key))) goto done; truekey = makekey(key); nullfree(key); key = NULL; @@ -199,13 +199,13 @@ simplemeta(void) report(PASS,".zarray: def",map); free(truekey); truekey = NULL; - truekey = makekey(ZMETAROOT); + truekey = makekey(Z2GROUPROOT); if((stat = nczmap_write(map, truekey, strlen(metadata1), metadata1))) goto done; report(PASS,".nczarr: writemetadata",map); free(truekey); truekey = NULL; - if((stat=nczm_concat(META1,ZARRAY,&key))) + if((stat=nczm_concat(META1,Z2ARRAY,&key))) goto done; truekey = makekey(key); free(key); key = NULL; @@ -225,7 +225,7 @@ simplemeta(void) report(PASS,"re-open",map); /* Read previously written */ - truekey = makekey(ZMETAROOT); + truekey = makekey(Z2GROUPROOT); if((stat = nczmap_exists(map, truekey))) goto done; report(PASS,".nczarr: exists",map); @@ -245,7 +245,7 @@ simplemeta(void) else report(PASS,".nczarr: content verify",map); nullfree(content); content = NULL; - if((stat=nczm_concat(META1,ZARRAY,&key))) + if((stat=nczm_concat(META1,Z2ARRAY,&key))) goto done; truekey = makekey(key); nullfree(key); key = NULL; From 9e31c39b5d1f76b95cededb6fe2ba222d9f6fc4d Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Thu, 4 Dec 2025 11:36:36 +0100 Subject: [PATCH 3/9] zarr.*: NCZ_uploadjson, set `const NCjson*` --- libnczarr/zarr.h | 2 +- libnczarr/zutil.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index fd41cdc49b..9a317f3c68 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -69,7 +69,7 @@ EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* ty EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative); EXTERNL int ncz_fill_value_sort(nc_type nctype, int*); EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size); -EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json); +EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json); EXTERNL int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp); EXTERNL int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist); EXTERNL int NCZ_grpname_full(int gid, char** pathp); diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index 0d031fc65e..1e4b4541a2 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -272,7 +272,7 @@ NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp) @author Dennis Heimbigner */ int -NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json) +NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json) { int stat = NC_NOERR; char* content = NULL; From c89cf3dd75d9aad91ae47047c7fac70eddbce5d5 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 25 Nov 2025 18:05:07 +0100 Subject: [PATCH 4/9] Expose ncstrndup as strndup to whole libnetcdf --- include/ncuri.h | 6 ++++++ libdispatch/ncuri.c | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/ncuri.h b/include/ncuri.h index c5646ff4b9..010cc473b9 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -59,6 +59,12 @@ typedef struct NCURI { extern "C" { #endif +#ifndef HAVE_STRNDUP +#define strndup ncstrndup +/* Not all systems have strndup, so provide one*/ +char *ncstrndup(const char *s, size_t len); +#endif + EXTERNL int ncuriparse(const char* s, NCURI** ncuri); EXTERNL void ncurifree(NCURI* ncuri); diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index db79e6cd41..80fc5391f9 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -79,7 +79,6 @@ static const char* userpwdallow = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!$&'()*+,-.;=_~?#/"; #ifndef HAVE_STRNDUP -#define strndup ncstrndup /* Not all systems have strndup, so provide one*/ char* ncstrndup(const char* s, size_t len) From 9e4f897367d401bc4c85ccc7f50b622ddf37dc48 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Fri, 5 Dec 2025 17:23:58 +0100 Subject: [PATCH 5/9] Address strndup review --- include/ncconfigure.h | 6 ++++++ include/ncuri.h | 5 ----- libdispatch/dmissing.c | 14 ++++++++++++++ libdispatch/ncuri.c | 14 -------------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/include/ncconfigure.h b/include/ncconfigure.h index b93b468e00..d67fb39e08 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -67,6 +67,12 @@ char* strdup(const char*); #endif #endif +#ifndef HAVE_STRNDUP +#ifndef strndup +char *strndup(const char *s, size_t len); +#endif +#endif + #ifndef HAVE_STRLCAT #ifndef strlcat #define strlcat nc_strlcat diff --git a/include/ncuri.h b/include/ncuri.h index 010cc473b9..6bf6e9451a 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -59,11 +59,6 @@ typedef struct NCURI { extern "C" { #endif -#ifndef HAVE_STRNDUP -#define strndup ncstrndup -/* Not all systems have strndup, so provide one*/ -char *ncstrndup(const char *s, size_t len); -#endif EXTERNL int ncuriparse(const char* s, NCURI** ncuri); EXTERNL void ncurifree(NCURI* ncuri); diff --git a/libdispatch/dmissing.c b/libdispatch/dmissing.c index 2b82723cbb..3b1ba7c9b3 100644 --- a/libdispatch/dmissing.c +++ b/libdispatch/dmissing.c @@ -43,6 +43,20 @@ strdup(const char* s) } #endif +#ifndef HAVE_STRNDUP +char* +strndup(const char* s, size_t len) +{ + char* dup; + if(s == NULL) return NULL; + dup = (char*)malloc(len+1); + if(dup == NULL) return NULL; + memcpy((void*)dup,s,len); + dup[len] = '\0'; + return dup; +} +#endif + #if !defined(_MSC_VER) && !defined(WIN32) #ifndef HAVE_STRLCAT diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index 80fc5391f9..e93f17d2b4 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -78,20 +78,6 @@ static const char* queryallow = static const char* userpwdallow = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!$&'()*+,-.;=_~?#/"; -#ifndef HAVE_STRNDUP -/* Not all systems have strndup, so provide one*/ -char* -ncstrndup(const char* s, size_t len) -{ - char* dup; - if(s == NULL) return NULL; - dup = (char*)malloc(len+1); - if(dup == NULL) return NULL; - memcpy((void*)dup,s,len); - dup[len] = '\0'; - return dup; -} -#endif /* Forward */ static int collectprefixparams(char* text, char** nextp); static void freestringlist(NClist* list); From b0fdbe87fc4563c34d4cd207a58217f51eac1fa2 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 2 Dec 2025 11:50:16 +0100 Subject: [PATCH 6/9] zmetadata: introduce header --- libnczarr/CMakeLists.txt | 1 + libnczarr/Makefile.am | 1 + libnczarr/zincludes.h | 1 + libnczarr/zinternal.h | 1 + libnczarr/zmetadata.h | 105 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+) create mode 100644 libnczarr/zmetadata.h diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt index c460e1d76e..87499e0820 100644 --- a/libnczarr/CMakeLists.txt +++ b/libnczarr/CMakeLists.txt @@ -39,6 +39,7 @@ zdispatch.h zincludes.h zinternal.h zmap.h +zmetadata.h zodom.h zprovenance.h zplugins.h diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am index 4227516267..f661418110 100644 --- a/libnczarr/Makefile.am +++ b/libnczarr/Makefile.am @@ -59,6 +59,7 @@ zdispatch.h \ zincludes.h \ zinternal.h \ zmap.h \ +zmetadata.h \ zodom.h \ zprovenance.h \ zplugins.h \ diff --git a/libnczarr/zincludes.h b/libnczarr/zincludes.h index 79bdab3a8c..9fa2429e7c 100644 --- a/libnczarr/zincludes.h +++ b/libnczarr/zincludes.h @@ -51,6 +51,7 @@ extern "C" { #include "ncutil.h" #include "zmap.h" +#include "zmetadata.h" #include "zinternal.h" #include "zdispatch.h" #include "zprovenance.h" diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 0592410ea6..bbed0c9802 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -122,6 +122,7 @@ typedef struct NCZ_FILE_INFO { NCZcommon common; struct NCZMAP* map; /* implementation */ struct NCauth* auth; + struct NCZ_Metadata metadata; struct nczarr { int zarr_version; struct { diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h new file mode 100644 index 0000000000..43f9544fdf --- /dev/null +++ b/libnczarr/zmetadata.h @@ -0,0 +1,105 @@ +/* Copyright 2018-2018 University Corporation for Atmospheric + Research/Unidata. */ + +/* +Zarr Metadata Handling + +Encapsulates Zarr metadata operations across versions, supporting both +consolidated access and per-file access. Provides a common interface +for metadata operations. + +The dispatcher is defined by the type NCZ_Metadata. +It offers 2 types of operations that allow decoupling/abstract +filesystem access, content reading of the JSON metadata files +1. Listings: (involves either listing or parsing consolidated view) + - variables within a group + - groups withing a group +2. Retrieve JSON representation of (sub)groups, arrays and attributes. + Directly read from filesystem/objectstore or retrieve the JSON + object from the consolidated view respective to the group or variable + +Note: This will also be the case of zarr v3 +(the elements will be extracted from zarr.json instead) +*/ + +#ifndef ZMETADATA_H +#define ZMETADATA_H +struct NCZ_FILE_INFO; + +#if defined(__cplusplus) +extern "C" +{ +#endif +/* This is the version of the metadata table. It should be changed + * when new functions are added to the metadata table. */ +#ifndef NCZ_METADATA_VERSION +#define NCZ_METADATA_VERSION 1 +#endif /*NCZ_METADATA_VERSION*/ + +#define Z2METADATA "/.zmetadata" +#define ZARRFORMAT2 2 + +#define ZARR_NOT_CONSOLIDATED 0 +#define ZARR_CONSOLIDATED 1 + +typedef enum { + NCZMD_NULL, + NCZMD_GROUP, + NCZMD_ATTRS, + NCZMD_ARRAY +} NCZMD_MetadataType; + +typedef struct NCZ_Metadata +{ + int zarr_format; /* Zarr format version */ + int dispatch_version; /* Dispatch table version*/ + size64_t flags; /* Metadata handling flags */ + NCjson *jcsl; // Consolidated JSON view or NULL + int (*list_nodes)(struct NCZ_FILE_INFO*, const char * key, NClist *groups, NClist *vars); + int (*list_groups)(struct NCZ_FILE_INFO*, const char * key, NClist *subgrpnames); + int (*list_variables)(struct NCZ_FILE_INFO*, const char * key, NClist *varnames); + int (*fetch_json_content)(struct NCZ_FILE_INFO*, NCZMD_MetadataType, const char *name, NCjson **jobj); + int (*update_json_content)(struct NCZ_FILE_INFO*, NCZMD_MetadataType, const char *name, const NCjson *jobj); + int (*validate_consolidated)(const NCjson *jobj); +} NCZ_Metadata; + +extern const NCZ_Metadata *NCZ_metadata_handler2; + +extern int NCZMD_set_metadata_handler(struct NCZ_FILE_INFO *zfile); +extern int NCZMD_get_metadata_format(struct NCZ_FILE_INFO*zfile, int *zarrformat); +extern void NCZMD_free_metadata_handler(NCZ_Metadata * zmd); + +extern int NCZMD_list_nodes(struct NCZ_FILE_INFO*zfile, const char * key, NClist *groups, NClist *vars); + +/// @brief Lists groups under a given group key. +/// @param zfile - The zarr file info structure +/// @param key - The group key within which to list groups +/// @param groups - Pointer to NClist to receive group names +/// @return NO_ERROR on success, error code on failure +extern int NCZMD_list_groups(struct NCZ_FILE_INFO*zfile, const char * key, NClist *groups); + +/// @brief Lists variables under a given group key. +/// @param zfile - The zarr file info structure +/// @param key - The group key within which to list variables +/// @param variables - Pointer to NClist to receive variable names +/// @return NO_ERROR on success, error code on failure +extern int NCZMD_list_variables(struct NCZ_FILE_INFO*zfile, const char * key, NClist *variables); + +/// @brief Fetches the JSON metadata a group given a group key. +/// @param zfile - The zarr file info structure +/// @param key - The group key whose metadata to fetch +/// @param jgroup - Pointer to NCjson to receive the group metadata +/// @return NO_ERROR on success, error code on failure +extern int NCZMD_fetch_json_group(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jgroup); +extern int NCZMD_fetch_json_attrs(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jattrs); +extern int NCZMD_fetch_json_array(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jarrays); + +extern int NCZMD_update_json_group(struct NCZ_FILE_INFO*zfile, const char *key, const NCjson *jgroup); +extern int NCZMD_update_json_attrs(struct NCZ_FILE_INFO*zfile, const char *key, const NCjson *jattrs); +extern int NCZMD_update_json_array(struct NCZ_FILE_INFO *zfile, const char *key, const NCjson *jarrays); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZMETADATA_H */ From 39f1535b2c5a591e3fcbd7af21ed210a928b6488 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Wed, 3 Dec 2025 11:12:42 +0100 Subject: [PATCH 7/9] Document zmetadata.h --- libnczarr/zmetadata.h | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h index 43f9544fdf..bedc761e45 100644 --- a/libnczarr/zmetadata.h +++ b/libnczarr/zmetadata.h @@ -65,10 +65,28 @@ typedef struct NCZ_Metadata extern const NCZ_Metadata *NCZ_metadata_handler2; +/// @brief Sets the metadata handler for the given zarr file based on +/// environment variables, file creation mode, and dataset contents. +/// @param zfile - The zarr file info structure +/// @return NC_NOERR on success, NC_EZARRMETA on failure extern int NCZMD_set_metadata_handler(struct NCZ_FILE_INFO *zfile); + +/// @brief Determines the Zarr format version set on the metadata handler or by +/// probing the dataset for the existence of zarr metadata objects (.z*). +/// @param zfile - The zarr file info structure +/// @param zarrformat - Pointer to int to receive the zarr format version extern int NCZMD_get_metadata_format(struct NCZ_FILE_INFO*zfile, int *zarrformat); + +/// @brief Frees any resources associated with the metadata handler +/// @param zmd - Potinter to the metadata handler structure extern void NCZMD_free_metadata_handler(NCZ_Metadata * zmd); +/// @brief Lists groups and/or variables under a given group key. +/// @param zfile - The zarr file info structure +/// @param key - The group key within which to list nodes +/// @param groups - Pointer to NClist to receive group names, NULL to skip +/// @param vars - Pointer to NClist to receive variable names, NULL to skip +/// @return NO_ERROR on success, error code on failure extern int NCZMD_list_nodes(struct NCZ_FILE_INFO*zfile, const char * key, NClist *groups, NClist *vars); /// @brief Lists groups under a given group key. @@ -91,11 +109,40 @@ extern int NCZMD_list_variables(struct NCZ_FILE_INFO*zfile, const char * key, NC /// @param jgroup - Pointer to NCjson to receive the group metadata /// @return NO_ERROR on success, error code on failure extern int NCZMD_fetch_json_group(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jgroup); + +/// @brief Fetches the JSON attributes given a key, either of group or array. +/// @param zfile - The zarr file info structure +/// @param key - The key whose attributes to fetch +/// @param jattrs - Pointer to NCjson to receive the attributes +/// @return NO_ERROR on success, error code on failure extern int NCZMD_fetch_json_attrs(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jattrs); + +/// @brief Fetches the JSON metadata of an array given its key. +/// @param zfile - The zarr file info structure +/// @param key - The array key whose metadata to fetch +/// @param jarrays - Pointer to NCjson to receive the array metadata +/// @return NO_ERROR on success, error code on failure extern int NCZMD_fetch_json_array(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jarrays); +/// @brief Updates the JSON metadata of a group given a group key. +/// @param zfile - The zarr file info structure +/// @param key - The group key whose metadata to update +/// @param jgroup - The NCjson containing the new group metadata +/// @return NO_ERROR on success, error code on failure extern int NCZMD_update_json_group(struct NCZ_FILE_INFO*zfile, const char *key, const NCjson *jgroup); + +/// @brief Updates the JSON attributes given a key, either of group or array. +/// @param zfile - The zarr file info structure +/// @param key - The key whose attributes to update +/// @param jattrs - The NCjson containing the new attributes +/// @return NO_ERROR on success, error code on failure extern int NCZMD_update_json_attrs(struct NCZ_FILE_INFO*zfile, const char *key, const NCjson *jattrs); + +/// @brief Updates the JSON metadata of an array given its key. +/// @param zfile - The zarr file info structure +/// @param key - The array key whose metadata to update +/// @param jarrays - The NCjson containing the new array metadata +/// @return NO_ERROR on success, error code on failure extern int NCZMD_update_json_array(struct NCZ_FILE_INFO *zfile, const char *key, const NCjson *jarrays); #if defined(__cplusplus) From cec7253e97ba59118a2ce6de1c614e0fd6df60e7 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 2 Dec 2025 11:55:35 +0100 Subject: [PATCH 8/9] zmetadata: introduce wrappers and implementation --- libnczarr/CMakeLists.txt | 2 + libnczarr/Makefile.am | 2 + libnczarr/zmetadata.c | 102 +++++++++++++++++++++++++++ libnczarr/zmetadata2.c | 144 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+) create mode 100644 libnczarr/zmetadata.c create mode 100644 libnczarr/zmetadata2.c diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt index 87499e0820..e2d0f3ea86 100644 --- a/libnczarr/CMakeLists.txt +++ b/libnczarr/CMakeLists.txt @@ -23,6 +23,8 @@ zgrp.c zinternal.c zmap.c zmap_file.c +zmetadata.c +zmetadata2.c zodom.c zopen.c zprov.c diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am index f661418110..8d236fc1f3 100644 --- a/libnczarr/Makefile.am +++ b/libnczarr/Makefile.am @@ -43,6 +43,8 @@ zgrp.c \ zinternal.c \ zmap.c \ zmap_file.c \ +zmetadata.c \ +zmetadata2.c\ zodom.c \ zopen.c \ zprov.c \ diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c new file mode 100644 index 0000000000..924e284845 --- /dev/null +++ b/libnczarr/zmetadata.c @@ -0,0 +1,102 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "zincludes.h" + +static int +cmpstrings(const void* a1, const void* a2) +{ + const char** s1 = (const char**)a1; + const char** s2 = (const char**)a2; + return strcmp(*s1,*s2); +} + +int NCZMD_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars) +{ + int stat = NC_NOERR; + if((stat = zfile->metadata.list_nodes(zfile,key, groups, vars))){ + return stat; + } + qsort(groups->content, groups->length, sizeof(char*), cmpstrings); + qsort(vars->content, vars->length, sizeof(char*), cmpstrings); + return stat; +} + +int NCZMD_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *subgrpnames) +{ + int stat = NC_NOERR; + if((stat = zfile->metadata.list_groups(zfile,key, subgrpnames))){ + return stat; + } + qsort(subgrpnames->content, subgrpnames->length, sizeof(char*), cmpstrings); + return stat; +} + +int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist *varnames) +{ + int stat = NC_NOERR; + if((stat = zfile->metadata.list_variables(zfile, key, varnames))){ + return stat; + } + qsort(varnames->content, varnames->length, sizeof(char*), cmpstrings); + return stat; +} + +int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jgroup) { + return zfile->metadata.fetch_json_content(zfile, NCZMD_GROUP, key, jgroup); +} + +int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jattrs) { + return zfile->metadata.fetch_json_content(zfile, NCZMD_ATTRS, key, jattrs); +} + +int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jarray) { + return zfile->metadata.fetch_json_content(zfile, NCZMD_ARRAY, key, jarray); +} + +int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jgroup) { + return zfile->metadata.update_json_content(zfile, NCZMD_GROUP, key, jgroup); +} + +int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jattrs) { + return zfile->metadata.update_json_content(zfile, NCZMD_ATTRS, key , jattrs); +} + +int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jarray) { + return zfile->metadata.update_json_content(zfile, NCZMD_ARRAY, key, jarray); +} + +int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat) +{ + NCZ_Metadata *zmd = &(zfile->metadata); + + if (zmd->zarr_format >= ZARRFORMAT2) + { + *zarrformat = zmd->zarr_format; + return NC_NOERR; + } + + if (!nczmap_exists(zfile->map, "/" Z2ATTRS) && !nczmap_exists(zfile->map, "/" Z2GROUP) && !nczmap_exists(zfile->map, "/" Z2ARRAY)) + { + return NC_ENOTZARR; + } + + *zarrformat = ZARRFORMAT2; + return NC_NOERR; +} + + +int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile) +{ + zfile->metadata = *NCZ_metadata_handler2; + zfile->metadata.jcsl = NULL; + return NC_NOERR; +} + +void NCZMD_free_metadata_handler(NCZ_Metadata * zmd){ + if (zmd == NULL) return; + NCJreclaim(zmd->jcsl); + zmd->jcsl = NULL; +} diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c new file mode 100644 index 0000000000..9069a0c46b --- /dev/null +++ b/libnczarr/zmetadata2.c @@ -0,0 +1,144 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "zincludes.h" + +#define MINIMIM_CSL_REP_RAW "{\"metadata\":{},\"zarr_consolidated_format\":1}" + +int NCZMD_v2_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars); + +int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups); +int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist * variables); +int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj); + +int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *key, const NCjson *jobj); + +int validate_consolidated_json_noop_v2(const NCjson *json); + +static const NCZ_Metadata NCZ_md2_table = { + ZARRFORMAT2, + NCZ_METADATA_VERSION, + ZARR_NOT_CONSOLIDATED, + .jcsl = NULL, + + .list_nodes = NCZMD_v2_list_nodes, + .list_groups = NCZMD_v2_list_groups, + .list_variables = NCZMD_v2_list_variables, + + .fetch_json_content = fetch_json_content_v2, + .update_json_content = update_json_content_v2, + .validate_consolidated = validate_consolidated_json_noop_v2, +}; + +const NCZ_Metadata *NCZ_metadata_handler2 = &NCZ_md2_table; + +int NCZMD_v2_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *variables) +{ + size_t i; + int stat = NC_NOERR; + char *subkey = NULL; + char *zkey = NULL; + NClist *matches = nclistnew(); + + if ((stat = nczmap_search(zfile->map, key, matches))) + goto done; + for (i = 0; i < nclistlength(matches); i++) + { + const char *name = nclistget(matches, i); + if (name[0] == NCZM_DOT) + continue; + if ((stat = nczm_concat(key, name, &subkey))) + goto done; + if ((stat = nczm_concat(subkey, Z2GROUP, &zkey))) + goto done; + if (NC_NOERR == nczmap_exists(zfile->map, zkey) && groups != NULL) + nclistpush(groups, strdup(name)); + + nullfree(zkey); + zkey = NULL; + if ((stat = nczm_concat(subkey, Z2ARRAY, &zkey))) + goto done; + if (NC_NOERR == nczmap_exists(zfile->map, zkey) && variables != NULL) + nclistpush(variables, strdup(name)); + stat = NC_NOERR; + + nullfree(subkey); + subkey = NULL; + nullfree(zkey); + zkey = NULL; + } + +done: + nullfree(subkey); + nullfree(zkey); + nclistfreeall(matches); + return stat; +} + + +int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups) +{ + return NCZMD_v2_list_nodes(zfile, key, groups, NULL); +} + +int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist *variables) +{ + return NCZMD_v2_list_nodes(zfile, key, NULL, variables); +} + +static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **suffix){ + switch (zarr_obj_type) + { + case NCZMD_GROUP: + *suffix = Z2GROUP; + break; + case NCZMD_ATTRS: + *suffix = Z2ATTRS; + break; + case NCZMD_ARRAY: + *suffix = Z2ARRAY; + break; + default: + return NC_EINVAL; + } + return NC_NOERR; +} + +int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *prefix, NCjson **jobj) +{ + int stat = NC_NOERR; + const char *suffix; + char * key = NULL; + if ((stat = zarr_obj_type2suffix(zobj, &suffix)) + || (stat = nczm_concat(prefix, suffix, &key))){ + goto done; + } + + stat = NCZ_downloadjson(zfile->map, key, jobj); +done: + nullfree(key); + return stat; +} + +int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *prefix, const NCjson *jobj) +{ + int stat = NC_NOERR; + const char *suffix; + char * key = NULL; + if ((stat = zarr_obj_type2suffix(zobj, &suffix)) + || (stat = nczm_concat(prefix, suffix, &key))){ + goto done; + } + + stat = NCZ_uploadjson(zfile->map, key, jobj); +done: + nullfree(key); + return stat; +} + +int validate_consolidated_json_noop_v2(const NCjson *json){ + NC_UNUSED(json); + return NC_NOERR; +} From 28b1644541b07b9b2a55d0a0027a46dcac196cc9 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Wed, 3 Dec 2025 11:13:58 +0100 Subject: [PATCH 9/9] Document zmetadata2.c --- libnczarr/zmetadata2.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c index 9069a0c46b..3985136ebc 100644 --- a/libnczarr/zmetadata2.c +++ b/libnczarr/zmetadata2.c @@ -7,14 +7,51 @@ #define MINIMIM_CSL_REP_RAW "{\"metadata\":{},\"zarr_consolidated_format\":1}" +/// @brief Retrieve the group and variable names contained within a group specified by `key`. +/// The order of the names may be arbitrary +/// @param zfile - The zarr file info structure +/// @param key - the key of the node - group +/// @param groups - NClist where names will be added +/// @param variables - NClist where names will be added +/// @return `NC_NOERR` if succeeding int NCZMD_v2_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars); +/// @brief Retrieve the group names contained within a group specified by `key`. +/// The order of the names may be arbitrary +/// @param zfile - The zarr file info structure +/// @param key - the key of the node - group +/// @param groups - NClist where names will be added +/// @return `NC_NOERR` if succeeding int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups); + +/// @brief Retrieve the variable names contained by a group specified by `key`. +/// The order of the names may be arbitrary +/// @param zfile - The zarr file info structure +/// @param key - the key of the node - group +/// @param variables - NClist where names will be added +/// @return `NC_NOERR` if succeeding int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist * variables); + +/// @brief Retrieve JSON metadata of a given type for the specified `key` from the storage +/// @param zfile - The zarr file info structure +/// @param zobj - The type of metadata to set +/// @param key - the key of the node - group or array +/// @param jobj - JSON to be written +/// @return `NC_NOERR` if succeeding int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj); +/// @brief Write JSON metadata of a given type for the specified `key` to the storage +/// @param zfile - The zarr file info structure +/// @param zobj - The type of metadata to set +/// @param key - the key of the node - group or array +/// @param jobj - JSON to be written +/// @return `NC_NOERR` if succeeding int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *key, const NCjson *jobj); + +/// @brief Place holder for non consolidated handler +/// @param json - Not used! +/// @return `NC_NOERR` always int validate_consolidated_json_noop_v2(const NCjson *json); static const NCZ_Metadata NCZ_md2_table = {