Part 2 - Zarr consolidated metadata

include/ncconfigure.h

@@ -67,6 +67,12 @@ char* strdup(const char*);
 #endif
 #endif
 
+#ifndef HAVE_STRNDUP
+#ifndef strndup
+char *strndup(const char *s, size_t len);
+#endif
+#endif
+
 #ifndef HAVE_STRLCAT
 #ifndef strlcat
 #define strlcat nc_strlcat

include/ncuri.h

@@ -59,6 +59,7 @@ typedef struct NCURI {
 extern "C" {
 #endif
 
+
 EXTERNL int ncuriparse(const char* s, NCURI** ncuri);
 EXTERNL void ncurifree(NCURI* ncuri);
 

include/netcdf.h

@@ -531,8 +531,10 @@ by the desired type. */
 #define NC_EOBJECT       (-140)    /**< Some object exists when it should not */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
+#define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
+#define NC_EZARRMETA     (-144)    /**< Invalid (NC)Zarr file consolidated metadata */
 
-#define NC4_LAST_ERROR   (-142)    /**< @internal All netCDF errors > this. */
+#define NC4_LAST_ERROR   (-144)    /**< @internal All netCDF errors > this. */
 
 /*
  * Don't forget to update docs/all-error-codes.md if adding new error codes here!
@@ -964,7 +966,7 @@ nc_inq_var_endian(int ncid, int varid, int *endianp);
 
 /* Define a filter for a variable */
 EXTERNL int
-nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params);
+nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms);
 
 /* Learn about the first filter on a variable */
 EXTERNL int

libdispatch/dmissing.c

@@ -43,6 +43,20 @@ strdup(const char* s)
 }
 #endif
 
+#ifndef HAVE_STRNDUP
+char*
+strndup(const char* s, size_t len)
+{
+    char* dup;
+    if(s == NULL) return NULL;
+    dup = (char*)malloc(len+1);
+    if(dup == NULL) return NULL;
+    memcpy((void*)dup,s,len);
+    dup[len] = '\0';
+    return dup;
+}
+#endif
+
 #if !defined(_MSC_VER) && !defined(WIN32)
 
 #ifndef HAVE_STRLCAT

libdispatch/ncuri.c

@@ -78,21 +78,6 @@ static const char* queryallow =
 static const char* userpwdallow =
 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!$&'()*+,-.;=_~?#/";
 
-#ifndef HAVE_STRNDUP
-#define strndup ncstrndup
-/* Not all systems have strndup, so provide one*/
-char*
-ncstrndup(const char* s, size_t len)
-{
-    char* dup;
-    if(s == NULL) return NULL;
-    dup = (char*)malloc(len+1);
-    if(dup == NULL) return NULL;
-    memcpy((void*)dup,s,len);
-    dup[len] = '\0';
-    return dup;
-}
-#endif
 /* Forward */
 static int collectprefixparams(char* text, char** nextp);
 static void freestringlist(NClist* list);

libnczarr/CMakeLists.txt

@@ -39,6 +39,7 @@ zdispatch.h
 zincludes.h
 zinternal.h
 zmap.h
+zmetadata.h
 zodom.h
 zprovenance.h
 zplugins.h

libnczarr/Makefile.am

@@ -59,6 +59,7 @@ zdispatch.h \
 zincludes.h \
 zinternal.h \
 zmap.h \
+zmetadata.h \
 zodom.h \
 zprovenance.h \
 zplugins.h \

libnczarr/zarr.h

@@ -69,7 +69,7 @@ EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* ty
 EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
 EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
 EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
-EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
+EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json);
 EXTERNL int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
 EXTERNL int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist);
 EXTERNL int NCZ_grpname_full(int gid, char** pathp);

libnczarr/zincludes.h

@@ -51,6 +51,7 @@ extern "C" {
 #include "ncutil.h"
 
 #include "zmap.h"
+#include "zmetadata.h"
 #include "zinternal.h"
 #include "zdispatch.h"
 #include "zprovenance.h"

libnczarr/zinternal.h

@@ -38,11 +38,10 @@
 #  endif
 #endif
 
-#define ZMETAROOT "/.zgroup"
-#define ZMETAATTR "/.zattrs"
-#define ZGROUP ".zgroup"
-#define ZATTRS ".zattrs"
-#define ZARRAY ".zarray"
+#define Z2GROUPROOT "/.zgroup"
+#define Z2GROUP ".zgroup"
+#define Z2ATTRS ".zattrs"
+#define Z2ARRAY ".zarray"
 
 /* V2 Reserved Attributes */
 /*
@@ -123,6 +122,7 @@ typedef struct NCZ_FILE_INFO {
     NCZcommon common;
     struct NCZMAP* map; /* implementation */
     struct NCauth* auth;
+    struct NCZ_Metadata metadata;
     struct nczarr {
 	int zarr_version;
 	struct {

libnczarr/zmetadata.h

@@ -0,0 +1,152 @@
+/* Copyright 2018-2018 University Corporation for Atmospheric
+   Research/Unidata. */
+
+/*
+Zarr Metadata Handling
+
+Encapsulates Zarr metadata operations across versions, supporting both
+consolidated access and per-file access. Provides a common interface
+for metadata operations.
+
+The dispatcher is defined by the type NCZ_Metadata.
+It offers 2 types of operations that allow decoupling/abstract
+filesystem access, content reading of the JSON metadata files
+1. Listings: (involves either listing or parsing consolidated view)
+ - variables within a group
+ - groups withing a group
+2. Retrieve JSON representation of (sub)groups, arrays and attributes.
+	Directly read from filesystem/objectstore or retrieve the JSON
+	object from the consolidated view respective to the group or variable
+
+Note: This will also be the case of zarr v3
+(the elements will be extracted from zarr.json instead)
+*/
+
+#ifndef ZMETADATA_H
+#define ZMETADATA_H
+struct NCZ_FILE_INFO;
+
+#if defined(__cplusplus)
+extern "C"
+{
+#endif
+/* This is the version of the metadata table. It should be changed
+ * when new functions are added to the metadata table. */
+#ifndef NCZ_METADATA_VERSION
+#define NCZ_METADATA_VERSION 1
+#endif /*NCZ_METADATA_VERSION*/
+
+#define Z2METADATA "/.zmetadata"
+#define ZARRFORMAT2 2
+
+#define ZARR_NOT_CONSOLIDATED 0
+#define ZARR_CONSOLIDATED 1
+
+typedef enum {
+	NCZMD_NULL,
+	NCZMD_GROUP,
+	NCZMD_ATTRS,
+	NCZMD_ARRAY
+} NCZMD_MetadataType;
+
+typedef struct NCZ_Metadata
+{
+	int zarr_format;		/* Zarr format version */
+	int dispatch_version;   /* Dispatch table version*/
+	size64_t flags;			/* Metadata handling flags */
+	NCjson *jcsl; // Consolidated JSON view or NULL
+	int (*list_nodes)(struct NCZ_FILE_INFO*, const char * key, NClist *groups, NClist *vars);
+	int (*list_groups)(struct NCZ_FILE_INFO*, const char * key, NClist *subgrpnames);
+	int (*list_variables)(struct NCZ_FILE_INFO*, const char * key, NClist *varnames);
+	int (*fetch_json_content)(struct NCZ_FILE_INFO*, NCZMD_MetadataType, const char *name, NCjson **jobj);
+	int (*update_json_content)(struct NCZ_FILE_INFO*, NCZMD_MetadataType, const char *name, const NCjson *jobj);
+	int (*validate_consolidated)(const NCjson *jobj);
+} NCZ_Metadata;
+
+extern const NCZ_Metadata *NCZ_metadata_handler2;
+
+/// @brief Sets the metadata handler for the given zarr file based on
+/// 	environment variables, file creation mode, and dataset contents.
+/// @param zfile - The zarr file info structure
+/// @return NC_NOERR on success, NC_EZARRMETA on failure
+extern int NCZMD_set_metadata_handler(struct NCZ_FILE_INFO *zfile);
+
+/// @brief Determines the Zarr format version set on the metadata handler or by
+/// 	probing the dataset for the existence of zarr metadata objects (.z*).
+/// @param zfile - The zarr file info structure
+/// @param zarrformat - Pointer to int to receive the zarr format version
+extern int NCZMD_get_metadata_format(struct NCZ_FILE_INFO*zfile, int *zarrformat);
+
+/// @brief Frees any resources associated with the metadata handler
+/// @param zmd - Potinter to the metadata handler structure
+extern void NCZMD_free_metadata_handler(NCZ_Metadata * zmd);
+
+/// @brief Lists groups and/or variables under a given group key.
+/// @param zfile - The zarr file info structure
+/// @param key - The group key within which to list nodes
+/// @param groups - Pointer to NClist to receive group names, NULL to skip
+/// @param vars - Pointer to NClist to receive variable names, NULL to skip
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_list_nodes(struct NCZ_FILE_INFO*zfile, const char * key, NClist *groups, NClist *vars);
+
+/// @brief Lists groups under a given group key.
+/// @param zfile - The zarr file info structure
+/// @param key - The group key within which to list groups
+/// @param groups - Pointer to NClist to receive group names
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_list_groups(struct NCZ_FILE_INFO*zfile, const char * key, NClist *groups);
+
+/// @brief Lists variables under a given group key.
+/// @param zfile - The zarr file info structure
+/// @param key - The group key within which to list variables
+/// @param variables - Pointer to NClist to receive variable names
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_list_variables(struct NCZ_FILE_INFO*zfile, const char * key, NClist *variables);
+
+/// @brief Fetches the JSON metadata a group given a group key.
+/// @param zfile - The zarr file info structure
+/// @param key - The group key whose metadata to fetch
+/// @param jgroup -	Pointer to NCjson to receive the group metadata
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_fetch_json_group(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jgroup);
+
+/// @brief Fetches the JSON attributes given a key, either of group or array.
+/// @param zfile - The zarr file info structure
+/// @param key - The key whose attributes to fetch
+/// @param jattrs - Pointer to NCjson to receive the attributes
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_fetch_json_attrs(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jattrs);
+
+/// @brief Fetches the JSON metadata of an array given its key.
+/// @param zfile - The zarr file info structure
+/// @param key - The array key whose metadata to fetch
+/// @param jarrays - Pointer to NCjson to receive the array metadata
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_fetch_json_array(struct NCZ_FILE_INFO*zfile, const char *key, NCjson **jarrays);
+
+/// @brief Updates the JSON metadata of a group given a group key.
+/// @param zfile - The zarr file info structure
+/// @param key - The group key whose metadata to update
+/// @param jgroup -	The NCjson containing the new group metadata
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_update_json_group(struct NCZ_FILE_INFO*zfile, const char *key, const NCjson *jgroup);
+
+/// @brief Updates the JSON attributes given a key, either of group or array.
+/// @param zfile - The zarr file info structure
+/// @param key - The key whose attributes to update
+/// @param jattrs - The NCjson containing the new attributes
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_update_json_attrs(struct NCZ_FILE_INFO*zfile, const char *key, const NCjson *jattrs);
+
+/// @brief Updates the JSON metadata of an array given its key.
+/// @param zfile - The zarr file info structure
+/// @param key - The array key whose metadata to update
+/// @param jarrays - The NCjson containing the new array metadata
+/// @return NO_ERROR on success, error code on failure
+extern int NCZMD_update_json_array(struct NCZ_FILE_INFO *zfile, const char *key, const NCjson *jarrays);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* ZMETADATA_H */

libnczarr/zsync.c

@@ -182,13 +182,13 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     if((stat = NCZ_grpkey(grp,&fullpath)))
         goto done;
 
-    /* build ZGROUP contents */
+    /* build Z2GROUP contents */
     NCJnew(NCJ_DICT,&jgroup);
     snprintf(version,sizeof(version),"%d",zinfo->zarr.zarr_version);
     if((stat = NCJaddstring(jgroup,NCJ_STRING,"zarr_format"))<0) {stat = NC_EINVAL; goto done;}
     if((stat = NCJaddstring(jgroup,NCJ_INT,version))<0) {stat = NC_EINVAL; goto done;}
-    /* build ZGROUP path */
-    if((stat = nczm_concat(fullpath,ZGROUP,&key)))
+    /* build Z2GROUP path */
+    if((stat = nczm_concat(fullpath,Z2GROUP,&key)))
 	goto done;
     /* Write to map */
     if((stat=NCZ_uploadjson(map,key,jgroup))) goto done;
@@ -487,7 +487,7 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
     }
 
     /* build .zarray path */
-    if((stat = nczm_concat(fullpath,ZARRAY,&key)))
+    if((stat = nczm_concat(fullpath,Z2ARRAY,&key)))
 	goto done;
 
     /* Write to map */
@@ -1145,7 +1145,7 @@ define_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
     if((stat = NCZ_grpkey(grp,&fullpath))) goto done;
 
     /* Download .zgroup and .zattrs */
-    if((stat = downloadzarrobj(file,&zgrp->zgroup,fullpath,ZGROUP))) goto done;
+    if((stat = downloadzarrobj(file,&zgrp->zgroup,fullpath,Z2GROUP))) goto done;
     jgroup = zgrp->zgroup.obj;
     jattrs = zgrp->zgroup.atts;
 
@@ -1451,7 +1451,7 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname)
 	goto done;
 
     /* Download */
-    if((stat = downloadzarrobj(file,&zvar->zarray,varpath,ZARRAY))) goto done;
+    if((stat = downloadzarrobj(file,&zvar->zarray,varpath,Z2ARRAY))) goto done;
     jvar = zvar->zarray.obj;
     jatts = zvar->zarray.atts;
     assert(jvar == NULL || NCJsort(jvar) == NCJ_DICT);
@@ -1822,7 +1822,7 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp)
     if((stat = NCZ_grpkey(root,&fullpath))) goto done;
 
     /* Download the root group .zgroup and associated .zattrs */
-    if((stat = downloadzarrobj(file, &zroot->zgroup, fullpath, ZGROUP))) goto done;
+    if((stat = downloadzarrobj(file, &zroot->zgroup, fullpath, Z2GROUP))) goto done;
     jzgroup = zroot->zgroup.obj;    
 
     /* Look for superblock; first in .zattrs and then in .zgroup */
@@ -1988,7 +1988,7 @@ searchvars(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames)
 	if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */
 	/* See if name/.zarray exists */
 	if((stat = nczm_concat(grpkey,name,&varkey))) goto done;
-	if((stat = nczm_concat(varkey,ZARRAY,&zarray))) goto done;
+	if((stat = nczm_concat(varkey,Z2ARRAY,&zarray))) goto done;
 	if((stat = nczmap_exists(zfile->map,zarray)) == NC_NOERR)
 	    nclistpush(varnames,strdup(name));
 	stat = NC_NOERR;
@@ -2023,7 +2023,7 @@ searchsubgrps(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* subgrpnames)
 	if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */
 	/* See if name/.zgroup exists */
 	if((stat = nczm_concat(grpkey,name,&subkey))) goto done;
-	if((stat = nczm_concat(subkey,ZGROUP,&zgroup))) goto done;
+	if((stat = nczm_concat(subkey,Z2GROUP,&zgroup))) goto done;
 	if((stat = nczmap_exists(zfile->map,zgroup)) == NC_NOERR)
 	    nclistpush(subgrpnames,strdup(name));
 	stat = NC_NOERR;
@@ -2502,7 +2502,7 @@ upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts)
     if(stat) goto done;
 
     /* write .zattrs*/
-    if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done;
+    if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done;
     if((stat=NCZ_uploadjson(map,key,jatts))) goto done;
     nullfree(key); key = NULL;
 
@@ -2603,7 +2603,7 @@ downloadzarrobj(NC_FILE_INFO_T* file, struct ZARROBJ* zobj, const char* fullpath
     if((stat = nczm_concat(fullpath,objname,&key))) goto done;
     if((stat=NCZ_downloadjson(map,key,&zobj->obj))) goto done;
     nullfree(key); key = NULL;
-    if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done;
+    if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done;
     if((stat=NCZ_downloadjson(map,key,&zobj->atts))) goto done;
 done:
     nullfree(key);

libnczarr/zutil.c

@@ -272,7 +272,7 @@ NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp)
 @author Dennis Heimbigner
 */
 int
-NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json)
+NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json)
 {
     int stat = NC_NOERR;
     char* content = NULL;