Part 3 - Zarr consolidated metadata

include/ncconfigure.h

@@ -67,6 +67,12 @@ char* strdup(const char*);
 #endif
 #endif
 
+#ifndef HAVE_STRNDUP
+#ifndef strndup
+char *strndup(const char *s, size_t len);
+#endif
+#endif
+
 #ifndef HAVE_STRLCAT
 #ifndef strlcat
 #define strlcat nc_strlcat

include/ncuri.h

@@ -59,6 +59,7 @@ typedef struct NCURI {
 extern "C" {
 #endif
 
+
 EXTERNL int ncuriparse(const char* s, NCURI** ncuri);
 EXTERNL void ncurifree(NCURI* ncuri);
 

include/netcdf.h

@@ -531,8 +531,10 @@ by the desired type. */
 #define NC_EOBJECT       (-140)    /**< Some object exists when it should not */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
+#define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
+#define NC_EZARRMETA     (-144)    /**< Invalid (NC)Zarr file consolidated metadata */
 
-#define NC4_LAST_ERROR   (-142)    /**< @internal All netCDF errors > this. */
+#define NC4_LAST_ERROR   (-144)    /**< @internal All netCDF errors > this. */
 
 /*
  * Don't forget to update docs/all-error-codes.md if adding new error codes here!
@@ -964,7 +966,7 @@ nc_inq_var_endian(int ncid, int varid, int *endianp);
 
 /* Define a filter for a variable */
 EXTERNL int
-nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params);
+nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms);
 
 /* Learn about the first filter on a variable */
 EXTERNL int

libdispatch/dmissing.c

@@ -43,6 +43,20 @@ strdup(const char* s)
 }
 #endif
 
+#ifndef HAVE_STRNDUP
+char*
+strndup(const char* s, size_t len)
+{
+    char* dup;
+    if(s == NULL) return NULL;
+    dup = (char*)malloc(len+1);
+    if(dup == NULL) return NULL;
+    memcpy((void*)dup,s,len);
+    dup[len] = '\0';
+    return dup;
+}
+#endif
+
 #if !defined(_MSC_VER) && !defined(WIN32)
 
 #ifndef HAVE_STRLCAT

libdispatch/ncuri.c

@@ -78,21 +78,6 @@ static const char* queryallow =
 static const char* userpwdallow =
 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!$&'()*+,-.;=_~?#/";
 
-#ifndef HAVE_STRNDUP
-#define strndup ncstrndup
-/* Not all systems have strndup, so provide one*/
-char*
-ncstrndup(const char* s, size_t len)
-{
-    char* dup;
-    if(s == NULL) return NULL;
-    dup = (char*)malloc(len+1);
-    if(dup == NULL) return NULL;
-    memcpy((void*)dup,s,len);
-    dup[len] = '\0';
-    return dup;
-}
-#endif
 /* Forward */
 static int collectprefixparams(char* text, char** nextp);
 static void freestringlist(NClist* list);

libnczarr/CMakeLists.txt

@@ -23,6 +23,8 @@ zgrp.c
 zinternal.c
 zmap.c
 zmap_file.c
+zmetadata.c
+zmetadata2.c
 zodom.c
 zopen.c
 zprov.c
@@ -39,6 +41,7 @@ zdispatch.h
 zincludes.h
 zinternal.h
 zmap.h
+zmetadata.h
 zodom.h
 zprovenance.h
 zplugins.h

libnczarr/Makefile.am

@@ -43,6 +43,8 @@ zgrp.c \
 zinternal.c \
 zmap.c \
 zmap_file.c \
+zmetadata.c \
+zmetadata2.c\
 zodom.c \
 zopen.c \
 zprov.c \
@@ -59,6 +61,7 @@ zdispatch.h \
 zincludes.h \
 zinternal.h \
 zmap.h \
+zmetadata.h \
 zodom.h \
 zprovenance.h \
 zplugins.h \

libnczarr/zarr.h

@@ -69,7 +69,7 @@ EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* ty
 EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
 EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
 EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
-EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
+EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json);
 EXTERNL int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
 EXTERNL int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist);
 EXTERNL int NCZ_grpname_full(int gid, char** pathp);

libnczarr/zincludes.h

@@ -51,6 +51,7 @@ extern "C" {
 #include "ncutil.h"
 
 #include "zmap.h"
+#include "zmetadata.h"
 #include "zinternal.h"
 #include "zdispatch.h"
 #include "zprovenance.h"

libnczarr/zinternal.h

@@ -38,11 +38,10 @@
 #  endif
 #endif
 
-#define ZMETAROOT "/.zgroup"
-#define ZMETAATTR "/.zattrs"
-#define ZGROUP ".zgroup"
-#define ZATTRS ".zattrs"
-#define ZARRAY ".zarray"
+#define Z2GROUPROOT "/.zgroup"
+#define Z2GROUP ".zgroup"
+#define Z2ATTRS ".zattrs"
+#define Z2ARRAY ".zarray"
 
 /* V2 Reserved Attributes */
 /*
@@ -123,6 +122,7 @@ typedef struct NCZ_FILE_INFO {
     NCZcommon common;
     struct NCZMAP* map; /* implementation */
     struct NCauth* auth;
+    struct NCZ_Metadata metadata;
     struct nczarr {
 	int zarr_version;
 	struct {

libnczarr/zmetadata.c

@@ -0,0 +1,102 @@
+/*********************************************************************
+ *   Copyright 2018, UCAR/Unidata
+ *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ *********************************************************************/
+
+#include "zincludes.h"
+
+static int
+cmpstrings(const void* a1, const void* a2)
+{
+    const char** s1 = (const char**)a1;
+    const char** s2 = (const char**)a2;
+    return strcmp(*s1,*s2);
+}
+
+int NCZMD_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars)
+{
+    int stat = NC_NOERR;
+    if((stat = zfile->metadata.list_nodes(zfile,key, groups, vars))){
+        return stat;
+    }
+    qsort(groups->content, groups->length, sizeof(char*), cmpstrings);
+    qsort(vars->content, vars->length, sizeof(char*), cmpstrings);
+    return stat;
+}
+
+int NCZMD_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *subgrpnames)
+{
+    int stat = NC_NOERR;
+    if((stat = zfile->metadata.list_groups(zfile,key, subgrpnames))){
+        return stat;
+    }
+    qsort(subgrpnames->content, subgrpnames->length, sizeof(char*), cmpstrings);
+    return stat;
+}
+
+int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist *varnames)
+{
+	int stat = NC_NOERR;
+    if((stat = zfile->metadata.list_variables(zfile, key, varnames))){
+        return stat;
+    }
+    qsort(varnames->content, varnames->length, sizeof(char*), cmpstrings);
+    return stat;
+}
+
+int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jgroup) {
+	return zfile->metadata.fetch_json_content(zfile, NCZMD_GROUP, key, jgroup);
+}
+
+int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jattrs) {
+	return  zfile->metadata.fetch_json_content(zfile, NCZMD_ATTRS, key, jattrs);
+}
+
+int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jarray) {
+	return zfile->metadata.fetch_json_content(zfile, NCZMD_ARRAY, key, jarray);
+}
+
+int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jgroup) {
+	return zfile->metadata.update_json_content(zfile, NCZMD_GROUP, key, jgroup);
+}
+
+int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jattrs) {
+	return zfile->metadata.update_json_content(zfile, NCZMD_ATTRS, key , jattrs);
+}
+
+int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jarray) {
+	return zfile->metadata.update_json_content(zfile, NCZMD_ARRAY, key, jarray);
+}
+
+int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat)
+{
+    NCZ_Metadata *zmd = &(zfile->metadata);
+
+	if (zmd->zarr_format >= ZARRFORMAT2)
+	{
+		*zarrformat = zmd->zarr_format;
+		return NC_NOERR;
+	}
+
+	if (!nczmap_exists(zfile->map, "/" Z2ATTRS) && !nczmap_exists(zfile->map, "/" Z2GROUP) && !nczmap_exists(zfile->map, "/" Z2ARRAY))
+	{
+		return NC_ENOTZARR;
+	}
+
+	*zarrformat = ZARRFORMAT2;
+	return NC_NOERR;
+}
+
+
+int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile)
+{
+  zfile->metadata = *NCZ_metadata_handler2;
+  zfile->metadata.jcsl = NULL;
+  return NC_NOERR;
+}
+
+void NCZMD_free_metadata_handler(NCZ_Metadata * zmd){
+	if (zmd == NULL) return;
+	NCJreclaim(zmd->jcsl);
+    zmd->jcsl = NULL;
+}