Skip to content

Commit b1081bf

Browse files
WardFWardF
authored
Merge pull request #3221 from mannreis/zarr-csl-3
Part 3 - Zarr consolidated metadata
2 parents 0f4184f + 28b1644 commit b1081bf

File tree

4 files changed

+287
-0
lines changed

4 files changed

+287
-0
lines changed

‎libnczarr/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ zgrp.c
2323
zinternal.c
2424
zmap.c
2525
zmap_file.c
26+
zmetadata.c
27+
zmetadata2.c
2628
zodom.c
2729
zopen.c
2830
zprov.c

‎libnczarr/Makefile.am‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ zgrp.c \
4343
zinternal.c \
4444
zmap.c \
4545
zmap_file.c \
46+
zmetadata.c \
47+
zmetadata2.c\
4648
zodom.c \
4749
zopen.c \
4850
zprov.c \

‎libnczarr/zmetadata.c‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*********************************************************************
2+
* Copyright 2018, UCAR/Unidata
3+
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
4+
*********************************************************************/
5+
6+
#include "zincludes.h"
7+
8+
static int
9+
cmpstrings(const void* a1, const void* a2)
10+
{
11+
const char** s1 = (const char**)a1;
12+
const char** s2 = (const char**)a2;
13+
return strcmp(*s1,*s2);
14+
}
15+
16+
int NCZMD_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars)
17+
{
18+
int stat = NC_NOERR;
19+
if((stat = zfile->metadata.list_nodes(zfile,key, groups, vars))){
20+
return stat;
21+
}
22+
qsort(groups->content, groups->length, sizeof(char*), cmpstrings);
23+
qsort(vars->content, vars->length, sizeof(char*), cmpstrings);
24+
return stat;
25+
}
26+
27+
int NCZMD_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *subgrpnames)
28+
{
29+
int stat = NC_NOERR;
30+
if((stat = zfile->metadata.list_groups(zfile,key, subgrpnames))){
31+
return stat;
32+
}
33+
qsort(subgrpnames->content, subgrpnames->length, sizeof(char*), cmpstrings);
34+
return stat;
35+
}
36+
37+
int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist *varnames)
38+
{
39+
int stat = NC_NOERR;
40+
if((stat = zfile->metadata.list_variables(zfile, key, varnames))){
41+
return stat;
42+
}
43+
qsort(varnames->content, varnames->length, sizeof(char*), cmpstrings);
44+
return stat;
45+
}
46+
47+
int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jgroup) {
48+
return zfile->metadata.fetch_json_content(zfile, NCZMD_GROUP, key, jgroup);
49+
}
50+
51+
int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jattrs) {
52+
return zfile->metadata.fetch_json_content(zfile, NCZMD_ATTRS, key, jattrs);
53+
}
54+
55+
int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, const char *key, NCjson **jarray) {
56+
return zfile->metadata.fetch_json_content(zfile, NCZMD_ARRAY, key, jarray);
57+
}
58+
59+
int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jgroup) {
60+
return zfile->metadata.update_json_content(zfile, NCZMD_GROUP, key, jgroup);
61+
}
62+
63+
int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jattrs) {
64+
return zfile->metadata.update_json_content(zfile, NCZMD_ATTRS, key , jattrs);
65+
}
66+
67+
int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, const char *key, const NCjson *jarray) {
68+
return zfile->metadata.update_json_content(zfile, NCZMD_ARRAY, key, jarray);
69+
}
70+
71+
int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat)
72+
{
73+
NCZ_Metadata *zmd = &(zfile->metadata);
74+
75+
if (zmd->zarr_format >= ZARRFORMAT2)
76+
{
77+
*zarrformat = zmd->zarr_format;
78+
return NC_NOERR;
79+
}
80+
81+
if (!nczmap_exists(zfile->map, "/" Z2ATTRS) && !nczmap_exists(zfile->map, "/" Z2GROUP) && !nczmap_exists(zfile->map, "/" Z2ARRAY))
82+
{
83+
return NC_ENOTZARR;
84+
}
85+
86+
*zarrformat = ZARRFORMAT2;
87+
return NC_NOERR;
88+
}
89+
90+
91+
int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile)
92+
{
93+
zfile->metadata = *NCZ_metadata_handler2;
94+
zfile->metadata.jcsl = NULL;
95+
return NC_NOERR;
96+
}
97+
98+
void NCZMD_free_metadata_handler(NCZ_Metadata * zmd){
99+
if (zmd == NULL) return;
100+
NCJreclaim(zmd->jcsl);
101+
zmd->jcsl = NULL;
102+
}

‎libnczarr/zmetadata2.c‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*********************************************************************
2+
* Copyright 2018, UCAR/Unidata
3+
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
4+
*********************************************************************/
5+
6+
#include "zincludes.h"
7+
8+
#define MINIMIM_CSL_REP_RAW "{\"metadata\":{},\"zarr_consolidated_format\":1}"
9+
10+
/// @brief Retrieve the group and variable names contained within a group specified by `key`.
11+
/// The order of the names may be arbitrary
12+
/// @param zfile - The zarr file info structure
13+
/// @param key - the key of the node - group
14+
/// @param groups - NClist where names will be added
15+
/// @param variables - NClist where names will be added
16+
/// @return `NC_NOERR` if succeeding
17+
int NCZMD_v2_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *vars);
18+
19+
/// @brief Retrieve the group names contained within a group specified by `key`.
20+
/// The order of the names may be arbitrary
21+
/// @param zfile - The zarr file info structure
22+
/// @param key - the key of the node - group
23+
/// @param groups - NClist where names will be added
24+
/// @return `NC_NOERR` if succeeding
25+
int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups);
26+
27+
/// @brief Retrieve the variable names contained by a group specified by `key`.
28+
/// The order of the names may be arbitrary
29+
/// @param zfile - The zarr file info structure
30+
/// @param key - the key of the node - group
31+
/// @param variables - NClist where names will be added
32+
/// @return `NC_NOERR` if succeeding
33+
int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist * variables);
34+
35+
/// @brief Retrieve JSON metadata of a given type for the specified `key` from the storage
36+
/// @param zfile - The zarr file info structure
37+
/// @param zobj - The type of metadata to set
38+
/// @param key - the key of the node - group or array
39+
/// @param jobj - JSON to be written
40+
/// @return `NC_NOERR` if succeeding
41+
int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
42+
43+
/// @brief Write JSON metadata of a given type for the specified `key` to the storage
44+
/// @param zfile - The zarr file info structure
45+
/// @param zobj - The type of metadata to set
46+
/// @param key - the key of the node - group or array
47+
/// @param jobj - JSON to be written
48+
/// @return `NC_NOERR` if succeeding
49+
int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *key, const NCjson *jobj);
50+
51+
52+
/// @brief Place holder for non consolidated handler
53+
/// @param json - Not used!
54+
/// @return `NC_NOERR` always
55+
int validate_consolidated_json_noop_v2(const NCjson *json);
56+
57+
static const NCZ_Metadata NCZ_md2_table = {
58+
ZARRFORMAT2,
59+
NCZ_METADATA_VERSION,
60+
ZARR_NOT_CONSOLIDATED,
61+
.jcsl = NULL,
62+
63+
.list_nodes = NCZMD_v2_list_nodes,
64+
.list_groups = NCZMD_v2_list_groups,
65+
.list_variables = NCZMD_v2_list_variables,
66+
67+
.fetch_json_content = fetch_json_content_v2,
68+
.update_json_content = update_json_content_v2,
69+
.validate_consolidated = validate_consolidated_json_noop_v2,
70+
};
71+
72+
const NCZ_Metadata *NCZ_metadata_handler2 = &NCZ_md2_table;
73+
74+
int NCZMD_v2_list_nodes(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups, NClist *variables)
75+
{
76+
size_t i;
77+
int stat = NC_NOERR;
78+
char *subkey = NULL;
79+
char *zkey = NULL;
80+
NClist *matches = nclistnew();
81+
82+
if ((stat = nczmap_search(zfile->map, key, matches)))
83+
goto done;
84+
for (i = 0; i < nclistlength(matches); i++)
85+
{
86+
const char *name = nclistget(matches, i);
87+
if (name[0] == NCZM_DOT)
88+
continue;
89+
if ((stat = nczm_concat(key, name, &subkey)))
90+
goto done;
91+
if ((stat = nczm_concat(subkey, Z2GROUP, &zkey)))
92+
goto done;
93+
if (NC_NOERR == nczmap_exists(zfile->map, zkey) && groups != NULL)
94+
nclistpush(groups, strdup(name));
95+
96+
nullfree(zkey);
97+
zkey = NULL;
98+
if ((stat = nczm_concat(subkey, Z2ARRAY, &zkey)))
99+
goto done;
100+
if (NC_NOERR == nczmap_exists(zfile->map, zkey) && variables != NULL)
101+
nclistpush(variables, strdup(name));
102+
stat = NC_NOERR;
103+
104+
nullfree(subkey);
105+
subkey = NULL;
106+
nullfree(zkey);
107+
zkey = NULL;
108+
}
109+
110+
done:
111+
nullfree(subkey);
112+
nullfree(zkey);
113+
nclistfreeall(matches);
114+
return stat;
115+
}
116+
117+
118+
int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, const char * key, NClist *groups)
119+
{
120+
return NCZMD_v2_list_nodes(zfile, key, groups, NULL);
121+
}
122+
123+
int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, const char * key, NClist *variables)
124+
{
125+
return NCZMD_v2_list_nodes(zfile, key, NULL, variables);
126+
}
127+
128+
static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **suffix){
129+
switch (zarr_obj_type)
130+
{
131+
case NCZMD_GROUP:
132+
*suffix = Z2GROUP;
133+
break;
134+
case NCZMD_ATTRS:
135+
*suffix = Z2ATTRS;
136+
break;
137+
case NCZMD_ARRAY:
138+
*suffix = Z2ARRAY;
139+
break;
140+
default:
141+
return NC_EINVAL;
142+
}
143+
return NC_NOERR;
144+
}
145+
146+
int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *prefix, NCjson **jobj)
147+
{
148+
int stat = NC_NOERR;
149+
const char *suffix;
150+
char * key = NULL;
151+
if ((stat = zarr_obj_type2suffix(zobj, &suffix))
152+
|| (stat = nczm_concat(prefix, suffix, &key))){
153+
goto done;
154+
}
155+
156+
stat = NCZ_downloadjson(zfile->map, key, jobj);
157+
done:
158+
nullfree(key);
159+
return stat;
160+
}
161+
162+
int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj, const char *prefix, const NCjson *jobj)
163+
{
164+
int stat = NC_NOERR;
165+
const char *suffix;
166+
char * key = NULL;
167+
if ((stat = zarr_obj_type2suffix(zobj, &suffix))
168+
|| (stat = nczm_concat(prefix, suffix, &key))){
169+
goto done;
170+
}
171+
172+
stat = NCZ_uploadjson(zfile->map, key, jobj);
173+
done:
174+
nullfree(key);
175+
return stat;
176+
}
177+
178+
int validate_consolidated_json_noop_v2(const NCjson *json){
179+
NC_UNUSED(json);
180+
return NC_NOERR;
181+
}

0 commit comments

Comments
 (0)