Allow use of filters that expand chunks by a large factor (#5939)

* Change new chunk indexing methods to always encode chunk size as a 64
bit (size of lengths) integer, when using the 2.0 file format.

* Add CHANGELOG.md note

* Spelling

* Fix errors in parallel build

* Committing clang-format changes

* More parallel fixes.

* Committing clang-format changes

* Another parallel fix

* Fix parallel for real this time I hope

* Update function descriptions in dsets.c

* Fix spelling

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

Author: fortnern

release_docs/CHANGELOG.md

@@ -253,6 +253,8 @@ The Virtual Dataset Global Heap Block format has been updated to version 1 to su
 
 Use of the shared strings option for Virtual Datasets reduces memory overhead and optimizes dataset close operations.
 
+The chunked dataset file format has been updated to always use 64 bits to encode the size of filtered chunks. This will allow data filters that expand the chunks by a large amount to still work. Chunk sizes are still limited to `2^32 - 1`. This new format is only used when the HDF5 library version bounds lower bound is set to 2.0 or later.
+
 ### The `H5Dread_chunk()` signature has changed
 
 A new parameter, `nalloc`, has been added to `H5Dread_chunk()`. This parameter contains a pointer to a variable that holds the size of the buffer buf. If *nalloc is not large enough to hold the entire chunk being read, no data is read. On exit, the value of this variable is set to the buffer size needed to read the chunk.

src/H5Dbtree.c

@@ -41,7 +41,7 @@
 /* Local Macros */
 /****************/
 
-#define H5D_BTREE_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree.shared)
+#define H5D_BTREE_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->layout->storage.u.chunk.u.btree.shared)
 
 /******************/
 /* Local Typedefs */
@@ -819,13 +819,13 @@ H5D__btree_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR_UNUS
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
     assert(H5_addr_defined(dset_ohdr_addr));
 
-    idx_info->storage->u.btree.dset_ohdr_addr = dset_ohdr_addr;
+    idx_info->layout->storage.u.chunk.u.btree.dset_ohdr_addr = dset_ohdr_addr;
 
     /* Allocate the shared structure */
-    if (H5D__btree_shared_create(idx_info->f, idx_info->storage, idx_info->layout) < 0)
+    if (H5D__btree_shared_create(idx_info->f, &idx_info->layout->storage.u.chunk,
+                                 &idx_info->layout->u.chunk) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info");
 
 done:
@@ -861,15 +861,14 @@ H5D__btree_idx_create(const H5D_chk_idx_info_t *idx_info)
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
-    assert(!H5_addr_defined(idx_info->storage->idx_addr));
+    assert(!H5_addr_defined(idx_info->layout->storage.u.chunk.idx_addr));
 
     /* Initialize "user" data for B-tree callbacks, etc. */
-    udata.layout  = idx_info->layout;
-    udata.storage = idx_info->storage;
+    udata.layout  = &idx_info->layout->u.chunk;
+    udata.storage = &idx_info->layout->storage.u.chunk;
 
     /* Create the v1 B-tree for the chunk index */
-    if (H5B_create(idx_info->f, H5B_BTREE, &udata, &(idx_info->storage->idx_addr) /*out*/) < 0)
+    if (H5B_create(idx_info->f, H5B_BTREE, &udata, &(idx_info->layout->storage.u.chunk.idx_addr) /*out*/) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create B-tree");
 
 done:
@@ -929,8 +928,8 @@ H5D__btree_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open)
     FUNC_ENTER_PACKAGE_NOERR
 
     assert(idx_info);
-    assert(idx_info->storage);
-    assert(H5D_CHUNK_IDX_BTREE == idx_info->storage->idx_type);
+    assert(idx_info->layout);
+    assert(H5D_CHUNK_IDX_BTREE == idx_info->layout->storage.u.chunk.idx_type);
     assert(is_open);
 
     *is_open = H5D_BTREE_IDX_IS_OPEN(idx_info);
@@ -979,15 +978,14 @@ H5D__btree_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata,
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
-    assert(H5_addr_defined(idx_info->storage->idx_addr));
+    assert(H5_addr_defined(idx_info->layout->storage.u.chunk.idx_addr));
     assert(udata);
 
     /*
      * Create the chunk it if it doesn't exist, or reallocate the chunk if
      * its size changed.
      */
-    if (H5B_insert(idx_info->f, H5B_BTREE, idx_info->storage->idx_addr, udata) < 0)
+    if (H5B_insert(idx_info->f, H5B_BTREE, idx_info->layout->storage.u.chunk.idx_addr, udata) < 0)
         HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk");
 
 done:
@@ -1017,14 +1015,13 @@ H5D__btree_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->layout->ndims > 0);
-    assert(idx_info->storage);
-    assert(H5_addr_defined(idx_info->storage->idx_addr));
+    assert(idx_info->layout->u.chunk.ndims > 0);
+    assert(H5_addr_defined(idx_info->layout->storage.u.chunk.idx_addr));
     assert(udata);
 
     /* Go get the chunk information from the B-tree */
     found = false;
-    if (H5B_find(idx_info->f, H5B_BTREE, idx_info->storage->idx_addr, &found, udata) < 0)
+    if (H5B_find(idx_info->f, H5B_BTREE, idx_info->layout->storage.u.chunk.idx_addr, &found, udata) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTFIND, FAIL, "can't check for chunk in B-tree");
 
 done:
@@ -1115,20 +1112,19 @@ H5D__btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t c
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
-    assert(H5_addr_defined(idx_info->storage->idx_addr));
+    assert(H5_addr_defined(idx_info->layout->storage.u.chunk.idx_addr));
     assert(chunk_cb);
     assert(chunk_udata);
 
     /* Initialize userdata */
     memset(&udata, 0, sizeof udata);
-    udata.common.layout  = idx_info->layout;
-    udata.common.storage = idx_info->storage;
+    udata.common.layout  = &idx_info->layout->u.chunk;
+    udata.common.storage = &idx_info->layout->storage.u.chunk;
     udata.cb             = chunk_cb;
     udata.udata          = chunk_udata;
 
     /* Iterate over existing chunks */
-    if ((ret_value = H5B_iterate(idx_info->f, H5B_BTREE, idx_info->storage->idx_addr,
+    if ((ret_value = H5B_iterate(idx_info->f, H5B_BTREE, idx_info->layout->storage.u.chunk.idx_addr,
                                  H5D__btree_idx_iterate_cb, &udata)) < 0)
         HERROR(H5E_DATASET, H5E_BADITER, "unable to iterate over chunk B-tree");
 
@@ -1155,14 +1151,13 @@ H5D__btree_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
-    assert(H5_addr_defined(idx_info->storage->idx_addr));
+    assert(H5_addr_defined(idx_info->layout->storage.u.chunk.idx_addr));
     assert(udata);
 
     /* Remove the chunk from the v1 B-tree index and release the space for the
      * chunk (in the B-tree callback).
      */
-    if (H5B_remove(idx_info->f, H5B_BTREE, idx_info->storage->idx_addr, udata) < 0)
+    if (H5B_remove(idx_info->f, H5B_BTREE, idx_info->layout->storage.u.chunk.idx_addr, udata) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to remove chunk entry");
 
 done:
@@ -1192,23 +1187,22 @@ H5D__btree_idx_delete(const H5D_chk_idx_info_t *idx_info)
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
 
     /* Check if the index data structure has been allocated */
-    if (H5_addr_defined(idx_info->storage->idx_addr)) {
+    if (H5_addr_defined(idx_info->layout->storage.u.chunk.idx_addr)) {
         H5O_storage_chunk_t   tmp_storage; /* Local copy of storage info */
         H5D_chunk_common_ud_t udata;       /* User data for B-tree operations */
 
         /* Set up temporary chunked storage info */
-        tmp_storage = *idx_info->storage;
+        tmp_storage = idx_info->layout->storage.u.chunk;
 
         /* Set up the shared structure */
-        if (H5D__btree_shared_create(idx_info->f, &tmp_storage, idx_info->layout) < 0)
+        if (H5D__btree_shared_create(idx_info->f, &tmp_storage, &idx_info->layout->u.chunk) < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info");
 
         /* Set up B-tree user data */
         memset(&udata, 0, sizeof udata);
-        udata.layout  = idx_info->layout;
+        udata.layout  = &idx_info->layout->u.chunk;
         udata.storage = &tmp_storage;
 
         /* Delete entire B-tree */
@@ -1246,25 +1240,25 @@ H5D__btree_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_
     assert(idx_info_src->f);
     assert(idx_info_src->pline);
     assert(idx_info_src->layout);
-    assert(idx_info_src->storage);
     assert(idx_info_dst);
     assert(idx_info_dst->f);
     assert(idx_info_dst->pline);
     assert(idx_info_dst->layout);
-    assert(idx_info_dst->storage);
-    assert(!H5_addr_defined(idx_info_dst->storage->idx_addr));
+    assert(!H5_addr_defined(idx_info_dst->layout->storage.u.chunk.idx_addr));
 
     /* Create shared B-tree info for each file */
-    if (H5D__btree_shared_create(idx_info_src->f, idx_info_src->storage, idx_info_src->layout) < 0)
+    if (H5D__btree_shared_create(idx_info_src->f, &idx_info_src->layout->storage.u.chunk,
+                                 &idx_info_src->layout->u.chunk) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for source shared B-tree info");
-    if (H5D__btree_shared_create(idx_info_dst->f, idx_info_dst->storage, idx_info_dst->layout) < 0)
+    if (H5D__btree_shared_create(idx_info_dst->f, &idx_info_dst->layout->storage.u.chunk,
+                                 &idx_info_dst->layout->u.chunk) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL,
                     "can't create wrapper for destination shared B-tree info");
 
     /* Create the root of the B-tree that describes chunked storage in the dest. file */
     if (H5D__btree_idx_create(idx_info_dst) < 0)
         HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to initialize chunked storage");
-    assert(H5_addr_defined(idx_info_dst->storage->idx_addr));
+    assert(H5_addr_defined(idx_info_dst->layout->storage.u.chunk.idx_addr));
 
 done:
     FUNC_LEAVE_NOAPI_TAG(ret_value)
@@ -1323,16 +1317,16 @@ H5D__btree_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size)
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
     assert(index_size);
 
     /* Initialize B-tree node user-data */
     memset(&udata, 0, sizeof udata);
-    udata.layout  = idx_info->layout;
-    udata.storage = idx_info->storage;
+    udata.layout  = &idx_info->layout->u.chunk;
+    udata.storage = &idx_info->layout->storage.u.chunk;
 
     /* Get metadata information for B-tree */
-    if (H5B_get_info(idx_info->f, H5B_BTREE, idx_info->storage->idx_addr, &bt_info, NULL, &udata) < 0)
+    if (H5B_get_info(idx_info->f, H5B_BTREE, idx_info->layout->storage.u.chunk.idx_addr, &bt_info, NULL,
+                     &udata) < 0)
         HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree");
 
     /* Set the size of the B-tree */
@@ -1408,12 +1402,11 @@ H5D__btree_idx_dest(const H5D_chk_idx_info_t *idx_info)
     assert(idx_info->f);
     assert(idx_info->pline);
     assert(idx_info->layout);
-    assert(idx_info->storage);
 
     /* Free the raw B-tree node buffer */
-    if (NULL == idx_info->storage->u.btree.shared)
+    if (NULL == idx_info->layout->storage.u.chunk.u.btree.shared)
         HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil");
-    if (H5UC_DEC(idx_info->storage->u.btree.shared) < 0)
+    if (H5UC_DEC(idx_info->layout->storage.u.chunk.u.btree.shared) < 0)
         HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page");
 
 done: