VL-block chunks support lazy loading; the trailer format is identical as original

FrancescAlted · FrancescAlted · commit 0a42e98454df · 2026-03-20T18:51:27.000+01:00
diff --git a/README_CHUNK_FORMAT.rst b/README_CHUNK_FORMAT.rst
@@ -292,7 +292,8 @@ Trailer
 This is an optional section, mainly for lazy chunks use.  A lazy chunk is similar to a regular one, except that
 only the meta-information has been loaded.  The actual data from blocks is 'lazily' only loaded on demand.
 This allows for improved selectivity, and hence less input bandwidth demands, during partial chunk reads
-(e.g. `blosc1_getitem`) from data that is on disk.
+(e.g. `blosc1_getitem`) from data that is on disk.  Both regular chunks and variable-length-block
+(VL-blocks) chunks support lazy loading; the trailer format is identical for both.
 
 It is arranged like this::
 
diff --git a/blosc/blosc2.c b/blosc/blosc2.c
@@ -1724,10 +1724,6 @@ static int blosc_d(
   // Chunks with special values cannot be lazy
   bool is_lazy = ((context->header_overhead == BLOSC_EXTENDED_HEADER_LENGTH) &&
           (context->blosc2_flags & 0x08u) && !context->special_type);
-  if (is_lazy && vlblocks) {
-    BLOSC_TRACE_ERROR("VL-block lazy chunks are not supported yet.");
-    return BLOSC2_ERROR_INVALID_PARAM;
-  }
   if (is_lazy) {
     // The chunk is on disk, so just lazily load the block
     if (context->schunk == NULL) {
@@ -2512,6 +2508,8 @@ static int initialize_context_decompression(blosc2_context* context, blosc_heade
     return rc;
   }
   vlblocks = (context->blosc2_flags2 & BLOSC2_VL_BLOCKS) != 0;
+  bool is_lazy = ((context->header_overhead == BLOSC_EXTENDED_HEADER_LENGTH) &&
+                  (context->blosc2_flags & 0x08u));
 
   /* Check that we have enough space to decompress */
   if (context->sourcesize > (int32_t)context->destsize) {
@@ -2558,7 +2556,7 @@ static int initialize_context_decompression(blosc2_context* context, blosc_heade
   srcsize -= bstarts_end;
 
   /* Read optional dictionary if flag set */
-  if (context->blosc2_flags & BLOSC2_USEDICT) {
+  if ((context->blosc2_flags & BLOSC2_USEDICT) && !is_lazy) {
     context->use_dict = 1;
     // The dictionary section is after the bstarts block: [int32 size | raw bytes]
     if (srcsize < (signed)sizeof(int32_t)) {
@@ -2592,52 +2590,77 @@ static int initialize_context_decompression(blosc2_context* context, blosc_heade
 #endif   // HAVE_ZSTD
     // For LZ4/LZ4HC: dict_buffer and dict_size are sufficient; no digested object needed.
   }
+  else if ((context->blosc2_flags & BLOSC2_USEDICT) && is_lazy) {
+    // Lazy chunks don't carry the dictionary in their buffer (only header+bstarts+trailer).
+    // Reuse dict_ddict from a previous full-chunk initialization of this context if available.
+    context->use_dict = (context->dict_ddict != NULL) ? 1 : 0;
+  }
 
   if (vlblocks && !context->special_type && !memcpyed) {
-    int32_t max_blocksize = 0;
-    int32_t total_nbytes = 0;
-    int32_t prev_bstart = 0;
-    bool is_lazy = ((context->header_overhead == BLOSC_EXTENDED_HEADER_LENGTH) &&
-                    (context->blosc2_flags & 0x08u));
-    if (is_lazy) {
-      BLOSC_TRACE_ERROR("VL-block lazy chunks are not supported yet.");
-      return BLOSC2_ERROR_INVALID_PARAM;
-    }
-
     context->blocknbytes = malloc((size_t)context->nblocks * sizeof(int32_t));
     BLOSC_ERROR_NULL(context->blocknbytes, BLOSC2_ERROR_MEMORY_ALLOC);
     context->blockoffsets = malloc((size_t)context->nblocks * sizeof(int32_t));
     BLOSC_ERROR_NULL(context->blockoffsets, BLOSC2_ERROR_MEMORY_ALLOC);
     context->blockcbytes = malloc((size_t)context->nblocks * sizeof(int32_t));
     BLOSC_ERROR_NULL(context->blockcbytes, BLOSC2_ERROR_MEMORY_ALLOC);
 
-    for (int32_t i = 0; i < context->nblocks; ++i) {
-      int32_t bstart = sw32_(context->bstarts + i);
-      int32_t next_bstart = (i + 1 < context->nblocks) ? sw32_(context->bstarts + i + 1) : header->cbytes;
-      if (bstart < bstarts_end || bstart <= prev_bstart || next_bstart <= bstart ||
-          next_bstart > header->cbytes || bstart > context->srcsize - (int32_t)sizeof(int32_t)) {
-        BLOSC_TRACE_ERROR("Invalid VL-block offsets in chunk.");
-        return BLOSC2_ERROR_INVALID_HEADER;
+    if (is_lazy) {
+      // Lazy VL: block data is on disk, so blocknbytes is unknown at this point.
+      // Populate blockcbytes from bstarts differences; blocksize gets max(blockcbytes)
+      // as a safe upper bound so tmp buffers are large enough for the lazy block read.
+      int32_t max_csize = 0;
+      for (int32_t i = 0; i < context->nblocks; ++i) {
+        int32_t bstart = sw32_(context->bstarts + i);
+        int32_t next_bstart = (i + 1 < context->nblocks) ?
+            sw32_(context->bstarts + i + 1) : header->cbytes;
+        if (bstart < bstarts_end || next_bstart <= bstart ||
+            next_bstart > header->cbytes) {
+          BLOSC_TRACE_ERROR("Invalid VL-block offsets in lazy chunk.");
+          return BLOSC2_ERROR_INVALID_HEADER;
+        }
+        context->blocknbytes[i] = 0;   // unknown until block is read from disk
+        context->blockoffsets[i] = 0;  // unknown
+        context->blockcbytes[i] = next_bstart - bstart;
+        if (context->blockcbytes[i] > max_csize) {
+          max_csize = context->blockcbytes[i];
+        }
       }
-      context->blockoffsets[i] = total_nbytes;
-      context->blocknbytes[i] = sw32_(context->src + bstart);
-      context->blockcbytes[i] = next_bstart - bstart;
-      if (context->blocknbytes[i] <= 0) {
-        BLOSC_TRACE_ERROR("Invalid VL-block uncompressed size in chunk.");
-        return BLOSC2_ERROR_INVALID_HEADER;
+      context->blocksize = max_csize;
+      context->leftover = 0;
+    }
+    else {
+      int32_t max_blocksize = 0;
+      int32_t total_nbytes = 0;
+      int32_t prev_bstart = 0;
+      for (int32_t i = 0; i < context->nblocks; ++i) {
+        int32_t bstart = sw32_(context->bstarts + i);
+        int32_t next_bstart = (i + 1 < context->nblocks) ?
+            sw32_(context->bstarts + i + 1) : header->cbytes;
+        if (bstart < bstarts_end || bstart <= prev_bstart || next_bstart <= bstart ||
+            next_bstart > header->cbytes || bstart > context->srcsize - (int32_t)sizeof(int32_t)) {
+          BLOSC_TRACE_ERROR("Invalid VL-block offsets in chunk.");
+          return BLOSC2_ERROR_INVALID_HEADER;
+        }
+        context->blockoffsets[i] = total_nbytes;
+        context->blocknbytes[i] = sw32_(context->src + bstart);
+        context->blockcbytes[i] = next_bstart - bstart;
+        if (context->blocknbytes[i] <= 0) {
+          BLOSC_TRACE_ERROR("Invalid VL-block uncompressed size in chunk.");
+          return BLOSC2_ERROR_INVALID_HEADER;
+        }
+        total_nbytes += context->blocknbytes[i];
+        if (context->blocknbytes[i] > max_blocksize) {
+          max_blocksize = context->blocknbytes[i];
+        }
+        prev_bstart = bstart;
       }
-      total_nbytes += context->blocknbytes[i];
-      if (context->blocknbytes[i] > max_blocksize) {
-        max_blocksize = context->blocknbytes[i];
+      if (total_nbytes != context->sourcesize) {
+        BLOSC_TRACE_ERROR("VL-block sizes do not add up to chunk nbytes.");
+        return BLOSC2_ERROR_INVALID_HEADER;
       }
-      prev_bstart = bstart;
-    }
-    if (total_nbytes != context->sourcesize) {
-      BLOSC_TRACE_ERROR("VL-block sizes do not add up to chunk nbytes.");
-      return BLOSC2_ERROR_INVALID_HEADER;
+      context->blocksize = max_blocksize;
+      context->leftover = 0;
     }
-    context->blocksize = max_blocksize;
-    context->leftover = 0;
   }
 
   return 0;
@@ -3755,6 +3778,74 @@ static int decompress_single_vlblock(blosc2_context* context, int32_t nblock,
     return BLOSC2_ERROR_INVALID_PARAM;
   }
 
+  // For lazy VL chunks blocknbytes[nblock] == 0 because the uncompressed size is
+  // stored as the first 4 bytes of the block span on disk, not in the in-memory
+  // header.  Peek at the file to resolve it before we can allocate the output buffer.
+  if (context->blocknbytes[nblock] == 0) {
+    if (context->schunk == NULL || context->schunk->frame == NULL) {
+      BLOSC_TRACE_ERROR("Lazy VL block needs an associated super-chunk with a frame.");
+      return BLOSC2_ERROR_INVALID_PARAM;
+    }
+    blosc2_frame_s* frame = (blosc2_frame_s*)context->schunk->frame;
+    blosc2_io_cb *io_cb = blosc2_get_io_cb(context->schunk->storage->io->id);
+    if (io_cb == NULL) {
+      BLOSC_TRACE_ERROR("Error getting the input/output API");
+      return BLOSC2_ERROR_PLUGIN_IO;
+    }
+
+    // Lazy chunk trailer: [nchunk int32 | chunk_offset int64 | block_csizes int32*N]
+    int32_t trailer_offset = BLOSC_EXTENDED_HEADER_LENGTH +
+                             context->nblocks * (int32_t)sizeof(int32_t);
+    int32_t nchunk_lazy = *(const int32_t*)(context->src + trailer_offset);
+    int64_t chunk_offset = *(const int64_t*)(context->src + trailer_offset +
+                                             (int32_t)sizeof(int32_t));
+    int32_t bstart = sw32_(context->bstarts + nblock);
+
+    void* fp = NULL;
+    int64_t io_pos;
+    if (frame->sframe) {
+      char* chunkpath = malloc(strlen(frame->urlpath) + 1 + 8 + strlen(".chunk") + 1);
+      BLOSC_ERROR_NULL(chunkpath, BLOSC2_ERROR_MEMORY_ALLOC);
+      sprintf(chunkpath, "%s/%08X.chunk", frame->urlpath, nchunk_lazy);
+      fp = io_cb->open(chunkpath, "rb", context->schunk->storage->io->params);
+      free(chunkpath);
+      io_pos = bstart;
+    }
+    else {
+      fp = io_cb->open(frame->urlpath, "rb", context->schunk->storage->io->params);
+      io_pos = frame->file_offset + chunk_offset + bstart;
+    }
+    if (fp == NULL) {
+      BLOSC_TRACE_ERROR("Cannot open frame file for lazy VL block size peek.");
+      return BLOSC2_ERROR_FILE_OPEN;
+    }
+
+    // Read only the 4-byte uncompressed-size prefix of the block span.
+    uint8_t nbuf[sizeof(int32_t)];
+    uint8_t* nbufp = nbuf;
+    int64_t rbytes = io_cb->read((void**)&nbufp, 1, sizeof(int32_t), io_pos, fp);
+    io_cb->close(fp);
+    if (nbufp != nbuf) {
+      // io_cb allocated new memory; copy the result and free.
+      memcpy(nbuf, nbufp, sizeof(int32_t));
+      free(nbufp);
+    }
+    if (rbytes != (int64_t)sizeof(int32_t)) {
+      BLOSC_TRACE_ERROR("Cannot read VL-block uncompressed-size prefix from disk.");
+      return BLOSC2_ERROR_FILE_READ;
+    }
+    int32_t neblock = sw32_(nbuf);
+    if (neblock <= 0) {
+      BLOSC_TRACE_ERROR("Invalid VL-block uncompressed size read from disk.");
+      return BLOSC2_ERROR_INVALID_HEADER;
+    }
+    context->blocknbytes[nblock] = neblock;
+    // Keep blocksize as an upper bound for tmp buffer allocation.
+    if (neblock > context->blocksize) {
+      context->blocksize = neblock;
+    }
+  }
+
   int32_t bsize = context->blocknbytes[nblock];
   uint8_t* buf = malloc((size_t)bsize);
   if (buf == NULL) {
diff --git a/include/blosc2.h b/include/blosc2.h
@@ -1570,8 +1570,12 @@ BLOSC_EXPORT int blosc2_vlchunk_get_nblocks(const void* src, int32_t srcsize,
  * in the chunk are untouched.
  *
  * @param context A decompression context (#blosc2_context created with
- *   #blosc2_create_dctx).
+ *   #blosc2_create_dctx).  If @p src is a lazy chunk (obtained via
+ *   #blosc2_schunk_get_lazychunk), the caller must set @c context->schunk
+ *   to the owning super-chunk (which must have an associated frame) before
+ *   calling this function; the frame is used to read block data from disk.
  * @param src The buffer of compressed data.  Must carry the #BLOSC2_VL_BLOCKS flag.
+ *   May be a fully in-memory chunk or a lazy chunk proxy.
  * @param srcsize The length of the compressed data buffer.
  * @param nblock Zero-based index of the block to decompress.
  * @param dest On success, points to a newly allocated buffer containing the
diff --git a/tests/test_vlblocks.c b/tests/test_vlblocks.c
@@ -679,6 +679,140 @@ static char *test_vldecompress_block_ctx_dict(void) {
 #endif  /* HAVE_ZSTD */
 
 
+/* ---- Lazy VL chunk tests ------------------------------------------------- */
+
+/* Helper: build a 2-chunk file-backed schunk with 3 VL blocks per chunk.
+ * Returns the schunk (caller must blosc2_schunk_free it), or NULL on error. */
+static blosc2_schunk* make_lazy_vl_schunk(const char* urlpath, bool contiguous) {
+  blosc2_remove_urlpath(urlpath);
+
+  blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
+  cparams.typesize = 1;
+  blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
+  blosc2_storage storage = {
+      .contiguous = contiguous,
+      .urlpath = (char *)urlpath,
+      .cparams = &cparams,
+      .dparams = &dparams,
+  };
+
+  blosc2_schunk* schunk = blosc2_schunk_new(&storage);
+  if (schunk == NULL) return NULL;
+
+  blosc2_context* cctx = blosc2_create_cctx(cparams);
+  if (cctx == NULL) { blosc2_schunk_free(schunk); return NULL; }
+
+  int32_t destsize = total_nbytes() + BLOSC2_MAX_OVERHEAD + 64;
+  uint8_t* chunk = malloc((size_t)destsize);
+  if (chunk == NULL) { blosc2_free_ctx(cctx); blosc2_schunk_free(schunk); return NULL; }
+
+  int32_t cbytes = blosc2_vlcompress_ctx(cctx, (const void * const *)srcs, srcsizes, 3,
+                                         chunk, destsize);
+  if (cbytes <= 0 || blosc2_schunk_append_chunk(schunk, chunk, true) < 0) {
+    free(chunk); blosc2_free_ctx(cctx); blosc2_schunk_free(schunk); return NULL;
+  }
+  /* Append a second chunk (same data) so nchunks > 1. */
+  cbytes = blosc2_vlcompress_ctx(cctx, (const void * const *)srcs, srcsizes, 3,
+                                 chunk, destsize);
+  if (cbytes <= 0 || blosc2_schunk_append_chunk(schunk, chunk, true) < 0) {
+    free(chunk); blosc2_free_ctx(cctx); blosc2_schunk_free(schunk); return NULL;
+  }
+  free(chunk);
+  blosc2_free_ctx(cctx);
+  return schunk;
+}
+
+static char *test_lazy_vlchunk_get_nblocks(void) {
+  const char* urlpath = "test_lazy_vlnblocks.b2frame";
+  blosc2_schunk* schunk = make_lazy_vl_schunk(urlpath, true);
+  mu_assert("ERROR: cannot create lazy VL schunk", schunk != NULL);
+
+  for (int64_t nchunk = 0; nchunk < schunk->nchunks; ++nchunk) {
+    uint8_t* lazy_chunk = NULL;
+    bool needs_free = false;
+    int cbytes = blosc2_schunk_get_lazychunk(schunk, nchunk, &lazy_chunk, &needs_free);
+    mu_assert("ERROR: blosc2_schunk_get_lazychunk failed", cbytes > 0);
+
+    int32_t nblocks = -1;
+    int rc = blosc2_vlchunk_get_nblocks(lazy_chunk, cbytes, &nblocks);
+    mu_assert("ERROR: blosc2_vlchunk_get_nblocks failed on lazy chunk", rc == 0);
+    mu_assert("ERROR: wrong nblocks from lazy chunk", nblocks == 3);
+
+    if (needs_free) free(lazy_chunk);
+  }
+
+  blosc2_schunk_free(schunk);
+  blosc2_remove_urlpath(urlpath);
+  return EXIT_SUCCESS;
+}
+
+
+static char *test_lazy_vldecompress_block_ctx(void) {
+  const char* urlpath = "test_lazy_vldecomp.b2frame";
+  blosc2_schunk* schunk = make_lazy_vl_schunk(urlpath, true);
+  mu_assert("ERROR: cannot create lazy VL schunk", schunk != NULL);
+
+  /* Use the schunk's own dctx, which already has schunk->dctx->schunk set. */
+  blosc2_context* dctx = schunk->dctx;
+
+  for (int64_t nchunk = 0; nchunk < schunk->nchunks; ++nchunk) {
+    uint8_t* lazy_chunk = NULL;
+    bool needs_free = false;
+    int cbytes = blosc2_schunk_get_lazychunk(schunk, nchunk, &lazy_chunk, &needs_free);
+    mu_assert("ERROR: blosc2_schunk_get_lazychunk failed", cbytes > 0);
+
+    for (int i = 0; i < 3; ++i) {
+      uint8_t* blk = NULL;
+      int32_t blksize = -1;
+      int rc = blosc2_vldecompress_block_ctx(dctx, lazy_chunk, cbytes, i, &blk, &blksize);
+      mu_assert("ERROR: lazy blosc2_vldecompress_block_ctx failed", rc == srcsizes[i]);
+      mu_assert("ERROR: lazy returned size mismatch", blksize == srcsizes[i]);
+      mu_assert("ERROR: lazy content mismatch",
+                memcmp(blk, srcs[i], (size_t)srcsizes[i]) == 0);
+      free(blk);
+    }
+    if (needs_free) free(lazy_chunk);
+  }
+
+  blosc2_schunk_free(schunk);
+  blosc2_remove_urlpath(urlpath);
+  return EXIT_SUCCESS;
+}
+
+
+static char *test_lazy_vldecompress_block_ctx_sframe(void) {
+  const char* urlpath = "test_lazy_vldecomp_s.b2frame";
+  blosc2_schunk* schunk = make_lazy_vl_schunk(urlpath, false);
+  mu_assert("ERROR: cannot create lazy VL sframe schunk", schunk != NULL);
+
+  /* Use the schunk's own dctx. */
+  blosc2_context* dctx = schunk->dctx;
+
+  for (int64_t nchunk = 0; nchunk < schunk->nchunks; ++nchunk) {
+    uint8_t* lazy_chunk = NULL;
+    bool needs_free = false;
+    int cbytes = blosc2_schunk_get_lazychunk(schunk, nchunk, &lazy_chunk, &needs_free);
+    mu_assert("ERROR: sframe blosc2_schunk_get_lazychunk failed", cbytes > 0);
+
+    for (int i = 0; i < 3; ++i) {
+      uint8_t* blk = NULL;
+      int32_t blksize = -1;
+      int rc = blosc2_vldecompress_block_ctx(dctx, lazy_chunk, cbytes, i, &blk, &blksize);
+      mu_assert("ERROR: sframe lazy blosc2_vldecompress_block_ctx failed", rc == srcsizes[i]);
+      mu_assert("ERROR: sframe lazy returned size mismatch", blksize == srcsizes[i]);
+      mu_assert("ERROR: sframe lazy content mismatch",
+                memcmp(blk, srcs[i], (size_t)srcsizes[i]) == 0);
+      free(blk);
+    }
+    if (needs_free) free(lazy_chunk);
+  }
+
+  blosc2_schunk_free(schunk);
+  blosc2_remove_urlpath(urlpath);
+  return EXIT_SUCCESS;
+}
+
+
 static char *all_tests(void) {
   for (int i = 0; i < (int)ARRAY_SIZE(backends); ++i) {
     tdata = backends[i];
@@ -690,6 +824,9 @@ static char *all_tests(void) {
     mu_run_test(test_schunk_get_vlblock);
   }
   mu_run_test(test_vlblocks_mt_roundtrip);
+  mu_run_test(test_lazy_vlchunk_get_nblocks);
+  mu_run_test(test_lazy_vldecompress_block_ctx);
+  mu_run_test(test_lazy_vldecompress_block_ctx_sframe);
 #ifdef HAVE_ZSTD
   mu_run_test(test_vlblocks_dict_roundtrip);
   mu_run_test(test_vldecompress_block_ctx_dict);
