Set cframe variable-chunk flag and test fixed-size chunks

Author: FrancescAlted

blosc/frame.c

@@ -115,6 +115,9 @@ void *new_header_frame(blosc2_schunk *schunk, blosc2_frame_s *frame) {
   // General flags
   *h2p = BLOSC2_VERSION_FRAME_FORMAT;  // version
   *h2p += 0x10;  // 64-bit offsets.  We only support this for now.
+  if (schunk->chunksize == 0) {
+    *h2p |= FRAME_VARIABLE_CHUNKS;
+  }
   h2p += 1;
   if (h2p - h2 >= FRAME_HEADER_MINLEN) {
     return NULL;
@@ -1098,6 +1101,12 @@ int64_t frame_from_schunk(blosc2_schunk *schunk, blosc2_frame_s *frame) {
 
   // Now that we know them, fill the chunksize and frame length in header
   to_big(h2 + FRAME_CHUNKSIZE, &chunksize, sizeof(chunksize));
+  if (chunksize == 0) {
+    h2[FRAME_FLAGS] |= FRAME_VARIABLE_CHUNKS;
+  }
+  else {
+    h2[FRAME_FLAGS] &= (uint8_t)~FRAME_VARIABLE_CHUNKS;
+  }
   frame->len = h2len + cbytes + off_cbytes + FRAME_TRAILER_MINLEN;
   if (frame->sframe) {
     frame->len = h2len + off_cbytes + FRAME_TRAILER_MINLEN;

blosc/frame.h

@@ -22,6 +22,8 @@
 
 
 // Constants for metadata placement in header
+#define FRAME_VARIABLE_CHUNKS (1U << 6)
+
 #define FRAME_HEADER_MAGIC 2
 #define FRAME_HEADER_LEN (FRAME_HEADER_MAGIC + 8 + 1)  // 11
 #define FRAME_LEN (FRAME_HEADER_LEN + 4 + 1)  // 16

tests/test_variable_chunks.c

@@ -7,6 +7,7 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "frame.h"
 #include "test_common.h"
 
 int tests_run = 0;
@@ -36,6 +37,12 @@ static const char *updated_values[] = {
     "tiny",
 };
 
+static const char *fixed_values[] = {
+    "one",
+    "two",
+    "six",
+};
+
 static int append_string_chunk(blosc2_schunk *schunk, const char *value) {
   int32_t nbytes = (int32_t)strlen(value) + 1;
   uint8_t *chunk = malloc((size_t)nbytes + BLOSC2_MAX_OVERHEAD);
@@ -72,6 +79,21 @@ static int assert_values(blosc2_schunk *schunk, const char *expected[], int nite
   return 0;
 }
 
+static int assert_variable_chunk_flag(blosc2_schunk *schunk, bool expected) {
+  uint8_t *cframe = NULL;
+  bool needs_free = false;
+  int64_t cframe_len = blosc2_schunk_to_buffer(schunk, &cframe, &needs_free);
+  if (cframe_len < FRAME_HEADER_MINLEN || cframe == NULL) {
+    return -1;
+  }
+
+  bool actual = (cframe[FRAME_FLAGS] & FRAME_VARIABLE_CHUNKS) != 0;
+  if (needs_free) {
+    free(cframe);
+  }
+  return actual == expected ? 0 : -2;
+}
+
 static char *test_variable_chunks(void) {
   blosc2_remove_urlpath(tdata.urlpath);
 
@@ -94,6 +116,8 @@ static char *test_variable_chunks(void) {
   }
 
   mu_assert("ERROR: schunk should switch to variable chunksize mode", schunk->chunksize == 0);
+  mu_assert("ERROR: frame should signal variable chunk sizes",
+            assert_variable_chunk_flag(schunk, true) == 0);
   mu_assert("ERROR: chunk cannot be decompressed", assert_values(schunk, values, 3) == 0);
 
   for (int i = 1; i < 3; ++i) {
@@ -102,6 +126,8 @@ static char *test_variable_chunks(void) {
   }
 
   mu_assert("ERROR: schunk should keep variable chunksize mode", schunk->chunksize == 0);
+  mu_assert("ERROR: updated frame should still signal variable chunk sizes",
+            assert_variable_chunk_flag(schunk, true) == 0);
   mu_assert("ERROR: updated chunk cannot be decompressed", assert_values(schunk, updated_values, 3) == 0);
 
   blosc2_schunk_free(schunk);
@@ -111,6 +137,8 @@ static char *test_variable_chunks(void) {
     mu_assert("ERROR: reopened schunk is NULL", reopened != NULL);
     mu_assert("ERROR: reopened schunk should keep variable chunksize mode", reopened->chunksize == 0);
     mu_assert("ERROR: reopened schunk nchunks mismatch", reopened->nchunks == 3);
+    mu_assert("ERROR: reopened frame should signal variable chunk sizes",
+              assert_variable_chunk_flag(reopened, true) == 0);
     mu_assert("ERROR: reopened chunk cannot be decompressed", assert_values(reopened, updated_values, 3) == 0);
     blosc2_schunk_free(reopened);
   }
@@ -119,10 +147,53 @@ static char *test_variable_chunks(void) {
   return EXIT_SUCCESS;
 }
 
+static char *test_fixed_chunks_flag(void) {
+  blosc2_remove_urlpath(tdata.urlpath);
+
+  blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
+  cparams.typesize = 1;
+  blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
+  blosc2_storage storage = {
+      .contiguous = tdata.contiguous,
+      .urlpath = (char *)tdata.urlpath,
+      .cparams = &cparams,
+      .dparams = &dparams,
+  };
+
+  blosc2_schunk *schunk = blosc2_schunk_new(&storage);
+  mu_assert("ERROR: cannot create schunk", schunk != NULL);
+
+  for (int i = 0; i < 3; ++i) {
+    int nchunks = append_string_chunk(schunk, fixed_values[i]);
+    mu_assert("ERROR: fixed chunk cannot be appended", nchunks == i + 1);
+  }
+
+  mu_assert("ERROR: schunk should keep a fixed chunksize", schunk->chunksize == 4);
+  mu_assert("ERROR: frame should not signal variable chunk sizes",
+            assert_variable_chunk_flag(schunk, false) == 0);
+  mu_assert("ERROR: fixed chunk cannot be decompressed", assert_values(schunk, fixed_values, 3) == 0);
+
+  blosc2_schunk_free(schunk);
+
+  if (tdata.urlpath != NULL) {
+    blosc2_schunk *reopened = blosc2_schunk_open(tdata.urlpath);
+    mu_assert("ERROR: reopened schunk is NULL", reopened != NULL);
+    mu_assert("ERROR: reopened schunk should keep a fixed chunksize", reopened->chunksize == 4);
+    mu_assert("ERROR: reopened frame should not signal variable chunk sizes",
+              assert_variable_chunk_flag(reopened, false) == 0);
+    mu_assert("ERROR: reopened fixed chunk cannot be decompressed", assert_values(reopened, fixed_values, 3) == 0);
+    blosc2_schunk_free(reopened);
+  }
+
+  blosc2_remove_urlpath(tdata.urlpath);
+  return EXIT_SUCCESS;
+}
+
 static char *all_tests(void) {
   for (int i = 0; i < (int)ARRAY_SIZE(backends); ++i) {
     tdata = backends[i];
     mu_run_test(test_variable_chunks);
+    mu_run_test(test_fixed_chunks_flag);
   }
   return EXIT_SUCCESS;
 }