Change decompress_chunk to get_chunk into cbuffer

Author: Luke Shaw

blosc/b2nd.c

@@ -776,10 +776,6 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
   }
 
   uint8_t *buffer_b = buffer;
-  const int64_t *buffer_start = start;
-  const int64_t *buffer_stop = stop;
-  const int64_t *buffer_shape = shape;
-
   int8_t ndim = array->ndim;
 
   // 0-dim case
@@ -884,11 +880,11 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
   int64_t update_nchunks = 1;
   for (int i = 0; i < ndim; ++i) {
     int64_t pos = 0;
-    while (pos <= buffer_start[i]) {
+    while (pos <= start[i]) {
       pos += array->chunkshape[i];
     }
     update_start[i] = pos / array->chunkshape[i] - 1;
-    while (pos < buffer_stop[i]) {
+    while (pos < stop[i]) {
       pos += array->chunkshape[i];
     }
     update_shape[i] = pos / array->chunkshape[i] - update_start[i];
@@ -916,7 +912,7 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
     }
     bool chunk_empty = false;
     for (int i = 0; i < ndim; ++i) {
-      chunk_empty |= (chunk_stop[i] <= buffer_start[i] || chunk_start[i] >= buffer_stop[i]);
+      chunk_empty |= (chunk_stop[i] <= start[i] || chunk_start[i] >= stop[i]);
     }
     if (chunk_empty) {
       continue;
@@ -927,7 +923,7 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
       // Check if all the chunk is going to be updated and avoid the decompression
       bool decompress_chunk = false;
       for (int i = 0; i < ndim; ++i) {
-        decompress_chunk |= (chunk_start[i] < buffer_start[i] || chunk_stop[i] > buffer_stop[i]);
+        decompress_chunk |= (chunk_start[i] < start[i] || chunk_stop[i] > stop[i]);
       }
 
       if (decompress_chunk) {
@@ -1022,8 +1018,8 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
       // compute the start of the slice inside the block
       int64_t slice_start[B2ND_MAX_DIM] = {0};
       for (int i = 0; i < ndim; ++i) {
-        if (block_start[i] < buffer_start[i]) {
-          slice_start[i] = buffer_start[i] - block_start[i];
+        if (block_start[i] < start[i]) {
+          slice_start[i] = start[i] - block_start[i];
         } else {
           slice_start[i] = 0;
         }
@@ -1032,8 +1028,8 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
 
       int64_t slice_stop[B2ND_MAX_DIM] = {0};
       for (int i = 0; i < ndim; ++i) {
-        if (block_stop[i] > buffer_stop[i]) {
-          slice_stop[i] = block_shape[i] - (block_stop[i] - buffer_stop[i]);
+        if (block_stop[i] > stop[i]) {
+          slice_stop[i] = block_shape[i] - (block_stop[i] - stop[i]);
         } else {
           slice_stop[i] = block_stop[i] - block_start[i];
         }
@@ -1046,13 +1042,12 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
       }
 
       uint8_t *src = &buffer_b[0];
-      const int64_t *src_pad_shape = buffer_shape;
 
       int64_t src_start[B2ND_MAX_DIM] = {0};
       int64_t src_stop[B2ND_MAX_DIM] = {0};
       for (int i = 0; i < ndim; ++i) {
-        src_start[i] = slice_start[i] - buffer_start[i];
-        src_stop[i] = slice_stop[i] - buffer_start[i];
+        src_start[i] = slice_start[i] - start[i];
+        src_stop[i] = slice_stop[i] - start[i];
       }
 
       uint8_t *dst = &data[nblock * array->blocknitems * array->sc->typesize];
@@ -1070,12 +1065,12 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const
 
       if (set_slice) {
         b2nd_copy_buffer2(ndim, array->sc->typesize,
-                          src, src_pad_shape, src_start, src_stop,
+                          src, shape, src_start, src_stop,
                           dst, dst_pad_shape, dst_start);
       } else {
         b2nd_copy_buffer2(ndim, array->sc->typesize,
                           dst, dst_pad_shape, dst_start, dst_stop,
-                          src, src_pad_shape, src_start);
+                          src, shape, src_start);
       }
     }
 
@@ -1390,11 +1385,6 @@ int b2nd_concatenate(b2nd_context_t *ctx, const b2nd_array_t *src1, const b2nd_a
   void *buffer = malloc(src2->sc->typesize * src2->extchunknitems);
   BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_MEMORY_ALLOC);
   for (int64_t nchunk = 0; nchunk < src2->sc->nchunks; ++nchunk) {
-    if (blosc2_schunk_decompress_chunk(src2->sc, nchunk, buffer,
-                                      src2->sc->typesize * (int32_t)src2->extchunknitems) <= 0) {
-      BLOSC_TRACE_ERROR("Error decompressing chunk");
-      goto cleanup;
-                                   }
     // Get multidimensional chunk position
     int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
     int64_t chunkshape[B2ND_MAX_DIM] = {0};
@@ -1414,21 +1404,20 @@ int b2nd_concatenate(b2nd_context_t *ctx, const b2nd_array_t *src1, const b2nd_a
       if (stop[i] > src2->shape[i]) {
         stop[i] = src2->shape[i];  // Handle boundary chunks
       }
-
-      // Apply offset only for concatenation axis
-      if (i == axis) {
-        start[i] += src1_shape[i];
-        stop[i] += src1_shape[i];
-      }
     }
+    // Load chunk into buffer
+    BLOSC_ERROR(b2nd_get_slice_cbuffer(src2, start, stop, buffer, chunkshape, src2->sc->chunksize));
+
+    // Apply chunk offset only for concatenation axis
+    start[axis] += src1_shape[axis];
+    stop[axis] += src1_shape[axis];
 
     // Copy the chunk to the correct position
     BLOSC_ERROR(b2nd_set_slice_cbuffer(buffer, chunkshape,
                                        src2->sc->typesize * src2->extchunknitems,
                                        start, stop, *array));
   }
 
-cleanup:
   free(buffer);
 
   return BLOSC2_ERROR_SUCCESS;

blosc/directories.c

@@ -54,12 +54,12 @@
       sprintf(fname, "%s\\%s", dir_path, cfile.name);
 
       ret = remove(fname);
-      free(fname);
       if (ret < 0) {
         BLOSC_TRACE_ERROR("Could not remove file %s", fname);
         _findclose(file);
         return BLOSC2_ERROR_FAILURE;
       }
+      free(fname);
     }
 
     rmdir(dir_path);

tests/b2nd/test_b2nd_concatenate.c

@@ -51,15 +51,46 @@ static int fill_recursive_region(uint8_t *buffer,
 }
 
 /**
- * Fill a region of a multidimensional buffer with a constant value.
+ * Helper function for recursive region filling with arange
+ */
+static int fill_recursive_arange(uint8_t *buffer,
+                                int64_t *strides,
+                                int8_t ndim,
+                                int64_t *start,
+                                int64_t *stop,
+                                int64_t value,
+                                uint8_t typesize,
+                                int dim,
+                                int64_t current_offset) {
+  if (dim == ndim) {
+    // We've reached the innermost dimension, copy the value
+    memcpy(buffer + (current_offset * typesize), &value, typesize);
+    // Increment value for arange
+    return value + 1;
+  }
+
+  // Iterate through the current dimension within the region
+  int64_t val = value;
+  for (int64_t i = start[dim]; i < stop[dim]; i++) {
+    int64_t offset = current_offset + i * strides[dim];
+    val = fill_recursive_arange(buffer, strides, ndim, start, stop,
+                                    val, typesize, dim + 1, offset);
+    if (val < 0) return val;
+  }
+  return val;
+}
+
+/**
+ * Fill a region of a multidimensional buffer with a constant value or linspace.
  */
 int fill_buffer_region(uint8_t *buffer,
                        int64_t *buffer_shape,
                        int8_t ndim,
                        int64_t *start,
                        int64_t *stop,
                        const void *value,
-                       uint8_t typesize) {
+                       uint8_t typesize,
+                       bool arange) {
   // Calculate strides for the buffer
   int64_t strides[B2ND_MAX_DIM];
   strides[ndim - 1] = 1;
@@ -68,8 +99,15 @@ int fill_buffer_region(uint8_t *buffer,
   }
 
   // Start the recursive filling
-  return fill_recursive_region(buffer, strides, ndim, start, stop,
-                              value, typesize, 0, 0);
+  if (arange) {
+    // If arange is true, fill with increasing values
+    return fill_recursive_arange(buffer, strides, ndim, start, stop,
+                                 0, typesize, 0, 0);
+  }
+  else{
+    return fill_recursive_region(buffer, strides, ndim, start, stop,
+                                value, typesize, 0, 0);
+  }
 }
 
 
@@ -79,8 +117,8 @@ CUTEST_TEST_SETUP(concatenate) {
   // Add parametrization
   CUTEST_PARAMETRIZE(typesize, uint8_t, CUTEST_DATA(
       1,
-      // 2,
-      // 4,
+      2,
+      4,
       8,
       13,
   ));
@@ -100,9 +138,9 @@ CUTEST_TEST_SETUP(concatenate) {
       {1, 0, {50}, {25}, {5}, {20}, {25}, {5}},
       {1, 0, {2}, {25}, {5}, {49}, {25}, {5}},
       // 2-dim
-      {2, 0, {50, 50}, {25, 13}, {5, 8}, {50, 50}, {25, 13}, {5, 8}},
-      {2, 1, {50, 50}, {25, 13}, {5, 8}, {50, 50}, {25, 13}, {5, 8}},
-      {2, 0, {25, 50}, {25, 25}, {5, 5}, {2, 50}, {25, 25}, {5, 5}},
+      {2, 0, {10, 10}, {2, 2}, {1, 1}, {4, 10}, {2, 2}, {1, 1}},
+      {2, 1, {10, 8}, {2, 2}, {1, 1}, {10, 8}, {2, 2}, {1, 1}},
+      {2, 0, {4, 4}, {4, 4}, {2, 2}, {4, 4}, {4, 4}, {2, 2}}, //Fails
       {2, 1, {25, 50}, {25, 25}, {5, 5}, {25, 5}, {25, 25}, {5, 5}},
       // 3-dim
       {3, 0, {50, 5, 50}, {25, 13, 10}, {5, 8, 5}, {50, 5, 50}, {25, 13, 10}, {5, 8, 5}},
@@ -111,9 +149,9 @@ CUTEST_TEST_SETUP(concatenate) {
       {3, 0, {5, 5, 50}, {25, 13, 10}, {5, 8, 5}, {51, 5, 50}, {25, 13, 10}, {5, 8, 5}},
       // Inner 0-dims are supported
       {3, 1, {50, 1, 50}, {25, 13, 10}, {5, 8, 5}, {50, 0, 50}, {25, 13, 10}, {5, 8, 5}},
-      // TODO: the next is not working yet
+      // // TODO: the next is not working yet
       // {3, 2, {50, 50, 0}, {25, 13, 10}, {5, 8, 5}, {50, 50, 49}, {25, 13, 10}, {5, 8, 5}},
-      // 4-dim
+      // // 4-dim
       {4, 0, {5, 5, 5, 5}, {2, 5, 10, 5}, {5, 2, 5, 2}, {5, 5, 5, 5}, {5, 5, 10, 5}, {5, 2, 5, 2}},
       {4, 1, {5, 5, 5, 5}, {2, 5, 10, 5}, {5, 2, 5, 2}, {5, 5, 5, 5}, {5, 5, 10, 5}, {5, 2, 5, 2}},
       {4, 2, {5, 5, 5, 5}, {2, 13, 10, 5}, {5, 8, 5, 2}, {5, 5, 5, 5}, {5, 13, 10, 5}, {5, 8, 5, 2}},
@@ -122,18 +160,22 @@ CUTEST_TEST_SETUP(concatenate) {
       {4, 1, {5, 5, 5, 5}, {2, 13, 10, 5}, {5, 8, 5, 2}, {5, 6, 5, 5}, {15, 13, 10, 5}, {5, 8, 5, 2}},
       {4, 2, {5, 5, 5, 5}, {2, 13, 10, 5}, {5, 8, 5, 2}, {5, 5, 6, 5}, {15, 13, 10, 5}, {5, 8, 5, 2}},
       {4, 3, {5, 5, 5, 5}, {2, 13, 10, 5}, {5, 8, 5, 2}, {5, 5, 5, 6}, {15, 13, 10, 5}, {5, 8, 5, 2}},
-
+  //
   ));
   CUTEST_PARAMETRIZE(fill_value, int8_t, CUTEST_DATA(
       3,
      -5,
-//    113,
-//    33,
+   113,
+   33,
   ));
   CUTEST_PARAMETRIZE(copy, bool, CUTEST_DATA(
       true,
       false,
   ));
+  CUTEST_PARAMETRIZE(arange, bool, CUTEST_DATA(
+    true,
+    false,
+));
 
 }
 
@@ -143,6 +185,7 @@ CUTEST_TEST_TEST(concatenate) {
   CUTEST_GET_PARAMETER(typesize, uint8_t);
   CUTEST_GET_PARAMETER(fill_value, int8_t);
   CUTEST_GET_PARAMETER(copy, bool);
+  CUTEST_GET_PARAMETER(arange, bool);
 
   int axis = shapes.axis;
   char *urlpath = "test_concatenate.b2frame";
@@ -188,11 +231,7 @@ CUTEST_TEST_TEST(concatenate) {
 
   /* Create src2 with a value */
   b2nd_array_t *src2;
-  uint8_t *value = malloc(typesize);
-  // Fill a buffer with fill_value
-  for (int i = 0; i < typesize; ++i) {
-    value[i] = fill_value;
-  }
+
   blosc2_storage b2_storage2 = {.cparams=&cparams};
   if (backend.persistent) {
     b2_storage2.urlpath = urlpath2;
@@ -201,8 +240,31 @@ CUTEST_TEST_TEST(concatenate) {
   b2nd_context_t *ctx2 = b2nd_create_ctx(&b2_storage2, shapes.ndim, shapes.shape2,
                                          shapes.chunkshape2, shapes.blockshape2,
                                          NULL, 0, NULL, 0);
-  B2ND_TEST_ASSERT(b2nd_full(ctx2, &src2, value));
 
+  uint8_t *value = malloc(typesize);
+  // Fill a buffer with fill_value
+  for (int i = 0; i < typesize; ++i) {
+    value[i] = fill_value;
+  }
+
+  if (arange) {
+    int64_t buffsize = typesize;
+    for (int i = 0; i < shapes.ndim; ++i) {
+      buffsize *= shapes.shape2[i];  // Multiply by each dimension
+    }
+    // Allocate a buffer for the src2 array
+    uint8_t *buff = malloc(buffsize);
+    int64_t start_[B2ND_MAX_DIM] = {0};
+
+    fill_buffer_region(buff, shapes.shape2, shapes.ndim,
+                     start_, shapes.shape2, value, typesize, arange); // value is not used in this case
+    B2ND_TEST_ASSERT(b2nd_from_cbuffer(ctx2, &src2, buff, buffsize));
+
+  }
+
+  else {
+    B2ND_TEST_ASSERT(b2nd_full(ctx2, &src2, value));
+  }
   /* Concatenate src1 and src2 */
   b2nd_array_t *array = NULL;
   blosc2_storage b2_storage = {.cparams=&cparams};
@@ -234,7 +296,7 @@ CUTEST_TEST_TEST(concatenate) {
 
   // Fill the region with the value
   fill_buffer_region(helperbuffer, helpershape, shapes.ndim,
-                     start_src2, stop_src2, value, typesize);
+                     start_src2, stop_src2, value, typesize, arange); // value is not used if arange is true
 
   // Check the shape of the concatenated array
   for (int i = 0; i < ctx->ndim; ++i) {
@@ -296,6 +358,15 @@ CUTEST_TEST_TEST(concatenate) {
     CUTEST_ASSERT("Data in the concatenated array does not match the helperbuffer", is_true);
   }
 
+  // for (int i = 0; i < buffersize; i++) {
+  //   if (i%typesize == 0) {
+  //     fprintf(stderr, "\n");
+  //     if (i%(typesize*shapes.shape1[0]) == 0) {fprintf(stderr, "--------------------------------\n");}
+  //     fprintf(stderr, "%d ", buffer[i]);
+  //   }
+  // }
+
+
   /* Free mallocs */
   free(buffer_fill);
   free(buffer);