Merge pull request #180 from mulimoen/feature/chunking

Add example for chunking of data

Author: mulimoen

.github/workflows/ci.yml

@@ -134,8 +134,10 @@ jobs:
       - name: Build and test with filters
         run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc --exclude hdf5-derive
         if: matrix.rust != 'stable-gnu'
-      - name: Run example
-        run: cargo r --example simple --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc
+      - name: Run examples
+        run: |
+          cargo r --example simple --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc
+          cargo r --example chunking --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc
         if: matrix.rust != 'stable-gnu'
 
   apt:

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Unreleased
+
+### Added
+
+- `Error` now implements `From<Infallible>`, which allows passing convertible
+  extents (like tuples of integers) where `impl TryInto<Extents>` is required.
+
 ## 0.8.0
 
 Release date: Oct 23, 2021.

examples/chunking.rs

@@ -0,0 +1,38 @@
+//! Create, write, and read a chunked dataset
+
+use hdf5::{File, Result};
+use ndarray::Array2;
+
+fn main() -> Result<()> {
+    let file = File::create("chunking.h5")?;
+
+    let (ny, nx) = (100, 100);
+    let arr = Array2::from_shape_fn((ny, nx), |(j, i)| (1000 * j + i) as f32);
+
+    let ds = file
+        .new_dataset::<f32>()
+        .chunk((1, ny, nx))  // each chunk contains ny * nx elements
+        .shape((1.., ny, nx)) // first axis is unlimited with initial size of 1
+        .deflate(3)
+        .create("variable")?;
+
+    // writing one chunk at a time is the most efficient
+    ds.write_slice(&arr, (0, .., ..))?;
+
+    // dataset can be resized along an unlimited dimension
+    ds.resize((10, ny, nx))?;
+    ds.write_slice(&arr, (1, .., ..))?;
+
+    let chunksize = ds.chunk().unwrap();
+    assert_eq!(chunksize, &[1, ny, nx]);
+
+    let shape = ds.shape();
+    assert_eq!(shape, &[10, ny, nx]);
+
+    // it's best to read from a chunked dataset in a chunk-wise fashion
+    for k in 0..shape[0] {
+        let _arr: Array2<f32> = ds.read_slice((k, .., ..))?;
+    }
+
+    Ok(())
+}

src/error.rs

@@ -1,3 +1,4 @@
+use std::convert::Infallible;
 use std::error::Error as StdError;
 use std::fmt;
 use std::ops::Deref;
@@ -229,6 +230,12 @@ impl From<String> for Error {
     }
 }
 
+impl From<Infallible> for Error {
+    fn from(_: Infallible) -> Self {
+        unreachable!("Infallible error can never be constructed")
+    }
+}
+
 impl fmt::Debug for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match *self {