Merge pull request #490 from thewtex/image-to-string

.iwi.cbor.zstd Image IO support

Author: thewtex

CMakeLists.txt

@@ -73,7 +73,7 @@ FetchContent_Declare(
 )
 set(WITH_TESTS OFF CACHE BOOL "Build libcbor tests")
 set(WITH_EXAMPLES OFF CACHE BOOL "Build libcbor examples")
-
+set(SANITIZE OFF CACHE BOOL "Enable sanitizers in Debug mode in libcbor")
 
 list(APPEND CMAKE_MODULE_PATH ${libcbor_SOURCE_DIR}/CMakeModules)
 FetchContent_MakeAvailable(rapidjson_lib cli11 rang libcbor)

include/itkWASMImageIO.h

@@ -27,11 +27,13 @@ namespace itk
 {
 /** \class WASMImageIO
  *
- * \brief Read and write the an itk::Image in  format.
+ * \brief Read and write an itk::Image in a web-friendly format.
  *
- * This format is intended to facilitage data exchange in itk-wasm.
- * It reads and writes an itk-wasm Image object where TypedArrays are
- * replaced by binary files on the file system or in a ZIP file.
+ * This format is intended to facilitate data exchange in itk-wasm.
+ * It reads and writes an itk-wasm Image object in a CbOR file on the
+ * filesystem with JSON files and binary files for TypedArrays.
+ * 
+ * The file extensions used are .iwi and .iwi.cbor.
  *
  * \ingroup IOFilters
  * \ingroup WebAssemblyInterface
@@ -94,8 +96,8 @@ class WebAssemblyInterface_EXPORT WASMImageIO: public StreamingImageIOBase
     return 0;
   }
 
-  void ReadCBOR(void * buffer = nullptr);
-  void WriteCBOR(const void * buffer = nullptr);
+  void ReadCBOR(void * buffer = nullptr, unsigned char * cborBuffer = nullptr, size_t cborBufferLength = 0);
+  size_t WriteCBOR(const void * buffer = nullptr, unsigned char ** cborBuffer = nullptr, bool allocateCBORBuffer = false);
 
 private:
   ITK_DISALLOW_COPY_AND_ASSIGN(WASMImageIO);

include/itkWASMStringStream.h

@@ -20,6 +20,7 @@
 
 #include "itkWASMDataObject.h"
 #include "rapidjson/document.h"
+#include <string_view>
 
 namespace itk
 {

src/CMakeLists.txt

@@ -29,6 +29,8 @@ if(BUILD_ITK_WASM_IO_MODULES)
 set(common_link_flags " -s ALLOW_MEMORY_GROWTH=1 -s FORCE_FILESYSTEM=1 -s MODULARIZE=1 -s WASM=1 -lnodefs.js -s EXIT_RUNTIME=0 -s INVOKE_RUN=1 --post-js ${CMAKE_CURRENT_SOURCE_DIR}/emscripten-module/itkJSPost.js")
 set(esm_link_flags " -s EXPORT_ES6=1 -s USE_ES6_IMPORT_META=0")
 
+include(${CMAKE_CURRENT_SOURCE_DIR}/io/internal/pipelines/common/CompressStringify/BuildZstd.cmake)
+add_subdirectory(io/internal/pipelines/common/CompressStringify)
 add_subdirectory(io/internal/pipelines/image/ConvertImage)
 add_subdirectory(io/internal/pipelines/image/ReadDICOM)
 

src/io/getFileExtension.ts

@@ -6,6 +6,10 @@ function getFileExtension (filePath: string): string {
   } else if (extension.toLowerCase() === 'cbor') {
     const index = filePath.slice(0, -5).lastIndexOf('.')
     extension = filePath.slice((index - 1 >>> 0) + 2)
+  } else if (extension.toLowerCase() === 'zstd') {
+    // .iwi.cbor.zstd
+    const index = filePath.slice(0, -10).lastIndexOf('.')
+    extension = filePath.slice((index - 1 >>> 0) + 2)
   } else if (extension.toLowerCase() === 'zip') {
     const index = filePath.slice(0, -4).lastIndexOf('.')
     extension = filePath.slice((index - 1 >>> 0) + 2)

src/io/internal/extensionToImageIO.ts

@@ -17,6 +17,7 @@ const extensionToIO = new Map([
 
   ['iwi', 'itkWASMImageIO'],
   ['iwi.cbor', 'itkWASMImageIO'],
+  ['iwi.cbor.zstd', 'itkWASMZstdImageIO'],
 
   ['lsm', 'itkLSMImageIO'],
 

src/io/internal/pipelines/common/CompressStringify/BuildZstd.cmake

@@ -0,0 +1,25 @@
+include(FetchContent)
+option(ZSTD_BUILD_CONTRIB "BUILD_CONTRIB" OFF)
+option(ZSTD_BUILD_PROGRAMS "BUILD_PROGRAMS" OFF)
+option(ZSTD_BUILD_SHARED "BUILD_SHARED" OFF)
+option(ZSTD_BUILD_STATIC "BUILD_STATIC" ON)
+option(ZSTD_BUILD_TESTS "BUILD_TESTS" OFF)
+option(ZSTD_BUILD_LEGACY_SUPPORT "BUILD_LEGACY_SUPPORT" OFF)
+option(ZSTD_MULTITHREAD_SUPPORT "BUILD_MULTITHREAD_SUPPORT" OFF)
+option(ZSTD_BUILD_PROGRAMS_LINK_SHARED "BUILD_PROGRAMS_LINK_SHARED" OFF)
+option(ZSTD_BUILD_LZ4 "BUILD_LZ4" OFF)
+option(ZSTD_BUILD_LZMA "BUILD_LZMA" OFF)
+option(ZSTD_BUILD_ZLIB "BUILD_ZLIB" OFF)
+set(zstd_GIT_REPOSITORY "https://github.com/facebook/zstd.git")
+# v1.5.2
+set(zstd_GIT_TAG c9c7be85f49f45a581ec00c309afda5c62ba9ef2)
+FetchContent_Declare(
+  zstd_lib
+  GIT_REPOSITORY ${zstd_GIT_REPOSITORY}
+  GIT_TAG        ${zstd_GIT_TAG}
+)
+
+FetchContent_MakeAvailable(zstd_lib)
+set(zstd_lib_INCLUDE_DIR "${zstd_lib_SOURCE_DIR}/lib")
+include_directories(${zstd_lib_INCLUDE_DIR})
+add_subdirectory("${zstd_lib_SOURCE_DIR}/build/cmake" "${zstd_lib_BINARY_DIR}")

src/io/internal/pipelines/common/CompressStringify/CMakeLists.txt

@@ -0,0 +1,42 @@
+cmake_minimum_required(VERSION 3.16)
+project(CompressStringify)
+
+set(CMAKE_CXX_STANDARD 17)
+
+include(FetchContent)
+if (NOT TARGET libzstd_static)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/BuildZstd.cmake)
+endif()
+
+set(cpp_base64_GIT_REPOSITORY "https://github.com/thewtex/cpp-base64.git")
+set(cpp_base64_GET_TAG 9144cd53be930b37235ae552a92b5d2aa51e9325)
+FetchContent_Declare(
+  cpp_base64
+  GIT_REPOSITORY ${cpp_base64_GIT_REPOSITORY}
+  GIT_TAG        ${cpp_base64_GIT_TAG}
+)
+
+FetchContent_MakeAvailable(cpp_base64)
+
+find_package(ITK REQUIRED COMPONENTS WebAssemblyInterface)
+include(${ITK_USE_FILE})
+
+add_executable(CompressStringify CompressStringify.cxx)
+target_include_directories(CompressStringify PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${zstd_lib_INCLUDE_DIR})
+target_link_libraries(CompressStringify PUBLIC libzstd_static cpp-base64 ${ITK_LIBRARIES})
+
+add_executable(ParseStringDecompress ParseStringDecompress.cxx)
+target_include_directories(ParseStringDecompress PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${zstd_lib_INCLUDE_DIR})
+target_link_libraries(ParseStringDecompress PUBLIC libzstd_static cpp-base64 ${ITK_LIBRARIES})
+
+if (DEFINED WebAssemblyInterface_SOURCE_DIR)
+  foreach(target CompressStringify CompressStringify.umd ParseStringDecompress ParseStringDecompress.umd)
+    itk_module_target_label(${target})
+    itk_module_target_export(${target})
+    itk_module_target_install(${target})
+    set_property(TARGET ${target}
+      PROPERTY RUNTIME_OUTPUT_DIRECTORY
+      ${WebAssemblyInterface_BINARY_DIR}/image-io
+      )
+  endforeach()
+endif()

src/io/internal/pipelines/common/CompressStringify/CompressStringify.cxx

@@ -0,0 +1,110 @@
+/*=========================================================================
+
+ *  Copyright NumFOCUS
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *=========================================================================*/
+
+#include <fstream>
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <iterator>
+
+#include "zstd.h"
+#include "cpp-base64/base64.h"
+
+#include "itkPipeline.h"
+#include "itkInputBinaryStream.h"
+#include "itkOutputTextStream.h"
+#include "itkOutputBinaryStream.h"
+
+int compress(itk::wasm::Pipeline & pipeline, itk::wasm::InputBinaryStream & inputBinaryStream, int compressionLevel)
+{
+  itk::wasm::OutputBinaryStream outputBinaryStream;
+  pipeline.add_option("Output", outputBinaryStream, "Output compressed binary");
+
+  ITK_WASM_PARSE(pipeline);
+
+  std::string inputBinary;
+  inputBinary.assign( (std::istreambuf_iterator<char>(inputBinaryStream.Get()) ), 
+                      (std::istreambuf_iterator<char>()) ); 
+
+
+  const size_t compressedBufferSize = ZSTD_compressBound(inputBinary.size());
+  std::vector<char> compressedBinary(compressedBufferSize);
+
+  const size_t compressedSize = ZSTD_compress(compressedBinary.data(), compressedBufferSize, inputBinary.data(), inputBinary.size(), compressionLevel);
+  compressedBinary.resize(compressedSize);
+
+  std::ostream_iterator<char> oIt(outputBinaryStream.Get());
+  std::copy(compressedBinary.begin(), compressedBinary.end(), oIt);
+
+  return EXIT_SUCCESS;
+}
+
+int compressStringify(itk::wasm::Pipeline & pipeline, itk::wasm::InputBinaryStream & inputBinaryStream, int compressionLevel, const std::string & dataURLPrefix)
+{
+  itk::wasm::OutputTextStream outputTextStream;
+  pipeline.add_option("Output", outputTextStream, "Output dataURL+base64 compressed binary");
+
+  ITK_WASM_PARSE(pipeline);
+
+  std::string inputBinary;
+  inputBinary.assign( (std::istreambuf_iterator<char>(inputBinaryStream.Get()) ), 
+                      (std::istreambuf_iterator<char>()) ); 
+
+
+  const size_t compressedBufferSize = ZSTD_compressBound(inputBinary.size());
+  std::string compressedBinary;
+  compressedBinary.resize(compressedBufferSize);
+
+  const size_t compressedSize = ZSTD_compress(compressedBinary.data(), compressedBufferSize, inputBinary.data(), inputBinary.size(), compressionLevel);
+  compressedBinary.resize(compressedSize);
+
+  // Do we want/need this?
+  constexpr bool urlFriendly = false;
+  auto outputText = base64_encode(compressedBinary, urlFriendly);
+
+  outputTextStream.Get() << dataURLPrefix;
+  outputTextStream.Get() << outputText;
+
+  return EXIT_SUCCESS;
+}
+
+int main(int argc, char * argv[])
+{
+  itk::wasm::Pipeline pipeline("Given a binary, compress optionally base64 encode", argc, argv);
+
+  itk::wasm::InputBinaryStream inputBinaryStream;
+  pipeline.add_option("InputBinary", inputBinaryStream, "Input binary");
+
+  bool stringify = false;
+  pipeline.add_flag("-s,--stringify", stringify, "Stringify the output");
+
+  int compressionLevel = 3;
+  pipeline.add_option("-c,--compression-level", compressionLevel, "Compression level, typically 1-9");
+
+  std::string dataURLPrefix("data:application/iwi+cbor+zstd;base64,");
+  pipeline.add_option("-p,--data-url-prefix", dataURLPrefix, "dataURL prefix");
+
+  ITK_WASM_PRE_PARSE(pipeline);
+
+  if(stringify)
+  {
+    return compressStringify(pipeline, inputBinaryStream, compressionLevel, dataURLPrefix);
+  }
+  return compress(pipeline, inputBinaryStream, compressionLevel);
+}

src/io/internal/pipelines/common/CompressStringify/ParseStringDecompress.cxx

@@ -0,0 +1,112 @@
+/*=========================================================================
+ *
+ *  Copyright NumFOCUS
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *=========================================================================*/
+
+#include <fstream>
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <iterator>
+
+#include "zstd.h"
+#include "cpp-base64/base64.h"
+
+#include "itkPipeline.h"
+#include "itkInputBinaryStream.h"
+#include "itkInputTextStream.h"
+#include "itkOutputBinaryStream.h"
+
+int decompress(itk::wasm::Pipeline & pipeline)
+{
+  itk::wasm::InputBinaryStream inputBinaryStream;
+  pipeline.add_option("Input", inputBinaryStream, "Compressed input");
+
+  itk::wasm::OutputBinaryStream outputBinaryStream;
+  pipeline.add_option("Output", outputBinaryStream, "Output decompressed binary");
+
+  ITK_WASM_PARSE(pipeline);
+
+  std::string inputBinary;
+  inputBinary.assign( (std::istreambuf_iterator<char>(inputBinaryStream.Get()) ), 
+                      (std::istreambuf_iterator<char>()) ); 
+
+
+  const size_t decompressedBufferSize = ZSTD_getFrameContentSize(inputBinary.data(), inputBinary.size());
+  std::vector<char> decompressedBinary(decompressedBufferSize);
+
+  const size_t decompressedSize = ZSTD_decompress(decompressedBinary.data(), decompressedBufferSize, inputBinary.data(), inputBinary.size());
+  decompressedBinary.resize(decompressedSize);
+
+  std::ostream_iterator<char> oIt(outputBinaryStream.Get());
+  std::copy(decompressedBinary.begin(), decompressedBinary.end(), oIt);
+
+  return EXIT_SUCCESS;
+}
+
+int decodeDecompress(itk::wasm::Pipeline & pipeline)
+{
+  itk::wasm::InputTextStream inputTextStream;
+  pipeline.add_option("Input", inputTextStream, "Compressed input");
+
+  itk::wasm::OutputBinaryStream outputBinaryStream;
+  pipeline.add_option("Output", outputBinaryStream, "Output decompressed binary");
+
+  ITK_WASM_PARSE(pipeline);
+
+  // Skip dataURLPrefix
+  auto inputTextIt = std::istream_iterator<char>(inputTextStream.Get());
+  while (*inputTextIt != ',')
+  {
+    inputTextIt++;
+  }
+  inputTextIt++;
+
+  std::string inputText;
+  inputText.assign( (inputTextIt), 
+                    (std::istream_iterator<char>()) ); 
+
+  auto inputBinary = base64_decode(inputText);
+
+  const size_t decompressedBufferSize = ZSTD_getFrameContentSize(inputBinary.data(), inputBinary.size());
+  std::vector<char> decompressedBinary(decompressedBufferSize);
+
+  const size_t decompressedSize = ZSTD_decompress(decompressedBinary.data(), decompressedBufferSize, inputBinary.data(), inputBinary.size());
+  decompressedBinary.resize(decompressedSize);
+
+  std::ostream_iterator<char> oIt(outputBinaryStream.Get());
+  std::copy(decompressedBinary.begin(), decompressedBinary.end(), oIt);
+
+  return EXIT_SUCCESS;
+}
+
+int main(int argc, char * argv[])
+{
+  itk::wasm::Pipeline pipeline("Given a binary or string produced with CompressedStringify, decompress optionally base64 deencode", argc, argv);
+
+  bool parseString = false;
+  pipeline.add_flag("-s,--parse-string", parseString, "Parse the input string before decompression");
+
+  ITK_WASM_PRE_PARSE(pipeline);
+
+  if(parseString)
+  {
+    return decodeDecompress(pipeline);
+  }
+  return decompress(pipeline);
+}
+