ENH: OOC architecture rewrite — new bulk I/O API and infrastructure

CMakeLists.txt

@@ -53,6 +53,36 @@ option(SIMPLNX_DOWNLOAD_TEST_FILES "Download the test files" ON)
 # ------------------------------------------------------------------------------
 option(SIMPLNX_WRITE_TEST_OUTPUT "Write unit test output files" OFF)
 
+# ------------------------------------------------------------------------------
+# Controls which algorithm paths are exercised by dual-dispatch unit tests.
+#   0 (Both)       - tests run with forceOoc=false AND forceOoc=true (default)
+#   1 (OocOnly)    - tests run with forceOoc=true only (use for OOC builds)
+#   2 (InCoreOnly) - tests run with forceOoc=false only (quick validation)
+# ------------------------------------------------------------------------------
+set(SIMPLNX_TEST_ALGORITHM_PATH "0" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly")
+
+# ------------------------------------------------------------------------------
+# Out-of-core support compile-time switch
+# ------------------------------------------------------------------------------
+option(SIMPLNX_USE_OOC "Compile out-of-core support into simplnx (requires SIMPLNX_OOC_SOURCE_DIR)" OFF)
+set(SIMPLNX_OOC_SOURCE_DIR "" CACHE PATH "Path to the private SimplnxOoc source directory (required when SIMPLNX_USE_OOC=ON)")
+
+if(SIMPLNX_USE_OOC)
+  if(NOT SIMPLNX_OOC_SOURCE_DIR OR NOT EXISTS "${SIMPLNX_OOC_SOURCE_DIR}")
+    message(FATAL_ERROR
+      "SIMPLNX_USE_OOC=ON requires SIMPLNX_OOC_SOURCE_DIR to point at the private SimplnxOoc repo. "
+      "Set -DSIMPLNX_OOC_SOURCE_DIR=/path/to/SimplnxOoc/SimplnxOoc")
+  endif()
+else()
+  # With OOC compiled out there are no out-of-core algorithm paths to exercise,
+  # so pin the test path to InCoreOnly. Forcing it avoids a stale cached 0/1
+  # (Both/OocOnly) producing tests that can never run.
+  if(NOT SIMPLNX_TEST_ALGORITHM_PATH STREQUAL "2")
+    message(STATUS "SIMPLNX_USE_OOC=OFF: forcing SIMPLNX_TEST_ALGORITHM_PATH=2 (InCoreOnly).")
+    set(SIMPLNX_TEST_ALGORITHM_PATH "2" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly" FORCE)
+  endif()
+endif()
+
 # ------------------------------------------------------------------------------
 # Is the SimplnxCore Plugin enabled [DEFAULT=ON]
 # ------------------------------------------------------------------------------
@@ -261,6 +291,7 @@ if(SIMPLNX_ENABLE_MULTICORE)
   target_link_libraries(simplnx PUBLIC TBB::tbb)
 endif()
 
+
 target_link_libraries(simplnx
   PUBLIC
     fmt::fmt
@@ -293,6 +324,11 @@ if(SIMPLNX_ENABLE_LINK_FILESYSTEM)
 endif()
 
 set(SIMPLNX_GENERATED_DIR ${simplnx_BINARY_DIR}/generated)
+
+configure_file(
+  ${simplnx_SOURCE_DIR}/cmake/SimplnxConfig.hpp.in
+  ${SIMPLNX_GENERATED_DIR}/simplnx/Common/SimplnxConfig.hpp
+  @ONLY)
 set(SIMPLNX_GENERATED_HEADER_DIR ${simplnx_BINARY_DIR}/generated/simplnx)
 set(SIMPLNX_EXPORT_HEADER ${SIMPLNX_GENERATED_HEADER_DIR}/simplnx_export.hpp)
 
@@ -351,6 +387,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Common/Constants.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataTypeUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataVector.hpp
+  ${SIMPLNX_SOURCE_DIR}/Common/Extent.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/EulerAngle.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/Filesystem.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/IteratorUtility.hpp
@@ -465,6 +502,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/DynamicListArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyDataStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyListStore.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyStringStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataStore.hpp
@@ -546,6 +584,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataGroupUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataObjectUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/AlgorithmDispatch.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FilePathGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ColorTableUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FileUtilities.hpp
@@ -556,6 +595,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringInterpretationUtilities.hpp
@@ -770,6 +810,7 @@ set(SIMPLNX_SRCS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.cpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.cpp
@@ -893,6 +934,53 @@ target_include_directories(simplnx
     $<INSTALL_INTERFACE:include>
 )
 
+# ------------------------------------------------------------------------------
+# Out-of-core sources (SIMPLNX_USE_OOC=ON)
+#
+# Compile the private SimplnxOoc sources directly into libsimplnx, IN PLACE from
+# their own repo — no copy into the build tree. This keeps the public simplnx
+# source repo free of OOC code while letting simplnx core call SimplnxOoc::
+# functions directly with no separate library and no simplnx->SimplnxOoc->simplnx
+# link cycle. The parent of SIMPLNX_OOC_SOURCE_DIR is placed on simplnx's PUBLIC
+# include path so `#include "SimplnxOoc/X.hpp"` resolves for every consumer.
+#
+# Compiling the originals (rather than configure_file copies) means editing an
+# OOC source recompiles only that translation unit and its dependents, with NO
+# CMake re-configure: configure_file registers every input as a configure-time
+# dependency, so the old copy step forced a full re-configure on each edit.
+# ------------------------------------------------------------------------------
+if(SIMPLNX_USE_OOC)
+  file(GLOB SIMPLNX_OOC_SRCS CONFIGURE_DEPENDS "${SIMPLNX_OOC_SOURCE_DIR}/*.cpp")
+
+  # The partitioned store (unstructured/poly OOC) is deferred — exclude it from
+  # the compiled set so the deferred, #if 0'd code never reaches the compiler.
+  list(FILTER SIMPLNX_OOC_SRCS EXCLUDE REGEX "HDF5PartitionedStore")
+
+  # Compile the OOC sources straight from the private repo.
+  target_sources(simplnx PRIVATE ${SIMPLNX_OOC_SRCS})
+
+  # Put the parent of the SimplnxOoc source dir on simplnx's PUBLIC include path so
+  # `#include "SimplnxOoc/X.hpp"` resolves for simplnx itself and, transitively, for
+  # every consumer (e.g. DREAM3D-NX). Wrapped in BUILD_INTERFACE so the absolute
+  # developer path is never baked into an installed/exported simplnx.
+  get_filename_component(SIMPLNX_OOC_INCLUDE_ROOT "${SIMPLNX_OOC_SOURCE_DIR}" DIRECTORY)
+  target_include_directories(simplnx PUBLIC "$<BUILD_INTERFACE:${SIMPLNX_OOC_INCLUDE_ROOT}>")
+
+  # Export shim: the OOC code compiles into libsimplnx, so SIMPLNXOOC_EXPORT maps onto
+  # simplnx's own export macro. It lives under SIMPLNX_GENERATED_DIR (already on the
+  # PUBLIC include path), so `#include "SimplnxOoc/SimplnxOoc_export.hpp"` resolves from
+  # that root. The real OOC headers resolve from SIMPLNX_OOC_INCLUDE_ROOT — keep this
+  # generated SimplnxOoc dir limited to the shim so it can't shadow an in-place header.
+  file(WRITE "${SIMPLNX_GENERATED_DIR}/SimplnxOoc/SimplnxOoc_export.hpp"
+    "#pragma once\n#include \"simplnx/simplnx_export.hpp\"\n#define SIMPLNXOOC_EXPORT SIMPLNX_EXPORT\n")
+
+  # The chunked-store .cpp files explicitly instantiate many template classes,
+  # exceeding the default COFF section limit on MSVC.
+  if(MSVC)
+    set_source_files_properties(${SIMPLNX_OOC_SRCS} PROPERTIES COMPILE_OPTIONS "/bigobj")
+  endif()
+endif()
+
 cmake_dependent_option(SIMPLNX_DOWNLOAD_TEST_FILES_FIRST "Forces test files to download before simplnx builds" OFF "SIMPLNX_DOWNLOAD_TEST_FILES" OFF)
 if(SIMPLNX_DOWNLOAD_TEST_FILES_FIRST)
   add_dependencies(simplnx Fetch_Remote_Data_Files)
@@ -1063,6 +1151,13 @@ if(SIMPLNX_BUILD_TESTS)
   find_package(ZLIB REQUIRED)
   include(CTest)
   add_subdirectory(test)
+
+  # Build the private SimplnxOoc test suite against libsimplnx (which carries
+  # the OOC symbols when SIMPLNX_USE_OOC=ON). This single rule serves both ON
+  # contexts: standalone simplnx and the simplnx subdirectory inside DREAM3D-NX.
+  if(SIMPLNX_USE_OOC)
+    add_subdirectory("${SIMPLNX_OOC_SOURCE_DIR}/test" ${simplnx_BINARY_DIR}/SimplnxOoc_test)
+  endif()
 endif()
 
 if(SIMPLNX_BUILD_PYTHON)

cmake/Plugin.cmake

@@ -383,6 +383,7 @@ function(create_simplnx_plugin_unit_test)
   target_compile_definitions(${UNIT_TEST_TARGET}
     PRIVATE
       SIMPLNX_BUILD_DIR="$<TARGET_FILE_DIR:simplnx_test>"
+      SIMPLNX_TEST_ALGORITHM_PATH=${SIMPLNX_TEST_ALGORITHM_PATH}
   )
 
   target_compile_options(${UNIT_TEST_TARGET}

cmake/SimplnxConfig.hpp.in

@@ -0,0 +1,11 @@
+#pragma once
+
+// This header is generated from cmake/SimplnxConfig.hpp.in by configure_file().
+// It carries compile-time configuration into every translation unit that links
+// simplnx, including MOC, via simplnx's PUBLIC generated include directory.
+// Delivering the macro through a header (not per-target compile definitions)
+// keeps the value identical across all consumers, which is required for ODR
+// safety when the macro gates inline code in public headers.
+
+// Defined (to 1) when out-of-core support is compiled into libsimplnx.
+#cmakedefine SIMPLNX_USE_OOC

src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Common/ITKArrayHelper.hpp

@@ -855,7 +855,7 @@ Result<OutputActions> DataCheck(const DataStructure& dataStructure, const DataPa
   const auto& inputArray = dataStructure.getDataRefAs<IDataArray>(inputArrayPath);
   const auto& inputDataStore = inputArray.getIDataStoreRef();
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult<OutputActions>(Constants::k_OutOfCoreDataNotSupported,
                                           fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
@@ -877,7 +877,7 @@ Result<detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>> Execute(DataStr
 
   using ResultT = detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>;
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult(Constants::k_OutOfCoreDataNotSupported, fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
   }

src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp

@@ -30,7 +30,7 @@ std::string ComputeMD5HashTyped(const IDataArray& outputDataArray)
   usize arraySize = dataArray.getSize();
 
   MD5 md5;
-  if(outputDataArray.getDataFormat().empty())
+  if(outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore)
   {
     const T* dataPtr = dataArray.template getIDataStoreRefAs<DataStore<T>>().data();
     md5.update(reinterpret_cast<const uint8*>(dataPtr), arraySize * sizeof(T));
@@ -135,47 +135,7 @@ namespace ITKTestBase
 bool IsArrayInMemory(DataStructure& dataStructure, const DataPath& outputDataPath)
 {
   const auto& outputDataArray = dataStructure.getDataRefAs<IDataArray>(outputDataPath);
-  DataType outputDataType = outputDataArray.getDataType();
-
-  switch(outputDataType)
-  {
-  case DataType::float32: {
-    return dynamic_cast<const DataArray<float32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::float64: {
-    return dynamic_cast<const DataArray<float64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int8: {
-    return dynamic_cast<const DataArray<int8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint8: {
-    return dynamic_cast<const DataArray<uint8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int16: {
-    return dynamic_cast<const DataArray<int16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint16: {
-    return dynamic_cast<const DataArray<uint16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int32: {
-    return dynamic_cast<const DataArray<int32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint32: {
-    return dynamic_cast<const DataArray<uint32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int64: {
-    return dynamic_cast<const DataArray<int64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint64: {
-    return dynamic_cast<const DataArray<uint64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::boolean: {
-    [[fallthrough]];
-  }
-  default: {
-    return {};
-  }
-  }
+  return outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore;
 }
 //------------------------------------------------------------------------------
 std::string ComputeMd5Hash(DataStructure& dataStructure, const DataPath& outputDataPath)

src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/ComputeFZQuaternionsFilter.cpp

@@ -122,8 +122,8 @@ IFilter::PreflightResult ComputeFZQuaternionsFilter::preflightImpl(const DataStr
 
   nx::core::Result<OutputActions> resultOutputActions;
 
-  auto createArrayAction = std::make_unique<CreateArrayAction>(nx::core::DataType::float32, quatArray.getDataStore()->getTupleShape(), quatArray.getDataStore()->getComponentShape(),
-                                                               pFZQuatsArrayPathValue, CreateArrayAction::k_DefaultDataFormat, "0.0");
+  auto createArrayAction =
+      std::make_unique<CreateArrayAction>(nx::core::DataType::float32, quatArray.getDataStore()->getTupleShape(), quatArray.getDataStore()->getComponentShape(), pFZQuatsArrayPathValue, "", "0.0");
   resultOutputActions.value().appendAction(std::move(createArrayAction));
 
   // Return both the resultOutputActions and the preflightUpdatedValues via std::move()

src/Plugins/OrientationAnalysis/test/ComputeIPFColorsTest.cpp

@@ -79,9 +79,7 @@ TEST_CASE("OrientationAnalysis::ComputeIPFColors", "[OrientationAnalysis][Comput
     // This test file was produced by SIMPL/DREAM3D. our results should match theirs
     auto exemplarFilePath = fs::path(fmt::format("{}/so3_cubic_high_ipf_001.dream3d", unit_test::k_TestFilesDir));
     REQUIRE(fs::exists(exemplarFilePath));
-    auto result = DREAM3D::ImportDataStructureFromFile(exemplarFilePath, false);
-    REQUIRE(result.valid());
-    dataStructure = result.value();
+    dataStructure = UnitTest::LoadDataStructure(exemplarFilePath);
   }
 
   // Instantiate the filter, a DataStructure object and an Arguments Object
@@ -117,7 +115,7 @@ TEST_CASE("OrientationAnalysis::ComputeIPFColors", "[OrientationAnalysis][Comput
     SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
     {
       // Write out the DataStructure for later viewing/debugging
-      auto fileWriter = nx::core::HDF5::FileIO::WriteFile(std::filesystem::path(fmt::format("{}/ComputeIPFColors_Test.dream3d", unit_test::k_BinaryTestOutputDir)));
+      auto fileWriter = nx::core::HDF5::FileIO::WriteFile(fs::path(fmt::format("{}/ComputeIPFColors_Test.dream3d", unit_test::k_BinaryTestOutputDir)));
       auto resultH5 = HDF5::DataStructureWriter::WriteFile(dataStructure, fileWriter);
       SIMPLNX_RESULT_REQUIRE_VALID(resultH5);
     }

src/Plugins/OrientationAnalysis/test/RodriguesConvertorTest.cpp

@@ -50,6 +50,10 @@ TEST_CASE("OrientationAnalysis::RodriguesConvertorFilter", "[OrientationAnalysis
   (*exemplarData)[9] = 0.573462F;
   (*exemplarData)[10] = 0.655386F;
   (*exemplarData)[11] = 12.2066F;
+  (*exemplarData)[12] = 0.517892F;
+  (*exemplarData)[13] = 0.575435F;
+  (*exemplarData)[14] = 0.632979F;
+  (*exemplarData)[15] = 17.37815F;
   {
     // Instantiate the filter, a DataStructure object and an Arguments Object
     const RodriguesConvertorFilter filter;

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeArrayStatistics.cpp

@@ -1,6 +1,7 @@
 #include "ComputeArrayStatistics.hpp"
 
 #include "simplnx/DataStructure/AttributeMatrix.hpp"
+#include "simplnx/DataStructure/IDataStore.hpp"
 #include "simplnx/Utilities/DataArrayUtilities.hpp"
 #include "simplnx/Utilities/FilterUtilities.hpp"
 #include "simplnx/Utilities/HistogramUtilities.hpp"
@@ -31,7 +32,7 @@ bool CheckArraysInMemory(const nx::core::IParallelAlgorithm::AlgorithmArrays& ar
       continue;
     }
 
-    if(!arrayPtr->getIDataStoreRef().getDataFormat().empty())
+    if(arrayPtr->getIDataStoreRef().getStoreType() == nx::core::IDataStore::StoreType::OutOfCore)
     {
       return false;
     }