Merge pull request #823 from danrbailey/optionaldelayedload

Make Delayed Loading and Boost IOStreams Optional

Author: danrbailey

.github/workflows/weekly.yml

@@ -109,7 +109,7 @@ jobs:
       matrix:
         config:
           - { name: 'all',   build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
-          - { name: 'lite',  build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF' }
+          - { name: 'lite',  build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
           - { name: 'half',  build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
           - { name: 'sse',   build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=SSE42' }
           - { name: 'avx',   build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=AVX' }

CMakeLists.txt

@@ -79,6 +79,7 @@ option(OPENVDB_BUILD_AX_UNITTESTS "Build the OpenVDB AX unit tests" OFF)
 option(OPENVDB_BUILD_NANOVDB "Build the NanoVDB library. Turns ON if USE_NANOVDB is ON." ${USE_NANOVDB})
 
 option(OPENVDB_BUILD_MAYA_PLUGIN "Build the Maya plugin" OFF)
+option(OPENVDB_USE_DELAYED_LOADING "Build the core OpenVDB library with delayed-loading." ON)
 option(OPENVDB_ENABLE_RPATH "Build with RPATH information" ON)
 option(OPENVDB_CXX_STRICT "Enable or disable pre-defined compiler warnings" OFF)
 
@@ -247,6 +248,7 @@ mark_as_advanced(
   OPENVDB_BUILD_HOUDINI_ABITESTS
   OPENVDB_CXX_STRICT
   OPENVDB_ENABLE_RPATH
+  OPENVDB_USE_DELAYED_LOADING
   OPENVDB_FUTURE_DEPRECATION
   OPENVDB_SIMD
   OPENVDB_USE_DEPRECATED_ABI_7

cmake/FindOpenVDB.cmake

@@ -64,6 +64,8 @@ This will define the following variables:
   True if the OpenVDB Library has been built with log4cplus support
 ``OpenVDB_USES_IMATH_HALF``
   True if the OpenVDB Library has been built with Imath half support
+``OpenVDB_USES_DELAYED_LOADING``
+  True if the OpenVDB Library has been built with delayed-loading
 ``OpenVDB_ABI``
   Set if this module was able to determine the ABI number the located
   OpenVDB Library was built against. Unset otherwise.
@@ -555,6 +557,7 @@ set(OpenVDB_USES_BLOSC ${USE_BLOSC})
 set(OpenVDB_USES_ZLIB ${USE_ZLIB})
 set(OpenVDB_USES_LOG4CPLUS ${USE_LOG4CPLUS})
 set(OpenVDB_USES_IMATH_HALF ${USE_IMATH_HALF})
+set(OpenVDB_USES_DELAYED_LOADING ${OPENVDB_USE_DELAYED_LOADING})
 set(OpenVDB_DEFINITIONS)
 
 if(WIN32)
@@ -587,6 +590,7 @@ if(_OPENVDB_HAS_NEW_VERSION_HEADER)
   OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_IMATH_HALF" OpenVDB_USES_IMATH_HALF)
   OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_BLOSC" OpenVDB_USES_BLOSC)
   OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_ZLIB" OpenVDB_USES_ZLIB)
+  OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_DELAYED_LOADING" OpenVDB_USES_DELAYED_LOADING)
 elseif(NOT OPENVDB_USE_STATIC_LIBS)
   # Use GetPrerequisites to see which libraries this OpenVDB lib has linked to
   # which we can query for optional deps. This basically runs ldd/otoll/objdump
@@ -633,6 +637,11 @@ elseif(NOT OPENVDB_USE_STATIC_LIBS)
     if(NOT ${_HAS_DEP} EQUAL -1)
       set(OpenVDB_USES_IMATH_HALF ON)
     endif()
+
+    string(FIND ${PREREQUISITE} "boost_iostreams" _HAS_DEP)
+    if(NOT ${_HAS_DEP} EQUAL -1)
+      set(OpenVDB_USES_DELAYED_LOADING ON)
+    endif()
   endforeach()
 
   unset(_OPENVDB_PREREQUISITE_LIST)
@@ -675,7 +684,12 @@ endif()
 # namespaced headers are correctly prioritized. Otherwise other include paths
 # from shared installs (including houdini) may pull in the wrong headers
 
-set(_OPENVDB_VISIBLE_DEPENDENCIES Boost::iostreams)
+set(_OPENVDB_VISIBLE_DEPENDENCIES "")
+
+if(OpenVDB_USES_DELAYED_LOADING)
+  list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Boost::iostreams)
+  list(APPEND OpenVDB_DEFINITIONS OPENVDB_USE_DELAYED_LOADING)
+endif()
 
 if(OpenVDB_USES_IMATH_HALF)
   list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES $<TARGET_NAME_IF_EXISTS:IlmBase::Half> $<TARGET_NAME_IF_EXISTS:Imath::Imath>)

doc/changes.txt

@@ -2479,7 +2479,7 @@ Houdini:
   modified or deleted, so for safety,
   @vdblink::io::File::open() File::open@endlink automatically makes
   private copies of source files that are smaller than a user-specified limit
-  (see @vdblink::io::File::setCopyMaxBytes() File::setCopyMaxBytes@endlink).
+  (see io::File::setCopyMaxBytes()).
   The limit can be set to zero to disable copying, but if it cannot be
   guaranteed that a file will not be modified, then it is best not to enable
   delayed loading for that file.

doc/dependencies.txt

@@ -37,7 +37,7 @@ Reference Platform, but for those that do, their specified versions are
 
 Component               | Requirements                                      | Optional
 ----------------------- | ------------------------------------------------- | --------
-OpenVDB Core Library    | CMake, C++17 compiler, TBB::tbb, Boost::iostream  | Blosc, ZLib, Log4cplus, IlmBase::Half
+OpenVDB Core Library    | CMake, C++17 compiler, TBB::tbb                   | Blosc, ZLib, Log4cplus, IlmBase::Half, Boost::iostream
 OpenVDB Print           | Core Library dependencies                         | -
 OpenVDB LOD             | Core Library dependencies                         | -
 OpenVDB Render          | Core Library dependencies                         | OpenEXR, IlmBase, libpng

openvdb/openvdb/CMakeLists.txt

@@ -117,7 +117,10 @@ if(OPENVDB_CORE_SHARED AND NOT Boost_USE_STATIC_LIBS)
   set(Boost_USE_STATIC_LIBS OFF)
 endif()
 
-find_package(Boost ${MINIMUM_BOOST_VERSION} REQUIRED COMPONENTS iostreams)
+if(OPENVDB_USE_DELAYED_LOADING)
+  find_package(Boost ${MINIMUM_BOOST_VERSION} REQUIRED COMPONENTS iostreams)
+endif()
+
 if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BOOST_VERSION)
   # The X.Y.Z boost version value isn't available until CMake 3.14
   set(FULL_BOOST_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
@@ -250,7 +253,9 @@ endif()
 # if it's in a shared place (like /usr/local) it doesn't accidently pull in
 # other headers.
 
-list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Boost::iostreams)
+if(OPENVDB_USE_DELAYED_LOADING)
+  list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Boost::iostreams)
+endif()
 
 if(WIN32)
   # Boost headers contain #pragma commands on Windows which causes Boost
@@ -639,6 +644,9 @@ endif()
 if(USE_LOG4CPLUS)
   list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_LOG4CPLUS)
 endif()
+if(OPENVDB_USE_DELAYED_LOADING)
+  list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_DELAYED_LOADING)
+endif()
 
 ##########################################################################
 

openvdb/openvdb/io/Archive.cc

@@ -13,6 +13,7 @@
 #include <openvdb/util/logging.h>
 #include <openvdb/openvdb.h>
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
 // Boost.Interprocess uses a header-only portion of Boost.DateTime
 #ifdef __clang__
 #pragma clang diagnostic push
@@ -26,10 +27,6 @@
 #include <boost/interprocess/mapped_region.hpp>
 #include <boost/iostreams/device/array.hpp>
 #include <boost/iostreams/stream.hpp>
-#include <boost/uuid/uuid_generators.hpp>
-#include <boost/uuid/uuid_io.hpp>
-
-#include <atomic>
 
 #ifdef _WIN32
 #include <boost/interprocess/detail/os_file_functions.hpp> // open_existing_file(), close_file()
@@ -43,6 +40,13 @@ namespace boost { namespace interprocess { namespace detail {} namespace ipcdeta
 #include <sys/stat.h> // for stat()
 #include <unistd.h> // for unlink()
 #endif
+#endif // OPENVDB_USE_DELAYED_LOADING
+
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
+#include <atomic>
+
 #include <algorithm> // for std::find_if()
 #include <cerrno> // for errno
 #include <cstdlib> // for getenv()
@@ -437,6 +441,9 @@ operator<<(std::ostream& os, const StreamMetadata::AuxDataMap& auxData)
 ////////////////////////////////////////
 
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
+
+
 // Memory-mapping a VDB file permits threaded input (and output, potentially,
 // though that might not be practical for compressed files or files containing
 // multiple grids).  In particular, a memory-mapped file can be loaded lazily,
@@ -575,6 +582,9 @@ MappedFile::clearNotifier()
 }
 
 
+#endif // OPENVDB_USE_DELAYED_LOADING
+
+
 ////////////////////////////////////////
 
 
@@ -894,6 +904,7 @@ setGridBackgroundValuePtr(std::ios_base& strm, const void* background)
 }
 
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
 MappedFile::Ptr
 getMappedFilePtr(std::ios_base& strm)
 {
@@ -909,6 +920,7 @@ setMappedFilePtr(std::ios_base& strm, io::MappedFile::Ptr& mappedFile)
 {
     strm.pword(sStreamState.mappedFile) = &mappedFile;
 }
+#endif // OPENVDB_USE_DELAYED_LOADING
 
 
 StreamMetadata::Ptr
@@ -1098,7 +1110,11 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co
 bool
 Archive::isDelayedLoadingEnabled()
 {
+#ifdef OPENVDB_USE_DELAYED_LOADING
     return (nullptr == std::getenv("OPENVDB_DISABLE_DELAYED_LOAD"));
+#else
+    return false;
+#endif
 }
 
 

openvdb/openvdb/io/File.cc

@@ -9,12 +9,17 @@
 #include <openvdb/Exceptions.h>
 #include <openvdb/util/logging.h>
 #include <cstdint>
+
+#ifdef OPENVDB_USE_DELAYED_LOADING
 #include <boost/iostreams/copy.hpp>
 #ifndef _WIN32
 #include <sys/types.h>
-#include <sys/stat.h>
 #include <unistd.h>
 #endif
+#endif // OPENVDB_USE_DELAYED_LOADING
+
+#include <sys/stat.h> // stat()
+
 #include <cassert>
 #include <cstdlib> // for getenv(), strtoul()
 #include <cstring> // for strerror_r()
@@ -81,25 +86,27 @@ struct File::Impl
     std::string mFilename;
     // The file-level metadata
     MetaMap::Ptr mMeta;
-    // The memory-mapped file
-    MappedFile::Ptr mFileMapping;
-    // The buffer for the input stream, if it is a memory-mapped file
-    SharedPtr<std::streambuf> mStreamBuf;
     // The file stream that is open for reading
     std::unique_ptr<std::istream> mInStream;
     // File-level stream metadata (file format, compression, etc.)
     StreamMetadata::Ptr mStreamMetadata;
     // Flag indicating if we have read in the global information (header,
     // metadata, and grid descriptors) for this VDB file
     bool mIsOpen;
-    // File size limit for copying during delayed loading
-    Index64 mCopyMaxBytes;
     // Grid descriptors for all grids stored in the file, indexed by grid name
     NameMap mGridDescriptors;
     // All grids, indexed by unique name (used only when mHasGridOffsets is false)
     Archive::NamedGridMap mNamedGrids;
     // All grids stored in the file (used only when mHasGridOffsets is false)
     GridPtrVecPtr mGrids;
+#ifdef OPENVDB_USE_DELAYED_LOADING
+    // The memory-mapped file
+    MappedFile::Ptr mFileMapping;
+    // The buffer for the input stream, if it is a memory-mapped file
+    SharedPtr<std::streambuf> mStreamBuf;
+    // File size limit for copying during delayed loading
+    Index64 mCopyMaxBytes;
+#endif
 }; // class File::Impl
 
 
@@ -110,7 +117,9 @@ File::File(const std::string& filename): mImpl(new Impl)
 {
     mImpl->mFilename = filename;
     mImpl->mIsOpen = false;
+#ifdef OPENVDB_USE_DELAYED_LOADING
     mImpl->mCopyMaxBytes = Impl::getDefaultCopyMaxBytes();
+#endif
     setInputHasGridOffsets(true);
 }
 
@@ -137,7 +146,9 @@ File::operator=(const File& other)
         mImpl->mFilename = otherImpl.mFilename;
         mImpl->mMeta = otherImpl.mMeta;
         mImpl->mIsOpen = false; // don't want two file objects reading from the same stream
+#ifdef OPENVDB_USE_DELAYED_LOADING
         mImpl->mCopyMaxBytes = otherImpl.mCopyMaxBytes;
+#endif
         mImpl->mGridDescriptors = otherImpl.mGridDescriptors;
         mImpl->mNamedGrids = otherImpl.mNamedGrids;
         mImpl->mGrids = otherImpl.mGrids;
@@ -241,6 +252,7 @@ File::getSize() const
 }
 
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
 Index64
 File::copyMaxBytes() const
 {
@@ -253,6 +265,7 @@ File::setCopyMaxBytes(Index64 bytes)
 {
     mImpl->mCopyMaxBytes = bytes;
 }
+#endif
 
 
 ////////////////////////////////////////
@@ -266,7 +279,11 @@ File::isOpen() const
 
 
 bool
+#ifdef OPENVDB_USE_DELAYED_LOADING
 File::open(bool delayLoad, const MappedFile::Notifier& notifier)
+#else
+File::open(bool /*delayLoad = true*/)
+#endif // OPENVDB_USE_DELAYED_LOADING
 {
     if (isOpen()) {
         OPENVDB_THROW(IoError, filename() << " is already open");
@@ -276,10 +293,13 @@ File::open(bool delayLoad, const MappedFile::Notifier& notifier)
     // Open the file.
     std::unique_ptr<std::istream> newStream;
     SharedPtr<std::streambuf> newStreamBuf;
+#ifdef OPENVDB_USE_DELAYED_LOADING
     MappedFile::Ptr newFileMapping;
     if (!delayLoad || !Archive::isDelayedLoadingEnabled()) {
+#endif
         newStream.reset(new std::ifstream(
             filename().c_str(), std::ios_base::in | std::ios_base::binary));
+#ifdef OPENVDB_USE_DELAYED_LOADING
     } else {
         bool isTempFile = false;
         std::string fname = filename();
@@ -318,6 +338,7 @@ File::open(bool delayLoad, const MappedFile::Notifier& notifier)
             OPENVDB_THROW(IoError, ostr.str());
         }
     }
+#endif // OPENVDB_USE_DELAYED_LOADING
 
     if (newStream->fail()) {
         OPENVDB_THROW(IoError, "could not open file " << filename());
@@ -335,9 +356,11 @@ File::open(bool delayLoad, const MappedFile::Notifier& notifier)
         throw;
     }
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
     mImpl->mFileMapping = newFileMapping;
     if (mImpl->mFileMapping) mImpl->mFileMapping->setNotifier(notifier);
     mImpl->mStreamBuf = newStreamBuf;
+#endif
     mImpl->mInStream.swap(newStream);
 
     // Tag the input stream with the file format and library version numbers
@@ -348,7 +371,9 @@ File::open(bool delayLoad, const MappedFile::Notifier& notifier)
     Archive::setFormatVersion(inputStream());
     Archive::setLibraryVersion(inputStream());
     Archive::setDataCompression(inputStream());
+#ifdef OPENVDB_USE_DELAYED_LOADING
     io::setMappedFilePtr(inputStream(), mImpl->mFileMapping);
+#endif
 
     // Read in the VDB metadata.
     mImpl->mMeta = MetaMap::Ptr(new MetaMap);
@@ -396,9 +421,11 @@ File::close()
     mImpl->mGrids.reset();
     mImpl->mNamedGrids.clear();
     mImpl->mInStream.reset();
-    mImpl->mStreamBuf.reset();
     mImpl->mStreamMetadata.reset();
+#ifdef OPENVDB_USE_DELAYED_LOADING
+    mImpl->mStreamBuf.reset();
     mImpl->mFileMapping.reset();
+#endif
 
     mImpl->mIsOpen = false;
     setInputHasGridOffsets(true);

openvdb/openvdb/io/File.h

@@ -54,6 +54,7 @@ class OPENVDB_API File: public Archive
     /// @details The file does not necessarily exist on disk yet.
     const std::string& filename() const;
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
     /// @brief Open the file, read the file header and the file-level metadata,
     /// and populate the grid descriptors, but do not load any grids into memory.
     /// @details If @a delayLoad is true, map the file into memory and enable delayed loading
@@ -64,6 +65,9 @@ class OPENVDB_API File: public Archive
     /// @return @c true if the file's UUID has changed since it was last read.
     /// @see setCopyMaxBytes
     bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier());
+#else
+    bool open(bool /*delayLoad*/ = false);
+#endif
 
     /// Return @c true if the file has been opened for reading.
     bool isOpen() const;
@@ -75,6 +79,7 @@ class OPENVDB_API File: public Archive
     /// @throw IoError if the file size cannot be determined.
     Index64 getSize() const;
 
+#ifdef OPENVDB_USE_DELAYED_LOADING
     /// @brief Return the size in bytes above which this file will not be
     /// automatically copied during delayed loading.
     Index64 copyMaxBytes() const;
@@ -90,6 +95,7 @@ class OPENVDB_API File: public Archive
     /// by setting the environment variable @c OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES
     /// to the desired number of bytes.
     void setCopyMaxBytes(Index64 bytes);
+#endif
 
     /// Return @c true if a grid of the given name exists in this file.
     bool hasGrid(const Name&) const;