Added tools::memUsageIfLoaded() for returning the deserialized memory usage of an OpenVDB Tree

 - Requires ABI change for AttributeArrays
 - Updated vdb_print to print both in-core and total memory usage

Signed-off-by: Nick Avramoussis <[email protected]>

Author: Idclip

openvdb/openvdb/cmd/openvdb_print.cc

@@ -9,6 +9,7 @@
 #include <string>
 #include <vector>
 #include <openvdb/openvdb.h>
+#include <openvdb/tools/Count.h>
 #include <openvdb/util/logging.h>
 
 
@@ -206,8 +207,33 @@ printShortListing(const StringVec& filenames, bool metadata)
             std::cout << "  " << std::right << std::setw(8)
                 << sizeAsString(grid->activeVoxelCount(), "Vox");
 
-            // Print the grid's in-core size, in bytes.
-            std::cout << " " << std::right << std::setw(6) << bytesAsString(grid->memUsage());
+            // Print the grid's size, in bytes
+
+            // no support for memUsageIfLoaded until ABI >= 10 for points::PointDataGrid types
+#if OPENVDB_ABI_VERSION_NUMBER < 10
+            using ListT = openvdb::GridTypes::Remove<openvdb::points::PointDataGrid>;
+#else
+            using ListT = openvdb::GridTypes;
+#endif
+            bool success =
+                grid->apply<ListT>([&](const auto& typed){
+                    // @todo combine these methods to avoid iterating across the tree twice
+                    const openvdb::Index64 incore = openvdb::tools::memUsage(typed.tree());
+                    const openvdb::Index64 total = openvdb::tools::memUsageIfLoaded(typed.tree());
+
+                    std::cout << " " << std::right << std::setw(6) << bytesAsString(incore) << " (In Core)";
+                    std::cout << " " << std::right << std::setw(6) << bytesAsString(total) << " (Total)";
+                });
+
+#if OPENVDB_ABI_VERSION_NUMBER < 10
+            if (!success) {
+                // could be a points grid, print in-core memory only
+                grid->apply<openvdb::GridTypes>([&](const auto& typed){
+                    const openvdb::Index64 incore = openvdb::tools::memUsage(typed.tree());
+                    std::cout << " " << std::right << std::setw(6) << bytesAsString(incore) << " (In Core)";
+                });
+            }
+#endif
 
             std::cout << std::endl;
 

openvdb/openvdb/points/AttributeArray.h

@@ -194,6 +194,14 @@ class OPENVDB_API AttributeArray
     /// Return the number of bytes of memory used by this attribute.
     virtual size_t memUsage() const = 0;
 
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+    /// Return the number of bytes of memory used by this attribute array once it
+    /// has been deserialized (this may be different to memUsage() if delay-loading
+    /// is in use). Note that this method does NOT consider the fact that a
+    /// uniform attribute could be expanded and only deals with delay-loading.
+    virtual size_t memUsageIfLoaded() const = 0;
+#endif
+
     /// Create a new attribute array of the given (registered) type, length and stride.
     /// @details If @a lock is non-null, the AttributeArray registry mutex
     /// has already been locked
@@ -645,6 +653,14 @@ class TypedAttributeArray final: public AttributeArray
     /// Return the number of bytes of memory used by this attribute.
     size_t memUsage() const override;
 
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+    /// Return the number of bytes of memory used by this attribute array once it
+    /// has been deserialized (this may be different to memUsage() if delay-loading
+    /// is in use). Note that this method does NOT consider the fact that a
+    /// uniform attribute could be expanded and only deals with delay-loading.
+    size_t memUsageIfLoaded() const override;
+#endif
+
     /// Return the value at index @a n (assumes in-core)
     ValueType getUnsafe(Index n) const;
     /// Return the value at index @a n
@@ -1375,6 +1391,15 @@ TypedAttributeArray<ValueType_, Codec_>::memUsage() const
     return sizeof(*this) + (bool(mData) ? this->arrayMemUsage() : 0);
 }
 
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+template<typename ValueType_, typename Codec_>
+size_t
+TypedAttributeArray<ValueType_, Codec_>::memUsageIfLoaded() const
+{
+    return sizeof(*this) + (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType);
+}
+#endif
+
 
 template<typename ValueType_, typename Codec_>
 typename TypedAttributeArray<ValueType_, Codec_>::ValueType

openvdb/openvdb/points/AttributeSet.cc

@@ -139,6 +139,19 @@ AttributeSet::memUsage() const
 }
 
 
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+size_t
+AttributeSet::memUsageIfLoaded() const
+{
+    size_t bytes = sizeof(*this) + mDescr->memUsage();
+    for (const auto& attr : mAttrs) {
+        bytes += attr->memUsageIfLoaded();
+    }
+    return bytes;
+}
+#endif
+
+
 size_t
 AttributeSet::find(const std::string& name) const
 {

openvdb/openvdb/points/AttributeSet.h

@@ -113,6 +113,13 @@ class OPENVDB_API AttributeSet
     /// Return the number of bytes of memory used by this attribute set.
     size_t memUsage() const;
 
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+    /// Return the number of bytes of memory used by this attribute set once it
+    /// has been deserialized (this may be different to memUsage() if delay-loading
+    /// is in use).
+    size_t memUsageIfLoaded() const;
+#endif
+
     /// @brief  Return the position of the attribute array whose name is @a name,
     ///         or @c INVALID_POS if no match is found.
     size_t find(const std::string& name) const;

openvdb/openvdb/points/PointDataGrid.h

@@ -503,6 +503,9 @@ class PointDataLeafNode : public tree::LeafNode<T, Log2Dim>, io::MultiPass {
 
 
     Index64 memUsage() const;
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+    Index64 memUsageIfLoaded() const;
+#endif
 
     void evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels = true) const;
 
@@ -1520,6 +1523,15 @@ PointDataLeafNode<T, Log2Dim>::memUsage() const
     return BaseLeaf::memUsage() + mAttributeSet->memUsage();
 }
 
+#if OPENVDB_ABI_VERSION_NUMBER >= 10
+template<typename T, Index Log2Dim>
+inline Index64
+PointDataLeafNode<T, Log2Dim>::memUsageIfLoaded() const
+{
+    return BaseLeaf::memUsageIfLoaded() + mAttributeSet->memUsageIfLoaded();
+}
+#endif
+
 template<typename T, Index Log2Dim>
 inline void
 PointDataLeafNode<T, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const

openvdb/openvdb/tools/Count.h

@@ -60,9 +60,21 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded = true);
 
 
 /// @brief Return the total amount of memory in bytes occupied by this tree.
+/// @details  This method returns the total in-core memory usage which can be
+///   different to the maximum possible memory usage for trees which have not
+///   been fully deserialized (via delay-loading). Thus, this is the current
+///   true memory consumption.
 template <typename TreeT>
 Index64 memUsage(const TreeT& tree, bool threaded = true);
 
+
+/// @brief Return the deserialized memory usage of this tree. This is not
+///   necessarily equal to the current memory usage (returned by tools::memUsage)
+///   if delay-loading is enabled. See File::open.
+template <typename TreeT>
+Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true);
+
+
 /// @brief Return the minimum and maximum active values in this tree.
 /// @note  Returns zeroVal<ValueType> for empty trees.
 template <typename TreeT>
@@ -287,21 +299,22 @@ struct MemUsageOp
     using RootT = typename TreeType::RootNodeType;
     using LeafT = typename TreeType::LeafNodeType;
 
-    MemUsageOp() = default;
-    MemUsageOp(const MemUsageOp&, tbb::split) { }
+    MemUsageOp(const bool inCoreOnly) : mInCoreOnly(inCoreOnly) {}
+    MemUsageOp(const MemUsageOp& other) : mCount(0), mInCoreOnly(other.mInCoreOnly) {}
+    MemUsageOp(const MemUsageOp& other, tbb::split) : MemUsageOp(other) {}
 
     // accumulate size of the root node in bytes
     bool operator()(const RootT& root, size_t)
     {
-        count += sizeof(root);
+        mCount += sizeof(root);
         return true;
     }
 
     // accumulate size of all child nodes in bytes
     template<typename NodeT>
     bool operator()(const NodeT& node, size_t)
     {
-        count += NodeT::NUM_VALUES * sizeof(typename NodeT::UnionType) +
+        mCount += NodeT::NUM_VALUES * sizeof(typename NodeT::UnionType) +
             node.getChildMask().memUsage() + node.getValueMask().memUsage() +
             sizeof(Coord);
         return true;
@@ -310,16 +323,18 @@ struct MemUsageOp
     // accumulate size of leaf node in bytes
     bool operator()(const LeafT& leaf, size_t)
     {
-        count += leaf.memUsage();
+        if (mInCoreOnly) mCount += leaf.memUsage();
+        else             mCount += leaf.memUsageIfLoaded();
         return false;
     }
 
     void join(const MemUsageOp& other)
     {
-        count += other.count;
+        mCount += other.mCount;
     }
 
-    openvdb::Index64 count{0};
+    openvdb::Index64 mCount{0};
+    const bool mInCoreOnly;
 }; // struct MemUsageOp
 
 /// @brief A DynamicNodeManager operator to find the minimum and maximum active values in this tree.
@@ -477,10 +492,24 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded)
 template <typename TreeT>
 Index64 memUsage(const TreeT& tree, bool threaded)
 {
-    count_internal::MemUsageOp<TreeT> op;
+    count_internal::MemUsageOp<TreeT> op(true);
+    tree::DynamicNodeManager<const TreeT> nodeManager(tree);
+    nodeManager.reduceTopDown(op, threaded);
+    return op.mCount + sizeof(tree);
+}
+
+template <typename TreeT>
+Index64 memUsageIfLoaded(const TreeT& tree, bool threaded)
+{
+    /// @note  For numeric (non-point) grids this really doesn't need to
+    ///   traverse the tree and could instead be computed from the node counts.
+    ///   We do so anyway as it ties this method into the tree data structure
+    ///   which makes sure that changes to the tree/nodes are reflected/kept in
+    ///   sync here.
+    count_internal::MemUsageOp<TreeT> op(false);
     tree::DynamicNodeManager<const TreeT> nodeManager(tree);
     nodeManager.reduceTopDown(op, threaded);
-    return op.count + sizeof(tree);
+    return op.mCount + sizeof(tree);
 }
 
 template <typename TreeT>

openvdb/openvdb/tools/PointIndexGrid.h

@@ -1490,6 +1490,7 @@ struct PointIndexLeafNode : public tree::LeafNode<T, Log2Dim>
 
 
     Index64 memUsage() const;
+    Index64 memUsageIfLoaded() const;
 
 
     ////////////////////////////////////////
@@ -1787,6 +1788,13 @@ PointIndexLeafNode<T, Log2Dim>::memUsage() const
     return BaseLeaf::memUsage() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices));
 }
 
+template<typename T, Index Log2Dim>
+inline Index64
+PointIndexLeafNode<T, Log2Dim>::memUsageIfLoaded() const
+{
+    return BaseLeaf::memUsageIfLoaded() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices));
+}
+
 } // namespace tools
 
 

openvdb/openvdb/tree/LeafBuffer.h

@@ -87,6 +87,7 @@ class LeafBuffer
 
     /// Return the memory footprint of this buffer in bytes.
     inline Index memUsage() const;
+    inline Index memUsageIfLoaded() const;
     /// Return the number of values contained in this buffer.
     static Index size() { return SIZE; }
 
@@ -275,6 +276,16 @@ LeafBuffer<T, Log2Dim>::memUsage() const
 }
 
 
+template<typename T, Index Log2Dim>
+inline Index
+LeafBuffer<T, Log2Dim>::memUsageIfLoaded() const
+{
+    size_t n = sizeof(*this);
+    n += SIZE * sizeof(ValueType);
+    return static_cast<Index>(n);
+}
+
+
 template<typename T, Index Log2Dim>
 inline const typename LeafBuffer<T, Log2Dim>::ValueType*
 LeafBuffer<T, Log2Dim>::data() const
@@ -425,6 +436,7 @@ class LeafBuffer<bool, Log2Dim>
     void swap(LeafBuffer& other) { if (&other != this) std::swap(mData, other.mData); }
 
     Index memUsage() const { return sizeof(*this); }
+    Index memUsageIfLoaded() const { return sizeof(*this); }
     static Index size() { return SIZE; }
 
     /// @brief Return a pointer to the C-style array of words encoding the bits.

openvdb/openvdb/tree/LeafNode.h

@@ -155,6 +155,7 @@ class LeafNode
 
     /// Return the memory in bytes occupied by this node.
     Index64 memUsage() const;
+    Index64 memUsageIfLoaded() const;
 
     /// Expand the given bounding box so that it includes this leaf node's active voxels.
     /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all
@@ -1469,6 +1470,16 @@ LeafNode<T, Log2Dim>::memUsage() const
 }
 
 
+template<typename T, Index Log2Dim>
+inline Index64
+LeafNode<T, Log2Dim>::memUsageIfLoaded() const
+{
+    // Use sizeof(*this) to capture alignment-related padding
+    // (but note that sizeof(*this) includes sizeof(mBuffer)).
+    return sizeof(*this) + mBuffer.memUsageIfLoaded() - sizeof(mBuffer);
+}
+
+
 template<typename T, Index Log2Dim>
 inline void
 LeafNode<T, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const

openvdb/openvdb/tree/LeafNodeBool.h

@@ -137,6 +137,7 @@ class LeafNode<bool, Log2Dim>
 
     /// Return the memory in bytes occupied by this node.
     Index64 memUsage() const;
+    Index64 memUsageIfLoaded() const;
 
     /// Expand the given bounding box so that it includes this leaf node's active voxels.
     /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all
@@ -896,6 +897,15 @@ LeafNode<bool, Log2Dim>::memUsage() const
 }
 
 
+template<Index Log2Dim>
+inline Index64
+LeafNode<bool, Log2Dim>::memUsageIfLoaded() const
+{
+    // Use sizeof(*this) to capture alignment-related padding
+    return sizeof(*this);
+}
+
+
 template<Index Log2Dim>
 inline void
 LeafNode<bool, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const