Merge pull request #1027 from Idclip/tbb2021

Added support for TBB 2021

Author: Idclip

.github/workflows/build.yml

@@ -200,3 +200,28 @@ jobs:
     - name: test
       shell: bash
       run: cd build && ctest -V
+
+  testmacos1015_tbb2021:
+    runs-on: macos-10.15
+    env:
+      CXX: clang++
+    steps:
+    - uses: actions/checkout@v2
+    - name: install
+      shell: bash
+      run: |
+        brew update
+        brew install bash cmake ilmbase openexr@2 boost tbb glfw
+        echo "IlmBase_ROOT=/usr/local/opt/ilmbase" >> $GITHUB_ENV
+        echo "OpenEXR_ROOT=/usr/local/opt/openexr@2" >> $GITHUB_ENV
+        echo "/usr/local/opt/openexr@2/bin" >> $GITHUB_PATH
+    - name: install
+      shell: bash
+      run: ./ci/install_gtest.sh 1.10.0
+    - name: build
+      shell: bash
+      run: ./ci/build.sh Release "core,bin,test" \
+         -DOPENVDB_CXX_STRICT=OFF -DUSE_BLOSC=OFF -DUSE_ZLIB=OFF
+    - name: test
+      shell: bash
+      run: cd build && ctest -V

ci/install_macos.sh

@@ -9,7 +9,6 @@ brew install ilmbase
 brew install openexr@2
 brew install boost
 brew install boost-python3 # also installs the dependent python version
-brew install gtest
 brew install tbb@2020
 brew install zlib
 brew install glfw

cmake/FindTBB.cmake

@@ -169,16 +169,30 @@ find_path(Tbb_INCLUDE_DIR tbb/tbb_stddef.h
   PATH_SUFFIXES ${CMAKE_INSTALL_INCLUDEDIR} include
 )
 
-if(EXISTS "${Tbb_INCLUDE_DIR}/tbb/tbb_stddef.h")
-  file(STRINGS "${Tbb_INCLUDE_DIR}/tbb/tbb_stddef.h"
+set(_tbb_version_file "${Tbb_INCLUDE_DIR}/tbb/tbb_stddef.h")
+
+if(NOT EXISTS ${_tbb_version_file})
+  # From TBB 2021, tbb_stddef is removed and the directory include/tbb is
+  # simply an alias for include/oneapi/tbb. Try and find the version header
+  # in oneapi/tbb
+  find_path(Tbb_INCLUDE_DIR oneapi/tbb/version.h
+    ${_FIND_TBB_ADDITIONAL_OPTIONS}
+    PATHS ${_TBB_INCLUDE_SEARCH_DIRS}
+    PATH_SUFFIXES ${CMAKE_INSTALL_INCLUDEDIR} include
+  )
+  set(_tbb_version_file "${Tbb_INCLUDE_DIR}/oneapi/tbb/version.h")
+endif()
+
+if(EXISTS ${_tbb_version_file})
+  file(STRINGS ${_tbb_version_file}
     _tbb_version_major_string REGEX "#define TBB_VERSION_MAJOR "
   )
   string(REGEX REPLACE "#define TBB_VERSION_MAJOR" ""
     _tbb_version_major_string "${_tbb_version_major_string}"
   )
   string(STRIP "${_tbb_version_major_string}" Tbb_VERSION_MAJOR)
 
-  file(STRINGS "${Tbb_INCLUDE_DIR}/tbb/tbb_stddef.h"
+  file(STRINGS ${_tbb_version_file}
      _tbb_version_minor_string REGEX "#define TBB_VERSION_MINOR "
   )
   string(REGEX REPLACE "#define TBB_VERSION_MINOR" ""
@@ -192,6 +206,8 @@ if(EXISTS "${Tbb_INCLUDE_DIR}/tbb/tbb_stddef.h")
   set(Tbb_VERSION ${Tbb_VERSION_MAJOR}.${Tbb_VERSION_MINOR})
 endif()
 
+unset(_tbb_version_file)
+
 # ------------------------------------------------------------------------
 #  Search for TBB lib DIR
 # ------------------------------------------------------------------------

openvdb/openvdb/CMakeLists.txt

@@ -439,6 +439,10 @@ set(OPENVDB_LIBRARY_UTIL_INCLUDE_FILES
   util/Util.h
 )
 
+set(OPENVDB_LIBRARY_THREAD_INCLUDE_FILES
+  thread/Threading.h
+)
+
 # @todo CMake >= 3.12, use an object library to consolidate shared/static
 # builds. There are limitations with earlier versions of CMake when used with
 # imported targets.
@@ -594,3 +598,4 @@ install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INST
 install(FILES ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/tools)
 install(FILES ${OPENVDB_LIBRARY_TREE_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/tree)
 install(FILES ${OPENVDB_LIBRARY_UTIL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/util)
+install(FILES ${OPENVDB_LIBRARY_THREAD_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/thread)

openvdb/openvdb/cmd/openvdb_render.cc

@@ -19,7 +19,14 @@
 #include <OpenEXR/ImfHeader.h>
 #include <OpenEXR/ImfOutputFile.h>
 #include <OpenEXR/ImfPixelType.h>
-#include <tbb/task_scheduler_init.h>
+
+// tbb/task_scheduler_init.h was removed in TBB 2021. The best construct to swap
+// to is tbb/global_control (for executables). global_control was only officially
+// added in TBB 2019U4 but exists in 2018 as a preview feature. To avoid more
+// compile time branching (as we still support 2018), we use it in 2018 too by
+// enabling the below define.
+#define TBB_PREVIEW_GLOBAL_CONTROL 1
+#include <tbb/global_control.h>
 #include <tbb/tick_count.h>
 
 #include <algorithm>
@@ -606,8 +613,12 @@ main(int argc, char *argv[])
     }
 
     try {
-        tbb::task_scheduler_init schedulerInit(
-            (opts.threads == 0) ? tbb::task_scheduler_init::automatic : opts.threads);
+        std::unique_ptr<tbb::global_control> control;
+        if (opts.threads > 0) {
+            // note, opts.threads == 0 means use all threads (default), so don't
+            // manually create a tbb::global_control in this case
+            control.reset(new tbb::global_control(tbb::global_control::max_allowed_parallelism, opts.threads));
+        }
 
         openvdb::initialize();
 

openvdb/openvdb/io/Queue.cc

@@ -5,20 +5,22 @@
 /// @author Peter Cucka
 
 #include "Queue.h"
-
 #include "File.h"
 #include "Stream.h"
-#include <openvdb/Exceptions.h>
-#include <openvdb/util/logging.h>
+#include "openvdb/Exceptions.h"
+#include "openvdb/util/logging.h"
+
 #include <tbb/concurrent_hash_map.h>
-#include <tbb/task.h>
-#include <tbb/tbb_thread.h> // for tbb::this_tbb_thread::sleep()
-#include <tbb/tick_count.h>
+#include <tbb/task_arena.h>
+
+#include <thread>
 #include <algorithm> // for std::max()
 #include <atomic>
 #include <iostream>
 #include <map>
 #include <mutex>
+#include <chrono>
+
 
 namespace openvdb {
 OPENVDB_USE_VERSION_NAMESPACE
@@ -28,18 +30,19 @@ namespace io {
 namespace {
 
 // Abstract base class for queuable TBB tasks that adds a task completion callback
-class Task: public tbb::task
+class Task
 {
 public:
     Task(Queue::Id id): mId(id) {}
-    ~Task() override {}
+    virtual ~Task() {}
 
     Queue::Id id() const { return mId; }
 
     void setNotifier(Queue::Notifier& notifier) { mNotify = notifier; }
+    virtual void execute() const = 0;
 
 protected:
-    void notify(Queue::Status status) { if (mNotify) mNotify(this->id(), status); }
+    void notify(Queue::Status status) const { if (mNotify) mNotify(this->id(), status); }
 
 private:
     Queue::Id mId;
@@ -48,18 +51,18 @@ class Task: public tbb::task
 
 
 // Queuable TBB task that writes one or more grids to a .vdb file or an output stream
-class OutputTask: public Task
+class OutputTask : public Task
 {
 public:
     OutputTask(Queue::Id id, const GridCPtrVec& grids, const Archive& archive,
         const MetaMap& metadata)
         : Task(id)
         , mGrids(grids)
         , mArchive(archive.copy())
-        , mMetadata(metadata)
-    {}
+        , mMetadata(metadata) {}
+    ~OutputTask() override {}
 
-    tbb::task* execute() override
+    void execute() const override
     {
         Queue::Status status = Queue::FAILED;
         try {
@@ -69,10 +72,8 @@ class OutputTask: public Task
             if (const char* msg = e.what()) {
                 OPENVDB_LOG_ERROR(msg);
             }
-        } catch (...) {
-        }
+        } catch (...) {}
         this->notify(status);
-        return nullptr; // no successor to this task
     }
 
 private:
@@ -94,7 +95,6 @@ struct Queue::Impl
     /// @todo Provide more information than just "succeeded" or "failed"?
     using StatusMap = tbb::concurrent_hash_map<Queue::Id, Queue::Status>;
 
-
     Impl()
         : mTimeout(Queue::DEFAULT_TIMEOUT)
         , mCapacity(Queue::DEFAULT_CAPACITY)
@@ -159,12 +159,15 @@ struct Queue::Impl
 
     bool canEnqueue() const { return mNumTasks < Int64(mCapacity); }
 
-    void enqueue(Task& task)
+    void enqueue(OutputTask& task)
     {
-        tbb::tick_count start = tbb::tick_count::now();
+        auto start = std::chrono::steady_clock::now();
         while (!canEnqueue()) {
-            tbb::this_tbb_thread::sleep(tbb::tick_count::interval_t(0.5/*sec*/));
-            if ((tbb::tick_count::now() - start).seconds() > double(mTimeout)) {
+            std::this_thread::sleep_for(/*0.5s*/std::chrono::milliseconds(500));
+            auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                std::chrono::steady_clock::now() - start);
+            const double seconds = double(duration.count()) / 1000.0;
+            if (seconds > double(mTimeout)) {
                 OPENVDB_THROW(RuntimeError,
                     "unable to queue I/O task; " << mTimeout << "-second time limit expired");
             }
@@ -173,7 +176,10 @@ struct Queue::Impl
             std::placeholders::_1, std::placeholders::_2);
         task.setNotifier(notify);
         this->setStatus(task.id(), Queue::PENDING);
-        tbb::task::enqueue(task);
+
+        // get the global task arena
+        tbb::task_arena arena(tbb::task_arena::attach{});
+        arena.enqueue([task = std::move(task)] { task.execute(); });
         ++mNumTasks;
     }
 
@@ -204,7 +210,7 @@ Queue::~Queue()
     /// (e.g., by keeping a static registry of queues that also dispatches
     /// or blocks notifications)?
     while (mImpl->mNumTasks > 0) {
-        tbb::this_tbb_thread::sleep(tbb::tick_count::interval_t(0.5/*sec*/));
+        std::this_thread::sleep_for(/*0.5s*/std::chrono::milliseconds(500));
     }
 }
 
@@ -290,16 +296,8 @@ Queue::Id
 Queue::writeGridVec(const GridCPtrVec& grids, const Archive& archive, const MetaMap& metadata)
 {
     const Queue::Id taskId = mImpl->mNextId++;
-    // From the "GUI Thread" chapter in the TBB Design Patterns guide
-    OutputTask* task =
-        new(tbb::task::allocate_root()) OutputTask(taskId, grids, archive, metadata);
-    try {
-        mImpl->enqueue(*task);
-    } catch (openvdb::RuntimeError&) {
-        // Destroy the task if it could not be enqueued, then rethrow the exception.
-        tbb::task::destroy(*task);
-        throw;
-    }
+    OutputTask task(taskId, grids, archive, metadata);
+    mImpl->enqueue(task);
     return taskId;
 }
 

openvdb/openvdb/math/Coord.h

@@ -13,8 +13,7 @@
 #include "Math.h"
 #include "Vec3.h"
 
-namespace tbb { class split; } // forward declaration
-
+#include <tbb/blocked_range.h> // for tbb::split
 
 namespace openvdb {
 OPENVDB_USE_VERSION_NAMESPACE

openvdb/openvdb/points/PointSample.h

@@ -10,16 +10,16 @@
 #ifndef OPENVDB_POINTS_POINT_SAMPLE_HAS_BEEN_INCLUDED
 #define OPENVDB_POINTS_POINT_SAMPLE_HAS_BEEN_INCLUDED
 
-#include <openvdb/util/NullInterrupter.h>
-#include <openvdb/tools/Interpolation.h>
+#include "openvdb/util/NullInterrupter.h"
+#include "openvdb/thread/Threading.h"
+#include "openvdb/tools/Interpolation.h"
 
 #include "PointDataGrid.h"
 #include "PointAttribute.h"
 
 #include <sstream>
 #include <type_traits>
 
-
 namespace openvdb {
 OPENVDB_USE_VERSION_NAMESPACE
 namespace OPENVDB_VERSION_NAME {
@@ -300,7 +300,7 @@ class PointDataSampler
             using TargetHandleT = AttributeWriteHandle<typename SamplerWrapperT::ValueType>;
 
             if (util::wasInterrupted(interrupter)) {
-                tbb::task::self().cancel_group_execution();
+                thread::cancelGroupExecution();
                 return;
             }
 

openvdb/openvdb/thread/Threading.h

@@ -0,0 +1,54 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: MPL-2.0
+//
+/// @file Threading.h
+
+#ifndef OPENVDB_THREAD_THREADING_HAS_BEEN_INCLUDED
+#define OPENVDB_THREAD_THREADING_HAS_BEEN_INCLUDED
+
+#include "openvdb/version.h"
+
+/// @note tbb/blocked_range.h is the ONLY include that persists from TBB 2020
+///   to TBB 2021 that itself includes the TBB specific version header files.
+///   In TBB 2020, the version header was called tbb/stddef.h. In 2021, it's
+///   called tbb/version.h. We include tbb/blocked_range.h here to indirectly
+///   access the version defines in a consistent way so that downstream
+///   software doesn't need to provide compile time defines.
+#include <tbb/blocked_range.h>
+#include <tbb/task.h>
+#include <tbb/task_group.h>
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace thread {
+
+inline bool cancelGroupExecution()
+{
+    // @note 12000 was the 2021.1-beta05 release. The 2021.1-beta08 release
+    //   introduced current_context().
+#if TBB_INTERFACE_VERSION >= 12002
+    auto ctx = tbb::task::current_context();
+    return ctx ? ctx->cancel_group_execution() : false;
+#else
+    return tbb::task::self().cancel_group_execution();
+#endif
+}
+
+inline bool isGroupExecutionCancelled()
+{
+    // @note 12000 was the 2021.1-beta05 release. The 2021.1-beta08 release
+    //   introduced current_context().
+#if TBB_INTERFACE_VERSION >= 12002
+    auto ctx = tbb::task::current_context();
+    return ctx ? ctx->is_group_execution_cancelled() : false;
+#else
+    return tbb::task::self().is_cancelled();
+#endif
+}
+
+} // namespace thread
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_THREAD_THREADING_HAS_BEEN_INCLUDED

openvdb/openvdb/tools/Composite.h

@@ -24,7 +24,6 @@
 #include <tbb/parallel_for.h>
 #include <tbb/parallel_reduce.h>
 #include <tbb/task_group.h>
-#include <tbb/task_scheduler_init.h>
 
 #include <type_traits>
 #include <functional>