Job system for activity tracking

Add a registry system centered on `class Registry`. The system supports automatic first-pass registration at construction time.
Add a job system, a set of base classes for jobs. The job system is generic, with a mixin class to add application-specific behavior. The mixin class for the library deals with `StorageManager` and `ContextResources`, etc. There's only one remaining `StorageManager` member variable; it's in the mixin class. The `storage_manager()` accessor functions are also in the same place.
`class JobRoot` is now a base class of `class Context`, now split into `ContextBase` and `Context` so that the storage manager member variable is available for the `JobRoot` constructor.
`class JobBranch` is now a base class of `class Query` and `class Consolidator`, among others. Rework all the constructors to take a single `JobParent` argument rather than separate arguments for resources, cancellation sources, and storage managers.
`JobParent::make_cancellation_source` is now the sole source of `CancellationSource` objects.
Add `Context::cancel_all_tasks()`, currently as a proxy for the same function in `StorageManager`. This allowed removal of the accessor `storage_manager()` in the context C API handle. Remove now-unused `tiledb_ctx_handle_t::make_cancellation_source()`.

Author: eric-hughes-tiledb

test/src/unit-Reader.cc

@@ -175,7 +175,7 @@ TEST_CASE_METHOD(
       array.memory_tracker(),
       tracker_,
       lq_state_machine,
-      CancellationSource(context.storage_manager()),
+      context.make_cancellation_source(),
       array.opened_array(),
       config,
       nullopt,

test/src/unit-enumerations.cc

@@ -2398,18 +2398,10 @@ TEST_CASE_METHOD(
   auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ);
   qc1.set_use_enumeration(false);
 
-  Query q1(
-      ctx_.resources(),
-      ctx_.cancellation_source(),
-      ctx_.storage_manager(),
-      array);
+  Query q1(ctx_, array);
   throw_if_not_ok(q1.set_condition(qc1));
 
-  Query q2(
-      ctx_.resources(),
-      ctx_.cancellation_source(),
-      ctx_.storage_manager(),
-      array);
+  Query q2(ctx_, array);
   ser_des_query(&q1, &q2, client_side, ser_type);
 
   auto qc2 = q2.condition();
@@ -2442,18 +2434,10 @@ TEST_CASE_METHOD(
 
   throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3));
 
-  Query q1(
-      ctx_.resources(),
-      ctx_.cancellation_source(),
-      ctx_.storage_manager(),
-      array);
+  Query q1(ctx_, array);
   throw_if_not_ok(q1.set_condition(qc3));
 
-  Query q2(
-      ctx_.resources(),
-      ctx_.cancellation_source(),
-      ctx_.storage_manager(),
-      array);
+  Query q2(ctx_, array);
   ser_des_query(&q1, &q2, client_side, ser_type);
 
   auto qc4 = q2.condition();

tiledb/CMakeLists.txt

@@ -304,6 +304,7 @@ set(TILEDB_CORE_SOURCES
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/cancellation_source.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/context.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/context_resources.cc
+  ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/job.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/storage_manager.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/subarray/range_subset.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/subarray/relevant_fragment_generator.cc

tiledb/api/c_api/context/context_api.cc

@@ -115,7 +115,7 @@ capi_return_t tiledb_ctx_is_supported_fs(
 }
 
 capi_return_t tiledb_ctx_cancel_tasks(tiledb_ctx_t* ctx) {
-  throw_if_not_ok(ctx->storage_manager()->cancel_all_tasks());
+  ctx->cancel_all_tasks();
   return TILEDB_OK;
 }
 

tiledb/api/c_api/context/context_api_internal.h

@@ -68,12 +68,8 @@ struct tiledb_ctx_handle_t
     return ctx_.resources().config();
   }
 
-  inline tiledb::sm::StorageManager* storage_manager() {
-    return ctx_.storage_manager();
-  }
-
-  inline tiledb::sm::CancellationSource cancellation_source() {
-    return ctx_.cancellation_source();
+  inline void cancel_all_tasks() {
+    return ctx_.cancel_all_tasks();
   }
 
   inline tiledb::sm::RestClient& rest_client() {

tiledb/api/c_api/group/group_api.cc

@@ -496,7 +496,7 @@ capi_return_t tiledb_group_consolidate_metadata(
   ensure_group_uri_argument_is_valid(group_uri);
 
   auto cfg = (config == nullptr) ? ctx->config() : config->config();
-  tiledb::sm::Group::consolidate_metadata(ctx->resources(), group_uri, cfg);
+  tiledb::sm::Group::consolidate_metadata(ctx->context(), group_uri, cfg);
 
   return TILEDB_OK;
 }
@@ -506,7 +506,7 @@ capi_return_t tiledb_group_vacuum_metadata(
   ensure_group_uri_argument_is_valid(group_uri);
 
   auto cfg = (config == nullptr) ? ctx->config() : config->config();
-  sm::Group::vacuum_metadata(ctx->resources(), group_uri, cfg);
+  sm::Group::vacuum_metadata(ctx->context(), group_uri, cfg);
 
   return TILEDB_OK;
 }

tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h

@@ -50,13 +50,15 @@ class StorageManagerStub {
   static constexpr bool is_overriding_class = true;
   StorageManagerStub(
       ContextResources&,
-      const std::shared_ptr<common::Logger>&,
       const Config& config)
       : config_(config) {
   }
 
-  inline Status cancel_all_tasks() {
-    return Status{};
+  void cancel_all_tasks() {
+  }
+
+  bool cancellation_in_progress() const {
+    return false;
   }
 };
 

tiledb/common/CMakeLists.txt

@@ -44,6 +44,7 @@ add_subdirectory(exception)
 add_subdirectory(governor)
 add_subdirectory(interval)
 add_subdirectory(random)
+add_subdirectory(registry)
 add_subdirectory(thread_pool)
 add_subdirectory(types)
 add_subdirectory(util)

tiledb/common/registry/CMakeLists.txt

@@ -0,0 +1,26 @@
+#
+# tiledb/common/registry/CMakeLists.txt
+#
+# The MIT License
+#
+# Copyright (c) 2023-2024 TileDB, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+add_test_subdirectory()