vendor: Update vendored sources to duckdb/duckdb@ef253ee

krlmlr · krlmlr · commit 4720a24eb26b · 2025-03-07T16:13:03.000+01:00
Provide callback when tasks are starting / stopping (duckdb/duckdb#16451) [tests] Add allow_unsigned_extensions require (duckdb/duckdb#16499)
diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp
@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "1-dev241"
+#define DUCKDB_PATCH_VERSION "1-dev248"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 2
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.2.1-dev241"
+#define DUCKDB_VERSION "v1.2.1-dev248"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "fba0ffea7e"
+#define DUCKDB_SOURCE_ID "ef253eea8e"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"
diff --git a/src/duckdb/src/include/duckdb/main/client_context_state.hpp b/src/duckdb/src/include/duckdb/main/client_context_state.hpp
@@ -80,6 +80,10 @@ class ClientContextState {
 	}
 	virtual void WriteProfilingInformation(std::ostream &ss) {
 	}
+	virtual void OnTaskStart(ClientContext &context) {
+	}
+	virtual void OnTaskStop(ClientContext &context) {
+	}
 
 public:
 	template <class TARGET>
diff --git a/src/duckdb/src/include/duckdb/parallel/executor_task.hpp b/src/duckdb/src/include/duckdb/parallel/executor_task.hpp
@@ -9,10 +9,10 @@
 #pragma once
 
 #include "duckdb/parallel/task.hpp"
-#include "duckdb/parallel/event.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 
 namespace duckdb {
+class Event;
 class PhysicalOperator;
 class ThreadContext;
 
@@ -34,6 +34,9 @@ class ExecutorTask : public Task {
 	unique_ptr<ThreadContext> thread_context;
 	optional_ptr<const PhysicalOperator> op;
 
+private:
+	ClientContext &context;
+
 public:
 	virtual TaskExecutionResult ExecuteTask(TaskExecutionMode mode) = 0;
 	TaskExecutionResult Execute(TaskExecutionMode mode) override;
diff --git a/src/duckdb/src/include/duckdb/parallel/pipeline.hpp b/src/duckdb/src/include/duckdb/parallel/pipeline.hpp
@@ -20,7 +20,6 @@
 namespace duckdb {
 
 class Executor;
-class Event;
 class MetaPipeline;
 class PipelineExecutor;
 class Pipeline;
diff --git a/src/duckdb/src/include/duckdb/parallel/task_executor.hpp b/src/duckdb/src/include/duckdb/parallel/task_executor.hpp
@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/parallel/task.hpp"
 #include "duckdb/execution/task_error_manager.hpp"
 
@@ -47,6 +48,8 @@ class TaskExecutor {
 	unique_ptr<ProducerToken> token;
 	atomic<idx_t> completed_tasks;
 	atomic<idx_t> total_tasks;
+	friend class BaseExecutorTask;
+	optional_ptr<ClientContext> context;
 };
 
 class BaseExecutorTask : public Task {
diff --git a/src/duckdb/src/include/duckdb/parallel/task_notifier.hpp b/src/duckdb/src/include/duckdb/parallel/task_notifier.hpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parallel/task_notifier.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/optional_ptr.hpp"
+
+namespace duckdb {
+class ClientContext;
+
+//! The TaskNotifier notifies ClientContextState listener about started / stopped tasks
+class TaskNotifier {
+public:
+	explicit TaskNotifier(optional_ptr<ClientContext> context_p);
+
+	~TaskNotifier();
+
+private:
+	optional_ptr<ClientContext> context;
+};
+
+} // namespace duckdb
diff --git a/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp b/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp
@@ -23,7 +23,7 @@ class TableStatistics;
 //! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
 class TableDataWriter {
 public:
-	explicit TableDataWriter(TableCatalogEntry &table);
+	explicit TableDataWriter(TableCatalogEntry &table, optional_ptr<ClientContext> client_context);
 	virtual ~TableDataWriter();
 
 public:
@@ -39,9 +39,11 @@ class TableDataWriter {
 
 	TaskScheduler &GetScheduler();
 	DatabaseInstance &GetDatabase();
+	optional_ptr<ClientContext> GetClientContext();
 
 protected:
 	DuckTableEntry &table;
+	optional_ptr<ClientContext> client_context;
 	//! Pointers to the start of each row group.
 	vector<RowGroupPointer> row_group_pointers;
 };
diff --git a/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp b/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp
@@ -97,7 +97,8 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
 	friend class SingleFileTableDataWriter;
 
 public:
-	SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager, CheckpointType checkpoint_type);
+	SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context, AttachedDatabase &db,
+	                           BlockManager &block_manager, CheckpointType checkpoint_type);
 
 	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
@@ -112,10 +113,15 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
 		return checkpoint_type;
 	}
 
+	optional_ptr<ClientContext> GetClientContext() const {
+		return client_context;
+	}
+
 public:
 	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
 
 private:
+	optional_ptr<ClientContext> client_context;
 	//! The metadata writer is responsible for writing schema information
 	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks
diff --git a/src/duckdb/src/include/duckdb/storage/storage_manager.hpp b/src/duckdb/src/include/duckdb/storage/storage_manager.hpp
@@ -100,7 +100,8 @@ class StorageManager {
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
-	virtual void CreateCheckpoint(CheckpointOptions options = CheckpointOptions()) = 0;
+	virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
+	                              CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -159,7 +160,7 @@ class SingleFileStorageManager : public StorageManager {
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
-	void CreateCheckpoint(CheckpointOptions options) override;
+	void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
diff --git a/src/duckdb/src/main/attached_database.cpp b/src/duckdb/src/main/attached_database.cpp
@@ -247,7 +247,7 @@ void AttachedDatabase::Close() {
 			}
 			CheckpointOptions options;
 			options.wal_action = CheckpointWALAction::DELETE_WAL;
-			storage->CreateCheckpoint(options);
+			storage->CreateCheckpoint(nullptr, options);
 		}
 	} catch (...) { // NOLINT
 	}
diff --git a/src/duckdb/src/parallel/executor_task.cpp b/src/duckdb/src/parallel/executor_task.cpp
@@ -1,18 +1,19 @@
-#include "duckdb/parallel/task.hpp"
+#include "duckdb/parallel/executor_task.hpp"
+#include "duckdb/parallel/task_notifier.hpp"
 #include "duckdb/execution/executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parallel/thread_context.hpp"
 
 namespace duckdb {
 
 ExecutorTask::ExecutorTask(Executor &executor_p, shared_ptr<Event> event_p)
-    : executor(executor_p), event(std::move(event_p)) {
+    : executor(executor_p), event(std::move(event_p)), context(executor_p.context) {
 	executor.RegisterTask();
 }
 
-ExecutorTask::ExecutorTask(ClientContext &context, shared_ptr<Event> event_p, const PhysicalOperator &op_p)
-    : executor(Executor::Get(context)), event(std::move(event_p)), op(&op_p) {
-	thread_context = make_uniq<ThreadContext>(context);
+ExecutorTask::ExecutorTask(ClientContext &context_p, shared_ptr<Event> event_p, const PhysicalOperator &op_p)
+    : executor(Executor::Get(context_p)), event(std::move(event_p)), op(&op_p), context(context_p) {
+	thread_context = make_uniq<ThreadContext>(context_p);
 	executor.RegisterTask();
 }
 
@@ -38,6 +39,7 @@ TaskExecutionResult ExecutorTask::Execute(TaskExecutionMode mode) {
 		if (thread_context) {
 			TaskExecutionResult result;
 			do {
+				TaskNotifier task_notifier {context};
 				thread_context->profiler.StartOperator(op);
 				// to allow continuous profiling, always execute in small steps
 				result = ExecuteTask(TaskExecutionMode::PROCESS_PARTIAL);
@@ -46,7 +48,9 @@ TaskExecutionResult ExecutorTask::Execute(TaskExecutionMode mode) {
 			} while (mode == TaskExecutionMode::PROCESS_ALL && result == TaskExecutionResult::TASK_NOT_FINISHED);
 			return result;
 		} else {
-			return ExecuteTask(mode);
+			TaskNotifier task_notifier {context};
+			auto result = ExecuteTask(mode);
+			return result;
 		}
 	} catch (std::exception &ex) {
 		executor.PushError(ErrorData(ex));
diff --git a/src/duckdb/src/parallel/task_executor.cpp b/src/duckdb/src/parallel/task_executor.cpp
@@ -1,4 +1,5 @@
 #include "duckdb/parallel/task_executor.hpp"
+#include "duckdb/parallel/task_notifier.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 
 namespace duckdb {
@@ -7,7 +8,8 @@ TaskExecutor::TaskExecutor(TaskScheduler &scheduler)
     : scheduler(scheduler), token(scheduler.CreateProducer()), completed_tasks(0), total_tasks(0) {
 }
 
-TaskExecutor::TaskExecutor(ClientContext &context) : TaskExecutor(TaskScheduler::GetScheduler(context)) {
+TaskExecutor::TaskExecutor(ClientContext &context_p) : TaskExecutor(TaskScheduler::GetScheduler(context_p)) {
+	context = context_p;
 }
 
 TaskExecutor::~TaskExecutor() {
@@ -69,6 +71,7 @@ TaskExecutionResult BaseExecutorTask::Execute(TaskExecutionMode mode) {
 		return TaskExecutionResult::TASK_FINISHED;
 	}
 	try {
+		TaskNotifier task_notifier {executor.context};
 		ExecuteTask();
 		executor.FinishTask();
 		return TaskExecutionResult::TASK_FINISHED;
diff --git a/src/duckdb/src/parallel/task_notifier.cpp b/src/duckdb/src/parallel/task_notifier.cpp
@@ -0,0 +1,23 @@
+#include "duckdb/parallel/task_notifier.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/client_context_state.hpp"
+
+namespace duckdb {
+
+TaskNotifier::TaskNotifier(optional_ptr<ClientContext> context_p) : context(context_p) {
+	if (context) {
+		for (auto &state : context->registered_state->States()) {
+			state->OnTaskStart(*context);
+		}
+	}
+}
+
+TaskNotifier::~TaskNotifier() {
+	if (context) {
+		for (auto &state : context->registered_state->States()) {
+			state->OnTaskStop(*context);
+		}
+	}
+}
+
+} // namespace duckdb
diff --git a/src/duckdb/src/storage/checkpoint/table_data_writer.cpp b/src/duckdb/src/storage/checkpoint/table_data_writer.cpp
@@ -10,7 +10,8 @@
 
 namespace duckdb {
 
-TableDataWriter::TableDataWriter(TableCatalogEntry &table_p) : table(table_p.Cast<DuckTableEntry>()) {
+TableDataWriter::TableDataWriter(TableCatalogEntry &table_p, optional_ptr<ClientContext> client_context_p)
+    : table(table_p.Cast<DuckTableEntry>()), client_context(client_context_p) {
 	D_ASSERT(table_p.IsDuckTable());
 }
 
@@ -40,7 +41,8 @@ DatabaseInstance &TableDataWriter::GetDatabase() {
 
 SingleFileTableDataWriter::SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager,
                                                      TableCatalogEntry &table, MetadataWriter &table_data_writer)
-    : TableDataWriter(table), checkpoint_manager(checkpoint_manager), table_data_writer(table_data_writer) {
+    : TableDataWriter(table, checkpoint_manager.GetClientContext()), checkpoint_manager(checkpoint_manager),
+      table_data_writer(table_data_writer) {
 }
 
 unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup &row_group) {
diff --git a/src/duckdb/src/storage/checkpoint_manager.cpp b/src/duckdb/src/storage/checkpoint_manager.cpp
@@ -36,10 +36,11 @@ namespace duckdb {
 
 void ReorderTableEntries(catalog_entry_vector_t &tables);
 
-SingleFileCheckpointWriter::SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager,
+SingleFileCheckpointWriter::SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context_p,
+                                                       AttachedDatabase &db, BlockManager &block_manager,
                                                        CheckpointType checkpoint_type)
-    : CheckpointWriter(db), partial_block_manager(block_manager, PartialBlockType::FULL_CHECKPOINT),
-      checkpoint_type(checkpoint_type) {
+    : CheckpointWriter(db), client_context(client_context_p),
+      partial_block_manager(block_manager, PartialBlockType::FULL_CHECKPOINT), checkpoint_type(checkpoint_type) {
 }
 
 BlockManager &SingleFileCheckpointWriter::GetBlockManager() {
diff --git a/src/duckdb/src/storage/storage_manager.cpp b/src/duckdb/src/storage/storage_manager.cpp
@@ -355,7 +355,7 @@ bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id)
 	return block_manager->IsRootBlock(checkpoint_id);
 }
 
-void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) {
+void SingleFileStorageManager::CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) {
 	if (InMemory() || read_only || !load_complete) {
 		return;
 	}
@@ -366,7 +366,7 @@ void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) {
 	if (GetWALSize() > 0 || config.options.force_checkpoint || options.action == CheckpointAction::ALWAYS_CHECKPOINT) {
 		// we only need to checkpoint if there is anything in the WAL
 		try {
-			SingleFileCheckpointWriter checkpointer(db, *block_manager, options.type);
+			SingleFileCheckpointWriter checkpointer(client_context, db, *block_manager, options.type);
 			checkpointer.CreateCheckpoint();
 		} catch (std::exception &ex) {
 			ErrorData error(ex);
diff --git a/src/duckdb/src/transaction/duck_transaction_manager.cpp b/src/duckdb/src/transaction/duck_transaction_manager.cpp
@@ -194,7 +194,7 @@ void DuckTransactionManager::Checkpoint(ClientContext &context, bool force) {
 		// we cannot do a full checkpoint if any transaction needs to read old data
 		options.type = CheckpointType::CONCURRENT_CHECKPOINT;
 	}
-	storage_manager.CreateCheckpoint(options);
+	storage_manager.CreateCheckpoint(context, options);
 }
 
 unique_ptr<StorageLockKey> DuckTransactionManager::SharedCheckpointLock() {
@@ -295,7 +295,7 @@ ErrorData DuckTransactionManager::CommitTransaction(ClientContext &context, Tran
 		options.action = CheckpointAction::ALWAYS_CHECKPOINT;
 		options.type = checkpoint_decision.type;
 		auto &storage_manager = db.GetStorageManager();
-		storage_manager.CreateCheckpoint(options);
+		storage_manager.CreateCheckpoint(context, options);
 	}
 	return error;
 }
diff --git a/src/duckdb/ub_src_parallel.cpp b/src/duckdb/ub_src_parallel.cpp
@@ -26,6 +26,8 @@
 
 #include "src/parallel/task_executor.cpp"
 
+#include "src/parallel/task_notifier.cpp"
+
 #include "src/parallel/task_scheduler.cpp"
 
 #include "src/parallel/thread_context.cpp"

Original file line number	Diff line number	Diff line change
`@@ -80,6 +80,10 @@ class ClientContextState {`
`80`	`80`	`}`
`81`	`81`	`virtual void WriteProfilingInformation(std::ostream &ss) {`
`82`	`82`	`}`
	`83`	`+ virtual void OnTaskStart(ClientContext &context) {`
	`84`	`+ }`
	`85`	`+ virtual void OnTaskStop(ClientContext &context) {`
	`86`	`+ }`
`83`	`87`
`84`	`88`	`public:`
`85`	`89`	`template <class TARGET>`
Original file line number	Diff line number	Diff line change
`@@ -247,7 +247,7 @@ void AttachedDatabase::Close() {`
`247`	`247`	`}`
`248`	`248`	`CheckpointOptions options;`
`249`	`249`	`options.wal_action = CheckpointWALAction::DELETE_WAL;`
`250`		`- storage->CreateCheckpoint(options);`
	`250`	`+ storage->CreateCheckpoint(nullptr, options);`
`251`	`251`	`}`
`252`	`252`	`} catch (...) { // NOLINT`
`253`	`253`	`}`