Enable Free Threading

pyproject.toml

@@ -72,6 +72,7 @@ minimum-version = "0.10"
 cmake.version = ">=3.29.0"
 ninja.version = ">=1.10"
 ninja.make-fallback = false
+cmake.build-type = "Debug"
 metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
 
 [tool.scikit-build.wheel]
@@ -106,11 +107,11 @@ cmake.define.CMAKE_C_FLAGS = "--coverage -O0"
 cmake.define.CMAKE_SHARED_LINKER_FLAGS = "--coverage"
 
 # Override: if we're in editable mode then make sure a build dir is set. Note that COVERAGE runs have their own
-#   build-dir, and we don't want to interfere with that. We're also disabling unity builds to help with debugging.
+#   build-dir, and we don't want to interfere with that.
 [[tool.scikit-build.overrides]]
 if.state = "editable"
 if.env.COVERAGE = false
-build-dir = "build/debug/"
+build-dir = "build/$UV_PYTHON/"
 editable.rebuild = true
 editable.mode = "redirect"
 cmake.build-type = "Debug"
@@ -379,4 +380,4 @@ manylinux-x86_64-image = "manylinux_2_28"
 manylinux-pypy_x86_64-image = "manylinux_2_28"
 manylinux-aarch64-image = "manylinux_2_28"
 manylinux-pypy_aarch64-image = "manylinux_2_28"
-enable = ["cpython-freethreading", "cpython-prerelease"]
+

src/duckdb_py/CMakeLists.txt

@@ -17,6 +17,7 @@ add_library(python_src OBJECT
         duckdb_python.cpp
         importer.cpp
         map.cpp
+        module_state.cpp
         path_like.cpp
         pyconnection.cpp
         pyexpression.cpp

src/duckdb_py/duckdb_python.cpp

@@ -20,6 +20,7 @@
 #include "duckdb_python/pybind11/conversions/python_udf_type_enum.hpp"
 #include "duckdb_python/pybind11/conversions/python_csv_line_terminator_enum.hpp"
 #include "duckdb/common/enums/statement_type.hpp"
+#include "duckdb_python/module_state.hpp"
 
 #include "duckdb.hpp"
 
@@ -31,6 +32,16 @@ namespace py = pybind11;
 
 namespace duckdb {
 
+// Private function to initialize module state
+void InitializeModuleState(py::module_ &m) {
+	auto state_ptr = new DuckDBPyModuleState();
+	SetModuleState(state_ptr);
+
+	// https://pybind11.readthedocs.io/en/stable/advanced/misc.html#module-destructors
+	auto capsule = py::capsule(state_ptr, [](void *p) { delete static_cast<DuckDBPyModuleState *>(p); });
+	m.attr("__duckdb_state") = capsule;
+}
+
 enum PySQLTokenType : uint8_t {
 	PY_SQL_TOKEN_IDENTIFIER = 0,
 	PY_SQL_TOKEN_NUMERIC_CONSTANT,
@@ -1007,7 +1018,22 @@ static void RegisterExpectedResultType(py::handle &m) {
 	expected_return_type.export_values();
 }
 
-PYBIND11_MODULE(DUCKDB_PYTHON_LIB_NAME, m) { // NOLINT
+// Only mark mod_gil_not_used for 3.14t or later
+// This is to not add support for 3.13t
+// Py_GIL_DISABLED check is not strictly necessary
+#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000
+PYBIND11_MODULE(DUCKDB_PYTHON_LIB_NAME, m, py::mod_gil_not_used(),
+                py::multiple_interpreters::not_supported()) { // NOLINT
+#else
+PYBIND11_MODULE(DUCKDB_PYTHON_LIB_NAME, m,
+                py::multiple_interpreters::not_supported()) { // NOLINT
+#endif
+
+	// Initialize module state completely during initialization
+	// PEP 489 wants calls for state to be module local, but currently
+	// static via g_module_state. 
+	InitializeModuleState(m);
+
 	py::enum_<duckdb::ExplainType>(m, "ExplainType")
 	    .value("STANDARD", duckdb::ExplainType::EXPLAIN_STANDARD)
 	    .value("ANALYZE", duckdb::ExplainType::EXPLAIN_ANALYZE)
@@ -1046,9 +1072,10 @@ PYBIND11_MODULE(DUCKDB_PYTHON_LIB_NAME, m) { // NOLINT
 	m.attr("__version__") = std::string(DuckDB::LibraryVersion()).substr(1);
 	m.attr("__standard_vector_size__") = DuckDB::StandardVectorSize();
 	m.attr("__git_revision__") = DuckDB::SourceID();
-	m.attr("__interactive__") = DuckDBPyConnection::DetectAndGetEnvironment();
-	m.attr("__jupyter__") = DuckDBPyConnection::IsJupyter();
-	m.attr("__formatted_python_version__") = DuckDBPyConnection::FormattedPythonVersion();
+	auto &module_state = GetModuleState();
+	m.attr("__interactive__") = module_state.environment != PythonEnvironmentType::NORMAL;
+	m.attr("__jupyter__") = module_state.environment == PythonEnvironmentType::JUPYTER;
+	m.attr("__formatted_python_version__") = module_state.formatted_python_version;
 	m.def("default_connection", &DuckDBPyConnection::DefaultConnection,
 	      "Retrieve the connection currently registered as the default to be used by the module");
 	m.def("set_default_connection", &DuckDBPyConnection::SetDefaultConnection,

src/duckdb_py/include/duckdb_python/module_state.hpp

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb_python/module_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb_python/pybind11/pybind_wrapper.hpp"
+#include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/main/db_instance_cache.hpp"
+#include "duckdb/main/database.hpp"
+#include "duckdb_python/import_cache/python_import_cache.hpp"
+#include "duckdb_python/pyconnection/pyconnection.hpp"
+#include <pybind11/critical_section.h>
+
+namespace duckdb {
+
+
+// Module state structure to hold per-interpreter state
+struct DuckDBPyModuleState {
+	// TODO: Make private / move behind a thread-safe accessor
+	shared_ptr<DuckDBPyConnection> default_connection_ptr;
+	mutex default_connection_mutex;
+
+	// Python environment tracking
+	PythonEnvironmentType environment = PythonEnvironmentType::NORMAL;
+	string formatted_python_version;
+
+	DuckDBPyModuleState();
+
+	shared_ptr<DuckDBPyConnection> GetDefaultConnection();
+	void SetDefaultConnection(shared_ptr<DuckDBPyConnection> connection);
+	void ClearDefaultConnection();
+
+	PythonImportCache* GetImportCache();
+	void ClearImportCache();
+
+	DBInstanceCache* GetInstanceCache();
+
+	static DuckDBPyModuleState& GetGlobalModuleState();
+	static void SetGlobalModuleState(DuckDBPyModuleState* state);
+
+private:
+	PythonImportCache import_cache;
+	std::unique_ptr<DBInstanceCache> instance_cache;
+#ifdef Py_GIL_DISABLED
+	py::object lock_object;
+#endif
+
+	// Static module state cache for performance optimization
+	// TODO: Replace with proper per-interpreter state for multi-interpreter support
+	static DuckDBPyModuleState* g_module_state;
+
+	// Non-copyable
+	DuckDBPyModuleState(const DuckDBPyModuleState &) = delete;
+	DuckDBPyModuleState &operator=(const DuckDBPyModuleState &) = delete;
+};
+
+DuckDBPyModuleState &GetModuleState();
+void SetModuleState(DuckDBPyModuleState *state);
+
+} // namespace duckdb

src/duckdb_py/include/duckdb_python/pyconnection/pyconnection.hpp

@@ -28,6 +28,7 @@
 
 namespace duckdb {
 struct BoundParameterData;
+struct DuckDBPyModuleState;
 
 enum class PythonEnvironmentType { NORMAL, INTERACTIVE, JUPYTER };
 
@@ -172,8 +173,7 @@ struct DuckDBPyConnection : public enable_shared_from_this<DuckDBPyConnection> {
 	case_insensitive_set_t registered_objects;
 
 public:
-	explicit DuckDBPyConnection() {
-	}
+	DuckDBPyConnection();
 	~DuckDBPyConnection();
 
 public:
@@ -190,9 +190,17 @@ struct DuckDBPyConnection : public enable_shared_from_this<DuckDBPyConnection> {
 	static std::string FormattedPythonVersion();
 	static shared_ptr<DuckDBPyConnection> DefaultConnection();
 	static void SetDefaultConnection(shared_ptr<DuckDBPyConnection> conn);
+	static shared_ptr<DuckDBPyConnection> GetDefaultConnection();
+	static void ClearDefaultConnection();
+	static void ClearImportCache();
 	static PythonImportCache *ImportCache();
 	static bool IsInteractive();
 
+	// Instance methods for optimized module state access
+	bool IsJupyterInstance() const;
+	bool IsInteractiveInstance() const;
+	std::string FormattedPythonVersionInstance() const;
+
 	unique_ptr<DuckDBPyRelation> ReadCSV(const py::object &name, py::kwargs &kwargs);
 
 	py::list ExtractStatements(const string &query);
@@ -337,11 +345,6 @@ struct DuckDBPyConnection : public enable_shared_from_this<DuckDBPyConnection> {
 	py::list ListFilesystems();
 	bool FileSystemIsRegistered(const string &name);
 
-	//! Default connection to an in-memory database
-	static DefaultConnectionHolder default_connection;
-	//! Caches and provides an interface to get frequently used modules+subtypes
-	static shared_ptr<PythonImportCache> import_cache;
-
 	static bool IsPandasDataframe(const py::object &object);
 	static PyArrowObjectType GetArrowType(const py::handle &obj);
 	static bool IsAcceptedArrowObject(const py::object &object);
@@ -357,10 +360,6 @@ struct DuckDBPyConnection : public enable_shared_from_this<DuckDBPyConnection> {
 	                               bool side_effects);
 	void RegisterArrowObject(const py::object &arrow_object, const string &name);
 	vector<unique_ptr<SQLStatement>> GetStatements(const py::object &query);
-
-	static PythonEnvironmentType environment;
-	static std::string formatted_python_version;
-	static void DetectEnvironment();
 };
 
 template <typename T>

src/duckdb_py/module_state.cpp

@@ -0,0 +1,126 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb_python/module_state.cpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#include "duckdb_python/module_state.hpp"
+#include <stdexcept>
+#include <chrono>
+#include <thread>
+
+// Enable debug prints for performance analysis
+#define DEBUG_MODULE_STATE 1
+
+namespace duckdb {
+
+// Forward declaration from pyconnection.cpp
+void InstantiateNewInstance(DuckDB &db);
+
+// Static member initialization - required for all static class members in C++
+DuckDBPyModuleState *DuckDBPyModuleState::g_module_state = nullptr;
+
+// Module state constructor
+DuckDBPyModuleState::DuckDBPyModuleState() {
+	// Create caches
+	instance_cache = make_uniq<DBInstanceCache>();
+	// import_cache: direct object due to frequent calls
+
+#ifdef Py_GIL_DISABLED
+	// Initialize lock object for critical sections
+	// TODO: Consider moving to finer-grained locks
+	lock_object = py::none();
+#endif
+
+	// Detects Python environment and version during intialization
+	// Moved from DuckDBPyConnection::DetectEnvironment()
+	py::module_ sys = py::module_::import("sys");
+	py::object version_info = sys.attr("version_info");
+	int major = py::cast<int>(version_info.attr("major"));
+	int minor = py::cast<int>(version_info.attr("minor"));
+	formatted_python_version = std::to_string(major) + "." + std::to_string(minor);
+
+	// If __main__ does not have a __file__ attribute, we are in interactive mode
+	auto main_module = py::module_::import("__main__");
+	if (!py::hasattr(main_module, "__file__")) {
+		environment = PythonEnvironmentType::INTERACTIVE;
+
+		if (ModuleIsLoaded<IpythonCacheItem>()) {
+			// Check to see if we are in a Jupyter Notebook
+			auto get_ipython = import_cache.IPython.get_ipython();
+			if (get_ipython.ptr() != nullptr) {
+				auto ipython = get_ipython();
+				if (py::hasattr(ipython, "config")) {
+					py::dict ipython_config = ipython.attr("config");
+					if (ipython_config.contains("IPKernelApp")) {
+						environment = PythonEnvironmentType::JUPYTER;
+					}
+				}
+			}
+		}
+	}
+}
+
+DuckDBPyModuleState &DuckDBPyModuleState::GetGlobalModuleState() {
+	// TODO: Externalize this static cache when adding multi-interpreter support
+	// For now, single interpreter assumption allows simple static caching
+	if (!g_module_state) {
+		throw InternalException("Module state not initialized - call SetGlobalModuleState() during module init");
+	}
+	return *g_module_state;
+}
+
+void DuckDBPyModuleState::SetGlobalModuleState(DuckDBPyModuleState *state) {
+#if DEBUG_MODULE_STATE
+	printf("DEBUG: SetGlobalModuleState() called - initializing static cache (built: %s %s)\n", __DATE__, __TIME__);
+#endif
+	g_module_state = state;
+}
+
+DuckDBPyModuleState &GetModuleState() {
+#if DEBUG_MODULE_STATE
+	printf("DEBUG: GetModuleState() called\n");
+#endif
+	return DuckDBPyModuleState::GetGlobalModuleState();
+}
+
+void SetModuleState(DuckDBPyModuleState *state) {
+	DuckDBPyModuleState::SetGlobalModuleState(state);
+}
+
+shared_ptr<DuckDBPyConnection> DuckDBPyModuleState::GetDefaultConnection() {
+	lock_guard<mutex> guard(default_connection_mutex);
+	// Reproduce exact logic from original DefaultConnectionHolder::Get()
+	if (!default_connection_ptr || default_connection_ptr->con.ConnectionIsClosed()) {
+		py::dict config_dict;
+		default_connection_ptr = DuckDBPyConnection::Connect(py::str(":memory:"), false, config_dict);
+	}
+	return default_connection_ptr;
+}
+
+void DuckDBPyModuleState::SetDefaultConnection(shared_ptr<DuckDBPyConnection> connection) {
+	lock_guard<mutex> guard(default_connection_mutex);
+	default_connection_ptr = std::move(connection);
+}
+
+void DuckDBPyModuleState::ClearDefaultConnection() {
+	lock_guard<mutex> guard(default_connection_mutex);
+	default_connection_ptr = nullptr;
+}
+
+PythonImportCache *DuckDBPyModuleState::GetImportCache() {
+	return &import_cache;
+}
+
+void DuckDBPyModuleState::ClearImportCache() {
+	// Direct object will be cleaned up automatically by destructor
+	// TODO: If explicit clearing is needed, add Clear() method to PythonImportCache
+}
+
+DBInstanceCache *DuckDBPyModuleState::GetInstanceCache() {
+	return instance_cache.get();
+}
+
+} // namespace duckdb

src/duckdb_py/native/python_conversion.cpp

@@ -961,6 +961,9 @@ void TransformPythonObjectInternal(py::handle ele, A &result, const B &param, bo
 					break;
 				}
 				if (conversion_target.id() == LogicalTypeId::UBIGINT) {
+					if (PyErr_Occurred()) {
+						PyErr_Clear();
+					}
 					throw InvalidInputException("Python Conversion Failure: Value out of range for type %s",
 					                            conversion_target);
 				}