separate the pyarrow filter pushdown to separate file, cleanup direct imports

Author: Tishj

Makefile

@@ -0,0 +1,4 @@
+PYTHON ?= python3
+
+format-main:
+	$(PYTHON) external/duckdb/scripts/format.py main --fix --noconfirm

scripts/cache_data.json

@@ -7,7 +7,19 @@
             "pyarrow.dataset",
             "pyarrow.Table",
             "pyarrow.RecordBatchReader",
-            "pyarrow.ipc"
+            "pyarrow.ipc",
+            "pyarrow.scalar",
+            "pyarrow.date32",
+            "pyarrow.time64",
+            "pyarrow.timestamp",
+            "pyarrow.uint8",
+            "pyarrow.uint16",
+            "pyarrow.uint32",
+            "pyarrow.uint64",
+            "pyarrow.binary_view",
+            "pyarrow.decimal32",
+            "pyarrow.decimal64",
+            "pyarrow.decimal128"
         ]
     },
     "pyarrow.dataset": {
@@ -709,5 +721,77 @@
         "name": "duckdb_source",
         "children": [],
         "required": false
+    },
+    "pyarrow.scalar": {
+        "type": "attribute",
+        "full_path": "pyarrow.scalar",
+        "name": "scalar",
+        "children": []
+    },
+    "pyarrow.date32": {
+        "type": "attribute",
+        "full_path": "pyarrow.date32",
+        "name": "date32",
+        "children": []
+    },
+    "pyarrow.time64": {
+        "type": "attribute",
+        "full_path": "pyarrow.time64",
+        "name": "time64",
+        "children": []
+    },
+    "pyarrow.timestamp": {
+        "type": "attribute",
+        "full_path": "pyarrow.timestamp",
+        "name": "timestamp",
+        "children": []
+    },
+    "pyarrow.uint8": {
+        "type": "attribute",
+        "full_path": "pyarrow.uint8",
+        "name": "uint8",
+        "children": []
+    },
+    "pyarrow.uint16": {
+        "type": "attribute",
+        "full_path": "pyarrow.uint16",
+        "name": "uint16",
+        "children": []
+    },
+    "pyarrow.uint32": {
+        "type": "attribute",
+        "full_path": "pyarrow.uint32",
+        "name": "uint32",
+        "children": []
+    },
+    "pyarrow.uint64": {
+        "type": "attribute",
+        "full_path": "pyarrow.uint64",
+        "name": "uint64",
+        "children": []
+    },
+    "pyarrow.binary_view": {
+        "type": "attribute",
+        "full_path": "pyarrow.binary_view",
+        "name": "binary_view",
+        "children": []
+    },
+    "pyarrow.decimal32": {
+        "type": "attribute",
+        "full_path": "pyarrow.decimal32",
+        "name": "decimal32",
+        "children": []
+    },
+    "pyarrow.decimal64": {
+        "type": "attribute",
+        "full_path": "pyarrow.decimal64",
+        "name": "decimal64",
+        "children": []
+    },
+    "pyarrow.decimal128": {
+        "type": "attribute",
+        "full_path": "pyarrow.decimal128",
+        "name": "decimal128",
+        "children": []
     }
 }

scripts/generate_import_cache_cpp.py

@@ -182,7 +182,7 @@ def to_string(self):
 
 for file in files:
     content = file.to_string()
-    path = f'src/include/duckdb_python/import_cache/modules/{file.file_name}'
+    path = f'src/duckdb_py/include/duckdb_python/import_cache/modules/{file.file_name}'
     import_cache_path = os.path.join(script_dir, '..', path)
     with open(import_cache_path, "w") as f:
         f.write(content)
@@ -237,7 +237,9 @@ def get_root_modules(files: List[ModuleFile]):
 
 """
 
-import_cache_path = os.path.join(script_dir, '..', 'src/include/duckdb_python/import_cache/python_import_cache.hpp')
+import_cache_path = os.path.join(
+    script_dir, '..', 'src/duckdb_py/include/duckdb_python/import_cache/python_import_cache.hpp'
+)
 with open(import_cache_path, "w") as f:
     f.write(import_cache_file)
 
@@ -252,7 +254,7 @@ def get_module_file_path_includes(files: List[ModuleFile]):
 module_includes = get_module_file_path_includes(files)
 
 modules_header = os.path.join(
-    script_dir, '..', 'src/include/duckdb_python/import_cache/python_import_cache_modules.hpp'
+    script_dir, '..', 'src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp'
 )
 with open(modules_header, "w") as f:
     f.write(module_includes)

scripts/generate_import_cache_json.py

@@ -170,7 +170,6 @@ def update_json(existing: dict, new: dict) -> dict:
 
         # If both values are dictionaries, update recursively.
         if isinstance(new_value, dict) and isinstance(old_value, dict):
-            print(key)
             updated = update_json(old_value, new_value)
             existing[key] = updated
         else:

scripts/imports.py

@@ -6,6 +6,22 @@
 pyarrow.Table
 pyarrow.RecordBatchReader
 pyarrow.ipc.MessageReader
+pyarrow.scalar
+pyarrow.date32
+pyarrow.time64
+pyarrow.timestamp
+pyarrow.timestamp
+pyarrow.timestamp
+pyarrow.timestamp
+pyarrow.timestamp
+pyarrow.uint8
+pyarrow.uint16
+pyarrow.uint32
+pyarrow.uint64
+pyarrow.binary_view
+pyarrow.decimal32
+pyarrow.decimal64
+pyarrow.decimal128
 
 import pandas
 

src/duckdb_py/arrow/CMakeLists.txt

@@ -1,4 +1,5 @@
 # this is used for clang-tidy checks
-add_library(python_arrow OBJECT arrow_array_stream.cpp arrow_export_utils.cpp)
+add_library(python_arrow OBJECT arrow_array_stream.cpp arrow_export_utils.cpp
+                                pyarrow_filter_pushdown.cpp)
 
 target_link_libraries(python_arrow PRIVATE _duckdb_dependencies)