fix double counting

Author: joocer

opteryx/__version__.py

@@ -1,9 +1,9 @@
 # THIS FILE IS AUTOMATICALLY UPDATED DURING THE BUILD PROCESS
 # DO NOT EDIT THIS FILE DIRECTLY
 
-__build__ = 1714
+__build__ = 1715
 __author__ = "@joocer"
-__version__ = "0.26.0-beta.1714"
+__version__ = "0.26.0-beta.1715"
 
 # Store the version here so:
 # 1) we don't load dependencies by storing it in __init__.py

opteryx/connectors/disk_connector.py

@@ -237,10 +237,9 @@ def read_dataset(
             return
 
         remaining_rows = limit if limit is not None else float("inf")
-        last_morsel = None
 
         def process_result(num_rows, raw_size, decoded):
-            nonlocal remaining_rows, last_morsel
+            nonlocal remaining_rows
             if decoded.num_rows > remaining_rows:
                 decoded = decoded.slice(0, remaining_rows)
             remaining_rows -= decoded.num_rows
@@ -249,7 +248,6 @@ def process_result(num_rows, raw_size, decoded):
             self.rows_seen += num_rows
             self.blobs_seen += 1
             self.statistics.bytes_raw += raw_size
-            last_morsel = decoded
             return decoded
 
         max_workers = min(self._max_workers, len(blob_names)) or 1
@@ -340,12 +338,8 @@ def process_result(num_rows, raw_size, decoded):
                     if remaining_rows <= 0:
                         break
 
-        if last_morsel is not None:
-            self.statistics.columns_read += last_morsel.num_columns
-        elif columns:
-            self.statistics.columns_read += len(columns)
-        elif self.schema:
-            self.statistics.columns_read += len(self.schema.columns)
+        # column-level statistics are recorded by the read node after morsels
+        # leave connector-level accounting to avoid double counting
 
     def _read_blob_task(self, blob_name: str, columns, predicates):
         decoder = get_decoder(blob_name)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "opteryx"
-version = "0.26.0-beta.1714"
+version = "0.26.0-beta.1715"
 description = "Query your data, where it lives"
 requires-python = '>=3.11'
 readme = {file = "README.md", content-type = "text/markdown"}

tests/unit/planner/test_projection_pushdown_parquet.py

@@ -2,7 +2,7 @@
 import sys
 import pytest
 
-sys.path.insert(1, os.path.join(sys.path[0], "../.."))
+sys.path.insert(1, os.path.join(sys.path[0], "../../.."))
 
 import opteryx