chore: restore RowsIndex.partial

Author: kszucs

libs/libcommon/src/libcommon/parquet_utils.py

@@ -191,6 +191,7 @@ class ParquetIndexWithMetadata:
     features: Features
     httpfs: HTTPFileSystem
     max_arrow_data_in_memory: int
+    partial: bool
     metadata_dir: Path
 
     file_offsets: np.ndarray = field(init=False)
@@ -400,6 +401,7 @@ def from_parquet_metadata_items(
         if not parquet_file_metadata_items:
             raise EmptyParquetMetadataError("No parquet files found.")
 
+        partial = parquet_export_is_partial(parquet_file_metadata_items[0]["url"])
         metadata_dir = Path(parquet_metadata_directory)
 
         with StepProfiler(
@@ -423,6 +425,7 @@ def from_parquet_metadata_items(
             features=features,
             httpfs=httpfs,
             max_arrow_data_in_memory=max_arrow_data_in_memory,
+            partial=partial,
             metadata_dir=metadata_dir,
         )
 

services/worker/tests/job_runners/config/test_parquet_metadata.py

@@ -404,6 +404,7 @@ def test_ParquetIndexWithMetadata_query(
         features=features,
         httpfs=httpfs,
         max_arrow_data_in_memory=999999999,
+        partial=False,
         metadata_dir=metadata_dir,
     )
     with patch("libcommon.parquet_utils.HTTPFile", AuthenticatedHTTPFile):