refactor(libcommon): remove the effectively unused arguments of Indexer (#3250)

* refactor(libcommon): remove the effectively unused arguments of `Indexer`

* style: remove unnecessarry imports

* refactor(libcommon): remove `unsupported_features` argument from `RowsIndex`

* style: remove unnecessarry imports

Author: kszucs

libs/libcommon/src/libcommon/parquet_utils.py

@@ -5,14 +5,13 @@
 from dataclasses import dataclass, field
 from functools import lru_cache
 from pathlib import Path
-from typing import Literal, Optional, TypedDict, Union
+from typing import Optional, TypedDict
 
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
 from datasets import Features, Value
-from datasets.features.features import FeatureType
 from datasets.table import cast_table_to_schema
 from datasets.utils.py_utils import size_str
 from fsspec.implementations.http import HTTPFile, HTTPFileSystem
@@ -458,7 +457,6 @@ def from_parquet_metadata_items(
         parquet_metadata_directory: StrPath,
         httpfs: HTTPFileSystem,
         max_arrow_data_in_memory: int,
-        unsupported_features: list[FeatureType] = [],
     ) -> "ParquetIndexWithMetadata":
         if not parquet_file_metadata_items:
             raise EmptyParquetMetadataError("No parquet files found.")
@@ -488,9 +486,10 @@ def from_parquet_metadata_items(
         ):
             if features is None:  # config-parquet version<6 didn't have features
                 features = Features.from_arrow_schema(pq.read_schema(metadata_paths[0]))
+            # TODO(kszucs): since unsupported_features is always empty list we may omit the call below
             supported_columns, unsupported_columns = get_supported_unsupported_columns(
                 features,
-                unsupported_features=unsupported_features,
+                unsupported_features=[],
             )
         return ParquetIndexWithMetadata(
             features=features,
@@ -515,7 +514,6 @@ def __init__(
         httpfs: HfFileSystem,
         parquet_metadata_directory: StrPath,
         max_arrow_data_in_memory: int,
-        unsupported_features: list[FeatureType] = [],
     ):
         self.dataset = dataset
         self.config = config
@@ -524,14 +522,12 @@ def __init__(
         self.parquet_index = self._init_parquet_index(
             parquet_metadata_directory=parquet_metadata_directory,
             max_arrow_data_in_memory=max_arrow_data_in_memory,
-            unsupported_features=unsupported_features,
         )
 
     def _init_parquet_index(
         self,
         parquet_metadata_directory: StrPath,
         max_arrow_data_in_memory: int,
-        unsupported_features: list[FeatureType] = [],
     ) -> ParquetIndexWithMetadata:
         with StepProfiler(method="rows_index._init_parquet_index", step="all"):
             # get the list of parquet files
@@ -561,7 +557,6 @@ def _init_parquet_index(
                 parquet_metadata_directory=parquet_metadata_directory,
                 httpfs=self.httpfs,
                 max_arrow_data_in_memory=max_arrow_data_in_memory,
-                unsupported_features=unsupported_features,
             )
 
     # note that this cache size is global for the class, not per instance
@@ -614,14 +609,10 @@ def __init__(
         parquet_metadata_directory: StrPath,
         httpfs: HTTPFileSystem,
         max_arrow_data_in_memory: int,
-        unsupported_features: list[FeatureType] = [],
-        all_columns_supported_datasets_allow_list: Union[Literal["all"], list[str]] = "all",
     ):
         self.parquet_metadata_directory = parquet_metadata_directory
         self.httpfs = httpfs
         self.max_arrow_data_in_memory = max_arrow_data_in_memory
-        self.unsupported_features = unsupported_features
-        self.all_columns_supported_datasets_allow_list = all_columns_supported_datasets_allow_list
 
     @lru_cache(maxsize=1)
     def get_rows_index(
@@ -630,17 +621,11 @@ def get_rows_index(
         config: str,
         split: str,
     ) -> RowsIndex:
-        filter_features = (
-            self.all_columns_supported_datasets_allow_list != "all"
-            and dataset not in self.all_columns_supported_datasets_allow_list
-        )
-        unsupported_features = self.unsupported_features if filter_features else []
         return RowsIndex(
             dataset=dataset,
             config=config,
             split=split,
             httpfs=self.httpfs,
             parquet_metadata_directory=self.parquet_metadata_directory,
             max_arrow_data_in_memory=self.max_arrow_data_in_memory,
-            unsupported_features=unsupported_features,
         )

services/rows/src/rows/routes/rows.py

@@ -3,7 +3,7 @@
 
 import logging
 from http import HTTPStatus
-from typing import Literal, Optional, Union
+from typing import Optional
 
 from fsspec.implementations.http import HTTPFileSystem
 from libapi.authentication import auth_check
@@ -33,11 +33,6 @@
 logger = logging.getLogger(__name__)
 
 
-ALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST: Union[Literal["all"], list[str]] = [
-    "halabi2016/arabic_speech_corpus"
-]  # for testing
-
-
 def create_rows_endpoint(
     cached_assets_storage_client: StorageClient,
     parquet_metadata_directory: StrPath,
@@ -57,7 +52,6 @@ def create_rows_endpoint(
         parquet_metadata_directory=parquet_metadata_directory,
         httpfs=HTTPFileSystem(headers={"authorization": f"Bearer {hf_token}"}),
         max_arrow_data_in_memory=max_arrow_data_in_memory,
-        all_columns_supported_datasets_allow_list=ALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST,
     )
 
     async def rows_endpoint(request: Request) -> Response:

services/worker/src/worker/job_runners/split/first_rows.py

@@ -296,8 +296,6 @@ def __init__(
         self.indexer = Indexer(
             parquet_metadata_directory=parquet_metadata_directory,
             httpfs=HTTPFileSystem(headers={"authorization": f"Bearer {self.app_config.common.hf_token}"}),
-            unsupported_features=[],
-            all_columns_supported_datasets_allow_list="all",
             max_arrow_data_in_memory=app_config.rows_index.max_arrow_data_in_memory,
         )
         self.storage_client = storage_client