refactor(libcommon): remove unused hf_token in parquet_utils (#3249)

refactor(libcommon): remove unused hf_token in `parquet_utils`

Author: kszucs

libs/libcommon/src/libcommon/parquet_utils.py

@@ -189,7 +189,6 @@ class ParquetIndexWithMetadata:
     num_bytes: list[int]
     num_rows: list[int]
     httpfs: HTTPFileSystem
-    hf_token: Optional[str]
     max_arrow_data_in_memory: int
     partial: bool
 
@@ -458,7 +457,6 @@ def from_parquet_metadata_items(
         features: Optional[Features],
         parquet_metadata_directory: StrPath,
         httpfs: HTTPFileSystem,
-        hf_token: Optional[str],
         max_arrow_data_in_memory: int,
         unsupported_features: list[FeatureType] = [],
     ) -> "ParquetIndexWithMetadata":
@@ -503,7 +501,6 @@ def from_parquet_metadata_items(
             num_bytes=num_bytes,
             num_rows=num_rows,
             httpfs=httpfs,
-            hf_token=hf_token,
             max_arrow_data_in_memory=max_arrow_data_in_memory,
             partial=partial,
         )
@@ -516,7 +513,6 @@ def __init__(
         config: str,
         split: str,
         httpfs: HfFileSystem,
-        hf_token: Optional[str],
         parquet_metadata_directory: StrPath,
         max_arrow_data_in_memory: int,
         unsupported_features: list[FeatureType] = [],
@@ -526,15 +522,13 @@ def __init__(
         self.split = split
         self.httpfs = httpfs
         self.parquet_index = self._init_parquet_index(
-            hf_token=hf_token,
             parquet_metadata_directory=parquet_metadata_directory,
             max_arrow_data_in_memory=max_arrow_data_in_memory,
             unsupported_features=unsupported_features,
         )
 
     def _init_parquet_index(
         self,
-        hf_token: Optional[str],
         parquet_metadata_directory: StrPath,
         max_arrow_data_in_memory: int,
         unsupported_features: list[FeatureType] = [],
@@ -566,7 +560,6 @@ def _init_parquet_index(
                 features=features,
                 parquet_metadata_directory=parquet_metadata_directory,
                 httpfs=self.httpfs,
-                hf_token=hf_token,
                 max_arrow_data_in_memory=max_arrow_data_in_memory,
                 unsupported_features=unsupported_features,
             )
@@ -623,11 +616,9 @@ def __init__(
         max_arrow_data_in_memory: int,
         unsupported_features: list[FeatureType] = [],
         all_columns_supported_datasets_allow_list: Union[Literal["all"], list[str]] = "all",
-        hf_token: Optional[str] = None,
     ):
         self.parquet_metadata_directory = parquet_metadata_directory
         self.httpfs = httpfs
-        self.hf_token = hf_token
         self.max_arrow_data_in_memory = max_arrow_data_in_memory
         self.unsupported_features = unsupported_features
         self.all_columns_supported_datasets_allow_list = all_columns_supported_datasets_allow_list
@@ -649,7 +640,6 @@ def get_rows_index(
             config=config,
             split=split,
             httpfs=self.httpfs,
-            hf_token=self.hf_token,
             parquet_metadata_directory=self.parquet_metadata_directory,
             max_arrow_data_in_memory=self.max_arrow_data_in_memory,
             unsupported_features=unsupported_features,

libs/libcommon/tests/test_parquet_utils.py

@@ -378,7 +378,6 @@ def rows_index_with_too_big_rows(
     dataset_sharded_with_config_parquet_metadata: dict[str, Any],
 ) -> Generator[RowsIndex, None, None]:
     indexer = Indexer(
-        hf_token="token",
         parquet_metadata_directory=parquet_metadata_directory,
         httpfs=HTTPFileSystem(),
         max_arrow_data_in_memory=1,
@@ -393,7 +392,6 @@ def indexer(
     parquet_metadata_directory: StrPath,
 ) -> Indexer:
     return Indexer(
-        hf_token="token",
         parquet_metadata_directory=parquet_metadata_directory,
         httpfs=HTTPFileSystem(),
         max_arrow_data_in_memory=9999999999,

services/rows/src/rows/routes/rows.py

@@ -54,7 +54,6 @@ def create_rows_endpoint(
     storage_clients: Optional[list[StorageClient]] = None,
 ) -> Endpoint:
     indexer = Indexer(
-        hf_token=hf_token,
         parquet_metadata_directory=parquet_metadata_directory,
         httpfs=HTTPFileSystem(headers={"authorization": f"Bearer {hf_token}"}),
         max_arrow_data_in_memory=max_arrow_data_in_memory,

services/worker/src/worker/job_runners/split/first_rows.py

@@ -294,7 +294,6 @@ def __init__(
         self.first_rows_config = app_config.first_rows
         self.parquet_metadata_directory = parquet_metadata_directory
         self.indexer = Indexer(
-            hf_token=self.app_config.common.hf_token,
             parquet_metadata_directory=parquet_metadata_directory,
             httpfs=HTTPFileSystem(headers={"authorization": f"Bearer {self.app_config.common.hf_token}"}),
             unsupported_features=[],

services/worker/tests/job_runners/config/test_parquet_metadata.py

@@ -395,7 +395,6 @@ def test_ParquetIndexWithMetadata_query(
         num_rows=[num_rows],
         num_bytes=[num_bytes],
         httpfs=httpfs,
-        hf_token=CI_USER_TOKEN,
         max_arrow_data_in_memory=999999999,
         partial=False,
     )