chore: debug

Author: kszucs

.github/workflows/l-libcommon.yml

@@ -38,3 +38,4 @@ jobs:
     with:
       working-directory: libs/libviewer
       poetry-args: "--with dev"
+    secrets: inherit

libs/libcommon/src/libcommon/parquet_utils.py

@@ -15,7 +15,6 @@
 from datasets.table import cast_table_to_schema
 from datasets.utils.py_utils import size_str
 from fsspec.implementations.http import HTTPFile, HTTPFileSystem
-from huggingface_hub import HfFileSystem
 from pyarrow.lib import ArrowInvalid
 
 from libcommon.constants import CONFIG_PARQUET_METADATA_KIND
@@ -405,7 +404,7 @@ def __init__(
         dataset: str,
         config: str,
         split: str,
-        httpfs: HfFileSystem,
+        httpfs: HTTPFileSystem,
         parquet_metadata_directory: StrPath,
         max_arrow_data_in_memory: int,
         max_scan_size: int,
@@ -467,7 +466,7 @@ def _init_dataset_info(self, parquet_metadata_directory: StrPath) -> None:
 
     def _init_parquet_index(
         self,
-        httpfs: HfFileSystem,
+        httpfs: HTTPFileSystem,
         parquet_metadata_directory: StrPath,
         max_arrow_data_in_memory: int,
     ) -> None:

libs/libviewer/src/dataset.rs

@@ -30,7 +30,7 @@ pub enum DatasetError {
     #[error("Arrow error: {0}")]
     Arrow(#[from] arrow::error::ArrowError),
 
-    #[error("Parquet error: {0}")]
+    #[error("{0}")]
     Parquet(#[from] ::parquet::errors::ParquetError),
 
     #[error("Object store error: {0}")]

libs/libviewer/src/parquet.rs

@@ -77,7 +77,6 @@ pub async fn read_metadata(
     path: impl Into<Path>,
 ) -> Result<Arc<ParquetMetaData>> {
     let path = path.into();
-    println!("Reading parquet metadata for {:?} on store {:?}", path, store);
 
     let mut object_reader = ParquetObjectReader::new(store, path.clone());
     let metadata_reader = ParquetMetaDataReader::new()
@@ -94,7 +93,13 @@ pub async fn read_metadata(
 
     let metadata = metadata_reader
         .load_via_suffix_and_finish(&mut object_reader)
-        .await?;
+        .await
+        .map_err(|e| {
+            ParquetError::General(format!(
+                "Failed to read metadata from path '{}' in store: {}: {}",
+                path, store, e
+            ))
+        })?;
 
     Ok(Arc::new(metadata))
 }