chore: debug

Author: kszucs

libs/libcommon/src/libcommon/constants.py

@@ -107,4 +107,4 @@
 
 YAML_FIELDS_TO_CHECK = ["dataset_info", "configs", "viewer", "language"]
 
-USE_LIBVIEWER_FOR_DATASETS = False
+USE_LIBVIEWER_FOR_DATASETS = True

libs/libviewer/src/dataset.rs

@@ -150,7 +150,7 @@ impl Dataset {
                 Some(rows) => (None, rows),
                 None => {
                     let metadata =
-                        read_metadata(self.metadata_store.clone(), file.metadata_path.as_ref())
+                        read_metadata(self.metadata_store.clone(), file.metadata_path.as_ref(), file.size)
                             .await?;
                     let num_rows = metadata.file_metadata().num_rows() as u64;
                     (Some(metadata), num_rows)
@@ -173,7 +173,7 @@ impl Dataset {
                 let metadata = match metadata {
                     Some(meta) => meta,
                     None => {
-                        read_metadata(self.metadata_store.clone(), file.metadata_path.as_ref())
+                        read_metadata(self.metadata_store.clone(), file.metadata_path.as_ref(), file.size)
                             .await?
                     }
                 };
@@ -199,7 +199,7 @@ impl Dataset {
             .unwrap_or(&self.files)
             .iter()
             .map(async move |file| {
-                let metadata = read_metadata(self.data_store.clone(), file.path.as_ref()).await?;
+                let metadata = read_metadata(self.data_store.clone(), file.path.as_ref(), file.size).await?;
                 write_metadata(
                     metadata,
                     self.metadata_store.clone(),

libs/libviewer/src/parquet.rs

@@ -75,6 +75,7 @@ impl<T: AsyncFileReader> AsyncFileReader for LimitedAsyncReader<T> {
 pub async fn read_metadata(
     store: Arc<dyn ObjectStore>,
     path: impl Into<Path>,
+    size: Option<u64>,
 ) -> Result<Arc<ParquetMetaData>> {
     let path = path.into();
 
@@ -84,22 +85,11 @@ pub async fn read_metadata(
         .with_offset_index_policy(PageIndexPolicy::Optional);
     // .with_prefetch_hint(16 * 1024);
 
-    // TODO(kszucs): if file_size is known then use load_and_finish
-    // let metadata = if let Some(file_size) = self.file_size {
-    //     metadata.load_and_finish(self, file_size).await?
-    // } else {
-    //     metadata.load_via_suffix_and_finish(self).await?
-    // };
-
-    let metadata = metadata_reader
-        .load_via_suffix_and_finish(&mut object_reader)
-        .await
-        .map_err(|e| {
-            ParquetError::General(format!(
-                "Failed to read metadata from path '{}' in store: {}: {}",
-                path, store, e
-            ))
-        })?;
+    let metadata = if let Some(file_size) = size {
+        metadata_reader.load_and_finish(&mut object_reader, file_size).await?
+    } else {
+        metadata_reader.load_via_suffix_and_finish(&mut object_reader).await?
+    };
 
     Ok(Arc::new(metadata))
 }