concurrent parquet upload and optimise manifest and snapshot update

Author: nikhilsinhaparseable

Cargo.lock

@@ -123,6 +123,7 @@ clokwerk = "0.4"
 derive_more = { version = "1", features = ["full"] }
 itertools = "0.14"
 once_cell = "1.20"
+rayon = "1.8"
 rand = "0.8.5"
 regex = "1.7.3"
 reqwest = { version = "0.11.27", default-features = false, features = [