Unpin universal-pathlib (#620)

* Unpin universal-pathlib

* Set minimum version

* Allow hats to set upath version

* Install nested-pandas from the requirements

* Fix pre-commit

* Fix mypy

Author: camposandro

pyproject.toml

@@ -25,8 +25,7 @@ dependencies = [
     "numpy>=2.2.0,<3", 
     "pandas>=2.0",
     "tqdm>=4.59.0",
-    # NOTE: package PINNED at <0.3.0, see https://github.com/astronomy-commons/lsdb/issues/1047
-    "universal-pathlib>=0.2.2,<0.3.0",
+    "universal-pathlib",
 ]
 
 # On a mac, install optional dependencies with `pip install '.[dev]'` (include the single quotes)

requirements.txt

@@ -1 +1,2 @@
-git+https://github.com/astronomy-commons/hats.git@main
+hats @ git+https://github.com/astronomy-commons/hats.git@main
+nested-pandas @ git+https://github.com/lincc-frameworks/nested-pandas.git@main

src/hats_import/catalog/map_reduce.py

@@ -297,7 +297,7 @@ def reduce_pixel_shards(
         healpix_pixel = HealpixPixel(destination_pixel_order, destination_pixel_number)
         pixel_dir = get_pixel_cache_directory(cache_shard_path, healpix_pixel)
 
-        merged_table = pq.read_table(pixel_dir, schema=schema)
+        merged_table = pq.read_table(pixel_dir.path, filesystem=pixel_dir.fs, schema=schema)
 
         rows_written = len(merged_table)
 

src/hats_import/catalog/resume_plan.py

@@ -295,12 +295,12 @@ def get_alignment_file(
                     threshold=pixel_threshold,
                     drop_empty_siblings=drop_empty_siblings,
                 )
-            with open(file_name, "wb") as pickle_file:
+            with file_name.open("wb") as pickle_file:
                 alignment = np.array([x if x is not None else [-1, -1, 0] for x in alignment], dtype=np.int64)
                 pickle.dump(alignment, pickle_file)
 
         if self.destination_pixel_map is None:
-            with open(file_name, "rb") as pickle_file:
+            with file_name.open("rb") as pickle_file:
                 alignment = pickle.load(pickle_file)
             pixel_list = np.unique(alignment, axis=0)
             self.destination_pixel_map = {

src/hats_import/hipscat_conversion/run_conversion.py

@@ -125,7 +125,7 @@ def _convert_partition_file(pixel, args, schema, ra_column, dec_column):
             / f"Npix={pixel.pixel}.parquet"
         )
 
-        table = pq.read_table(input_file, schema=schema)
+        table = pq.read_table(input_file.path, filesystem=input_file.fs, schema=schema)
 
         table = table.drop_columns(["_hipscat_index", "Norder", "Dir", "Npix"]).add_column(
             0,

src/hats_import/verification/run_verification.py

@@ -125,15 +125,15 @@ def from_args(cls, args: VerificationArguments) -> "Verifier":
                 all_files.append(unquote(child.path))
 
         files_ds = pds.dataset(all_files, filesystem=parquet_fs)
-        metadata_ds = pds.parquet_dataset(
-            hats.io.paths.get_parquet_metadata_pointer(args.input_catalog_path), filesystem=parquet_fs
-        )
+        metadata_pointer = hats.io.paths.get_parquet_metadata_pointer(args.input_catalog_path)
+        metadata_ds = pds.parquet_dataset(metadata_pointer.path, filesystem=parquet_fs)
 
         input_truth_schema = None
         if args.truth_schema is not None:
             input_truth_schema = pds.parquet_dataset(args.truth_schema, filesystem=parquet_fs).schema
+        common_metadata_pointer = hats.io.paths.get_common_metadata_pointer(args.input_catalog_path)
         common_metadata_schema = pds.parquet_dataset(
-            hats.io.paths.get_common_metadata_pointer(args.input_catalog_path), filesystem=parquet_fs
+            common_metadata_pointer.path, filesystem=parquet_fs
         ).schema
         constructed_truth_schema = cls._construct_truth_schema(
             input_truth_schema=input_truth_schema, common_metadata_schema=common_metadata_schema
@@ -458,7 +458,7 @@ def write_results(self) -> None:
         """Write the verification results to file at `args.output_path` / `args.output_filename`."""
         self.args.output_file_path.parent.mkdir(exist_ok=True, parents=True)
         # Write provenance info
-        with open(self.args.output_file_path, self.args.write_mode, encoding="utf8") as fout:
+        with self.args.output_file_path.open(self.args.write_mode, encoding="utf8") as fout:
             fout.writelines(
                 [
                     "# HATS verification results for\n",