diff --git a/src/hats/io/parquet_metadata.py b/src/hats/io/parquet_metadata.py index d8d7cbce..780569b6 100644 --- a/src/hats/io/parquet_metadata.py +++ b/src/hats/io/parquet_metadata.py @@ -258,6 +258,8 @@ def aggregate_column_statistics( """ total_metadata = file_io.read_parquet_metadata(metadata_file) num_row_groups = total_metadata.num_row_groups + if num_row_groups == 0: + return pd.DataFrame() first_row_group = total_metadata.row_group(0) good_column_indexes, column_names = _pick_columns( @@ -391,6 +393,8 @@ def per_pixel_statistics( """ total_metadata = file_io.read_parquet_metadata(metadata_file) num_row_groups = total_metadata.num_row_groups + if num_row_groups == 0: + return pd.DataFrame() first_row_group = total_metadata.row_group(0) good_column_indexes, column_names = _pick_columns( diff --git a/tests/conftest.py b/tests/conftest.py index 7d7ec753..d7dc0fc8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -225,6 +225,11 @@ def small_sky_collection_dir(test_data_dir) -> str: return test_data_dir / "small_sky_o1_collection" +@pytest.fixture +def small_sky_order1_empty_margin_dir(small_sky_collection_dir) -> str: + return small_sky_collection_dir / "small_sky_order1_margin_empty" + + @pytest.fixture def small_sky_source_pixels(): """Source catalog pixels""" diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 62cd46dd..c376868e 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -22,7 +22,12 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-29T21:24:26.515087Z", + "start_time": "2025-10-29T21:24:23.453807Z" + } + }, "outputs": [], "source": [ "import shutil\n", @@ -108,7 +113,12 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-29T21:24:35.273702Z", + "start_time": "2025-10-29T21:24:30.030262Z" + } + }, "outputs": [], "source": [ "remove_directory(\"./small_sky_o1_collection\")\n", @@ -127,6 +137,9 @@ " constant_healpix_order=1,\n", " )\n", " .add_margin(margin_threshold=7200, output_artifact_name=\"small_sky_order1_margin\", is_default=True)\n", + " .add_margin(\n", + " margin_threshold=10, output_artifact_name=\"small_sky_order1_margin_empty\", is_default=False\n", + " )\n", " .add_index(\n", " indexing_column=\"id\",\n", " output_artifact_name=\"small_sky_order1_id_index\",\n", diff --git a/tests/data/small_sky_o1_collection/collection.properties b/tests/data/small_sky_o1_collection/collection.properties index 9c1e7168..60bf09e6 100644 --- a/tests/data/small_sky_o1_collection/collection.properties +++ b/tests/data/small_sky_o1_collection/collection.properties @@ -1,13 +1,13 @@ #HATS Collection obs_collection=small_sky_o1_collection hats_primary_table_url=small_sky_order1 -all_margins=small_sky_order1_margin +all_margins=small_sky_order1_margin small_sky_order1_margin_empty default_margin=small_sky_order1_margin all_indexes=id small_sky_order1_id_index default_index=id -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC -hats_estsize=62 +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC +hats_estsize=64 hats_release_date=2025-08-22 hats_version=v1.0 obs_regime=Optical diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=44.parquet b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=44.parquet index 0f8110c1..a7b4700f 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=44.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=44.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=45.parquet b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=45.parquet index fc1f7cf7..9a9d78ed 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=45.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=45.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=46.parquet b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=46.parquet index 49179996..3d3e7bea 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=46.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=46.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=47.parquet index 6a9e0ba5..4445ecc4 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=47.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=47.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_common_metadata b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_common_metadata index b305553f..24fe03f2 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_common_metadata and b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_common_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_metadata b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_metadata index 804dce2d..3bf35300 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_metadata and b/tests/data/small_sky_o1_collection/small_sky_order1/dataset/_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/hats.properties b/tests/data/small_sky_o1_collection/small_sky_order1/hats.properties index 3061df47..91e9b58e 100644 --- a/tests/data/small_sky_o1_collection/small_sky_order1/hats.properties +++ b/tests/data/small_sky_o1_collection/small_sky_order1/hats.properties @@ -8,8 +8,8 @@ hats_col_healpix=_healpix_29 hats_col_healpix_order=29 hats_npix_suffix=.parquet hats_skymap_order=1 -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC hats_estsize=31 hats_release_date=2025-08-22 hats_version=v1.0 diff --git a/tests/data/small_sky_o1_collection/small_sky_order1/properties b/tests/data/small_sky_o1_collection/small_sky_order1/properties index 3061df47..91e9b58e 100644 --- a/tests/data/small_sky_o1_collection/small_sky_order1/properties +++ b/tests/data/small_sky_o1_collection/small_sky_order1/properties @@ -8,8 +8,8 @@ hats_col_healpix=_healpix_29 hats_col_healpix_order=29 hats_npix_suffix=.parquet hats_skymap_order=1 -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC hats_estsize=31 hats_release_date=2025-08-22 hats_version=v1.0 diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_common_metadata b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_common_metadata index e01c6a11..29afc899 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_common_metadata and b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_common_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_metadata b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_metadata index 6a936ad2..93d6bc99 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_metadata and b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/index/part.0.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/index/part.0.parquet index cb6f4ea1..c632c07b 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/index/part.0.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/index/part.0.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/hats.properties b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/hats.properties index eaa27289..9993ce57 100644 --- a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/hats.properties +++ b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/hats.properties @@ -5,8 +5,8 @@ hats_nrows=131 hats_primary_table_url=small_sky_o1_collection/small_sky_order1 hats_index_column=id hats_npix_suffix=.parquet -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC hats_estsize=3 hats_release_date=2025-08-22 hats_version=v1.0 diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/properties b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/properties index eaa27289..9993ce57 100644 --- a/tests/data/small_sky_o1_collection/small_sky_order1_id_index/properties +++ b/tests/data/small_sky_o1_collection/small_sky_order1_id_index/properties @@ -5,8 +5,8 @@ hats_nrows=131 hats_primary_table_url=small_sky_o1_collection/small_sky_order1 hats_index_column=id hats_npix_suffix=.parquet -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC hats_estsize=3 hats_release_date=2025-08-22 hats_version=v1.0 diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=4.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=4.parquet index 2b1f43d6..afb85e0f 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=4.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=4.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=7.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=7.parquet index f3b1c610..a3a7bdc7 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=7.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=7.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=8.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=8.parquet index c413badd..3d3ccd64 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=8.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=8.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=44.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=44.parquet index 58a1b407..95de014f 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=44.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=44.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=45.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=45.parquet index ee42b5ac..fc693622 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=45.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=45.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=46.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=46.parquet index 3965d9f0..3e2665a6 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=46.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=46.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=47.parquet index 64ea6686..23fad40a 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=47.parquet and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=47.parquet differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_common_metadata b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_common_metadata index b305553f..24fe03f2 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_common_metadata and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_common_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_metadata b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_metadata index eecc9c53..761ad088 100644 Binary files a/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_metadata and b/tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/hats.properties b/tests/data/small_sky_o1_collection/small_sky_order1_margin/hats.properties index 85f7a2a6..2f0bee14 100644 --- a/tests/data/small_sky_o1_collection/small_sky_order1_margin/hats.properties +++ b/tests/data/small_sky_o1_collection/small_sky_order1_margin/hats.properties @@ -11,9 +11,9 @@ hats_margin_threshold=7200.0 hats_npix_suffix=.parquet hats_order=1 moc_sky_fraction=0.33333 -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC -hats_estsize=21 +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC +hats_estsize=20 hats_release_date=2025-08-22 hats_version=v1.0 obs_regime=Optical diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin/properties b/tests/data/small_sky_o1_collection/small_sky_order1_margin/properties index 85f7a2a6..2f0bee14 100644 --- a/tests/data/small_sky_o1_collection/small_sky_order1_margin/properties +++ b/tests/data/small_sky_o1_collection/small_sky_order1_margin/properties @@ -11,9 +11,9 @@ hats_margin_threshold=7200.0 hats_npix_suffix=.parquet hats_order=1 moc_sky_fraction=0.33333 -hats_builder=hats-import v0.6.6, hats v0.6.6 -hats_creation_date=2025-10-06T14:36UTC -hats_estsize=21 +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC +hats_estsize=20 hats_release_date=2025-08-22 hats_version=v1.0 obs_regime=Optical diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_common_metadata b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_common_metadata new file mode 100644 index 00000000..24fe03f2 Binary files /dev/null and b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_common_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_metadata b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_metadata new file mode 100644 index 00000000..24fe03f2 Binary files /dev/null and b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_metadata differ diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/hats.properties b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/hats.properties new file mode 100644 index 00000000..610a3e1d --- /dev/null +++ b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/hats.properties @@ -0,0 +1,20 @@ +#HATS catalog +obs_collection=small_sky_order1_margin_empty +dataproduct_type=margin +hats_nrows=0 +hats_col_ra=ra +hats_col_dec=dec +hats_col_healpix=_healpix_29 +hats_col_healpix_order=29 +hats_primary_table_url=small_sky_o1_collection/small_sky_order1 +hats_margin_threshold=10.0 +hats_npix_suffix=.parquet +hats_order=1 +moc_sky_fraction=0.00000 +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC +hats_estsize=1 +hats_release_date=2025-08-22 +hats_version=v1.0 +obs_regime=Optical +default_index=id diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/partition_info.csv b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/partition_info.csv new file mode 100644 index 00000000..7117bdc8 --- /dev/null +++ b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/partition_info.csv @@ -0,0 +1 @@ +Norder,Npix diff --git a/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/properties b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/properties new file mode 100644 index 00000000..610a3e1d --- /dev/null +++ b/tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/properties @@ -0,0 +1,20 @@ +#HATS catalog +obs_collection=small_sky_order1_margin_empty +dataproduct_type=margin +hats_nrows=0 +hats_col_ra=ra +hats_col_dec=dec +hats_col_healpix=_healpix_29 +hats_col_healpix_order=29 +hats_primary_table_url=small_sky_o1_collection/small_sky_order1 +hats_margin_threshold=10.0 +hats_npix_suffix=.parquet +hats_order=1 +moc_sky_fraction=0.00000 +hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029 +hats_creation_date=2025-10-29T21:24UTC +hats_estsize=1 +hats_release_date=2025-08-22 +hats_version=v1.0 +obs_regime=Optical +default_index=id diff --git a/tests/hats/io/test_parquet_metadata.py b/tests/hats/io/test_parquet_metadata.py index 4cfd2fd4..eb4c6957 100644 --- a/tests/hats/io/test_parquet_metadata.py +++ b/tests/hats/io/test_parquet_metadata.py @@ -284,6 +284,13 @@ def test_aggregate_column_statistics_with_nulls(tmp_path): assert_column_stat_as_floats(result_frame, "Npix", min_value=1, max_value=6, null_count=4, row_count=6) +def test_aggregate_column_statistics_empty_catalog(small_sky_order1_empty_margin_dir): + partition_info_file = paths.get_parquet_metadata_pointer(small_sky_order1_empty_margin_dir) + + result_frame = aggregate_column_statistics(partition_info_file) + assert len(result_frame) == 0 + + def test_per_pixel_statistics(small_sky_order1_dir): partition_info_file = paths.get_parquet_metadata_pointer(small_sky_order1_dir) @@ -432,3 +439,10 @@ def test_per_pixel_statistics_with_rowgroups_empty_result(small_sky_source_dir): partition_info_file, include_pixels=[HealpixPixel(1, 4)], multi_index=True ) assert len(result_frame) == 0 + + +def test_per_pixel_statistics_empty_catalog(small_sky_order1_empty_margin_dir): + partition_info_file = paths.get_parquet_metadata_pointer(small_sky_order1_empty_margin_dir) + + result_frame = per_pixel_statistics(partition_info_file) + assert len(result_frame) == 0