Skip to content

Support empty parquet metadata files #580

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Support empty parquet metadata files #580

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/hats/io/parquet_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,8 @@ def aggregate_column_statistics(
"""
total_metadata = file_io.read_parquet_metadata(metadata_file)
num_row_groups = total_metadata.num_row_groups
if num_row_groups == 0:
return pd.DataFrame()
first_row_group = total_metadata.row_group(0)

good_column_indexes, column_names = _pick_columns(
Expand Down Expand Up @@ -391,6 +393,8 @@ def per_pixel_statistics(
"""
total_metadata = file_io.read_parquet_metadata(metadata_file)
num_row_groups = total_metadata.num_row_groups
if num_row_groups == 0:
return pd.DataFrame()
first_row_group = total_metadata.row_group(0)

good_column_indexes, column_names = _pick_columns(
Expand Down
5 changes: 5 additions & 0 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,11 @@ def small_sky_collection_dir(test_data_dir) -> str:
return test_data_dir / "small_sky_o1_collection"


@pytest.fixture
def small_sky_order1_empty_margin_dir(small_sky_collection_dir) -> str:
return small_sky_collection_dir / "small_sky_order1_margin_empty"


@pytest.fixture
def small_sky_source_pixels():
"""Source catalog pixels"""
Expand Down
17 changes: 15 additions & 2 deletions tests/data/generate_data.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"ExecuteTime": {
"end_time": "2025-10-29T21:24:26.515087Z",
"start_time": "2025-10-29T21:24:23.453807Z"
}
},
"outputs": [],
"source": [
"import shutil\n",
Expand Down Expand Up @@ -108,7 +113,12 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"ExecuteTime": {
"end_time": "2025-10-29T21:24:35.273702Z",
"start_time": "2025-10-29T21:24:30.030262Z"
}
},
"outputs": [],
"source": [
"remove_directory(\"./small_sky_o1_collection\")\n",
Expand All @@ -127,6 +137,9 @@
" constant_healpix_order=1,\n",
" )\n",
" .add_margin(margin_threshold=7200, output_artifact_name=\"small_sky_order1_margin\", is_default=True)\n",
" .add_margin(\n",
" margin_threshold=10, output_artifact_name=\"small_sky_order1_margin_empty\", is_default=False\n",
" )\n",
" .add_index(\n",
" indexing_column=\"id\",\n",
" output_artifact_name=\"small_sky_order1_id_index\",\n",
Expand Down
8 changes: 4 additions & 4 deletions tests/data/small_sky_o1_collection/collection.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#HATS Collection
obs_collection=small_sky_o1_collection
hats_primary_table_url=small_sky_order1
all_margins=small_sky_order1_margin
all_margins=small_sky_order1_margin small_sky_order1_margin_empty
default_margin=small_sky_order1_margin
all_indexes=id small_sky_order1_id_index
default_index=id
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_estsize=62
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=64
hats_release_date=2025-08-22
hats_version=v1.0
obs_regime=Optical
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=44.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=45.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=46.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1/dataset/Norder=1/Dir=0/Npix=47.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1/dataset/_common_metadata
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1/dataset/_metadata
View file
Open in desktop
Binary file not shown.
4 changes: 2 additions & 2 deletions tests/data/small_sky_o1_collection/small_sky_order1/hats.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ hats_col_healpix=_healpix_29
hats_col_healpix_order=29
hats_npix_suffix=.parquet
hats_skymap_order=1
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=31
hats_release_date=2025-08-22
hats_version=v1.0
Expand Down
4 changes: 2 additions & 2 deletions tests/data/small_sky_o1_collection/small_sky_order1/properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ hats_col_healpix=_healpix_29
hats_col_healpix_order=29
hats_npix_suffix=.parquet
hats_skymap_order=1
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=31
hats_release_date=2025-08-22
hats_version=v1.0
Expand Down
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_common_metadata
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/_metadata
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1_id_index/dataset/index/part.0.parquet
View file
Open in desktop
Binary file not shown.
4 changes: 2 additions & 2 deletions tests/data/small_sky_o1_collection/small_sky_order1_id_index/hats.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ hats_nrows=131
hats_primary_table_url=small_sky_o1_collection/small_sky_order1
hats_index_column=id
hats_npix_suffix=.parquet
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=3
hats_release_date=2025-08-22
hats_version=v1.0
Expand Down
4 changes: 2 additions & 2 deletions tests/data/small_sky_o1_collection/small_sky_order1_id_index/properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ hats_nrows=131
hats_primary_table_url=small_sky_o1_collection/small_sky_order1
hats_index_column=id
hats_npix_suffix=.parquet
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=3
hats_release_date=2025-08-22
hats_version=v1.0
Expand Down
Binary file modified ...ata/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=4.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified ...ata/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=7.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified ...ata/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=8.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified ...ta/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=44.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified ...ta/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=45.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified ...ta/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=46.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified ...ta/small_sky_o1_collection/small_sky_order1_margin/dataset/Norder=1/Dir=0/Npix=47.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_common_metadata
View file
Open in desktop
Binary file not shown.
Binary file modified tests/data/small_sky_o1_collection/small_sky_order1_margin/dataset/_metadata
View file
Open in desktop
Binary file not shown.
6 changes: 3 additions & 3 deletions tests/data/small_sky_o1_collection/small_sky_order1_margin/hats.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ hats_margin_threshold=7200.0
hats_npix_suffix=.parquet
hats_order=1
moc_sky_fraction=0.33333
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_estsize=21
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=20
hats_release_date=2025-08-22
hats_version=v1.0
obs_regime=Optical
Expand Down
6 changes: 3 additions & 3 deletions tests/data/small_sky_o1_collection/small_sky_order1_margin/properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ hats_margin_threshold=7200.0
hats_npix_suffix=.parquet
hats_order=1
moc_sky_fraction=0.33333
hats_builder=hats-import v0.6.6, hats v0.6.6
hats_creation_date=2025-10-06T14:36UTC
hats_estsize=21
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=20
hats_release_date=2025-08-22
hats_version=v1.0
obs_regime=Optical
Expand Down
Binary file added tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_common_metadata
View file
Open in desktop
Binary file not shown.
Binary file added tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/dataset/_metadata
View file
Open in desktop
Binary file not shown.
20 changes: 20 additions & 0 deletions tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/hats.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#HATS catalog
obs_collection=small_sky_order1_margin_empty
dataproduct_type=margin
hats_nrows=0
hats_col_ra=ra
hats_col_dec=dec
hats_col_healpix=_healpix_29
hats_col_healpix_order=29
hats_primary_table_url=small_sky_o1_collection/small_sky_order1
hats_margin_threshold=10.0
hats_npix_suffix=.parquet
hats_order=1
moc_sky_fraction=0.00000
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=1
hats_release_date=2025-08-22
hats_version=v1.0
obs_regime=Optical
default_index=id
1 change: 1 addition & 0 deletions tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/partition_info.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Norder,Npix
20 changes: 20 additions & 0 deletions tests/data/small_sky_o1_collection/small_sky_order1_margin_empty/properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#HATS catalog
obs_collection=small_sky_order1_margin_empty
dataproduct_type=margin
hats_nrows=0
hats_col_ra=ra
hats_col_dec=dec
hats_col_healpix=_healpix_29
hats_col_healpix_order=29
hats_primary_table_url=small_sky_o1_collection/small_sky_order1
hats_margin_threshold=10.0
hats_npix_suffix=.parquet
hats_order=1
moc_sky_fraction=0.00000
hats_builder=hats-import v0.6.8.dev6+g9754d7879.d20251029, hats v0.6.8.dev15+gea9a91e6a.d20251029
hats_creation_date=2025-10-29T21:24UTC
hats_estsize=1
hats_release_date=2025-08-22
hats_version=v1.0
obs_regime=Optical
default_index=id
14 changes: 14 additions & 0 deletions tests/hats/io/test_parquet_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,13 @@ def test_aggregate_column_statistics_with_nulls(tmp_path):
assert_column_stat_as_floats(result_frame, "Npix", min_value=1, max_value=6, null_count=4, row_count=6)


def test_aggregate_column_statistics_empty_catalog(small_sky_order1_empty_margin_dir):
partition_info_file = paths.get_parquet_metadata_pointer(small_sky_order1_empty_margin_dir)

result_frame = aggregate_column_statistics(partition_info_file)
assert len(result_frame) == 0


def test_per_pixel_statistics(small_sky_order1_dir):
partition_info_file = paths.get_parquet_metadata_pointer(small_sky_order1_dir)

Expand Down Expand Up @@ -432,3 +439,10 @@ def test_per_pixel_statistics_with_rowgroups_empty_result(small_sky_source_dir):
partition_info_file, include_pixels=[HealpixPixel(1, 4)], multi_index=True
)
assert len(result_frame) == 0


def test_per_pixel_statistics_empty_catalog(small_sky_order1_empty_margin_dir):
partition_info_file = paths.get_parquet_metadata_pointer(small_sky_order1_empty_margin_dir)

result_frame = per_pixel_statistics(partition_info_file)
assert len(result_frame) == 0
Loading