Merge pull request #71 from hubverse-org/mc/hub-data

refactor to use the hubdata package for hub access

Author: matthewcornell

demo/index.html

@@ -25,7 +25,7 @@
 </div>
 
 <script type="module">
-    import App from 'https://cdn.jsdelivr.net/gh/reichlab/predtimechart@3.0.0/dist/predtimechart.bundle.js';
+    import App from 'https://cdn.jsdelivr.net/gh/reichlab/predtimechart@3.1.1/dist/predtimechart.bundle.js';
     document.predtimechart = App;  // for debugging
 
     function replace_chars(the_string) {

pyproject.toml

@@ -17,7 +17,8 @@ dependencies = [
     "pyyaml",
     "structlog",
     "pyarrow",
-    "jsonschema"
+    "jsonschema",
+    "hubdata>=0.1.3",
 ]
 
 [project.optional-dependencies]

requirements/requirements.txt

@@ -10,20 +10,25 @@ attrs==24.2.0
     #   referencing
 build==1.2.1
     # via pip-tools
-click==8.1.7
+click==8.2.1
     # via
     #   hub-dashboard-predtimechart (pyproject.toml)
+    #   hubdata
     #   pip-tools
+hubdata==0.1.3
+    # via hub-dashboard-predtimechart (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
 jsonschema==4.23.0
     # via hub-dashboard-predtimechart (pyproject.toml)
 jsonschema-specifications==2023.12.1
     # via jsonschema
+markdown-it-py==3.0.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
 numpy==2.1.0
-    # via
-    #   pandas
-    #   pyarrow
+    # via pandas
 packaging==24.1
     # via
     #   build
@@ -36,8 +41,12 @@ pluggy==1.5.0
     # via pytest
 polars==1.11.0
     # via hub-dashboard-predtimechart (pyproject.toml)
-pyarrow==17.0.0
-    # via hub-dashboard-predtimechart (pyproject.toml)
+pyarrow==21.0.0
+    # via
+    #   hub-dashboard-predtimechart (pyproject.toml)
+    #   hubdata
+pygments==2.19.2
+    # via rich
 pyproject-hooks==1.1.0
     # via
     #   build
@@ -54,14 +63,18 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
+rich==14.1.0
+    # via hubdata
 rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
 six==1.16.0
     # via python-dateutil
 structlog==24.4.0
-    # via hub-dashboard-predtimechart (pyproject.toml)
+    # via
+    #   hub-dashboard-predtimechart (pyproject.toml)
+    #   hubdata
 tzdata==2024.1
     # via pandas
 wheel==0.44.0

src/hub_predtimechart/__init__.py

@@ -1 +1 @@
-__version__ = "2.2.0"
+__version__ = "2.2.1"

src/hub_predtimechart/app/generate_target_json_files.py

@@ -133,11 +133,11 @@ def ptc_target_data(model_task: ModelTask, target_data_df: pl.DataFrame, task_id
         else:
             target_data_df = target_data_df.filter(pl.col('as_of') == max_as_of.isoformat())
     else:
-      # the file is one that is assumed to be updated weekly and so we can
-      # assume that the effective as_of date for this file is the same as
-      # max_available_ref_date (the newest date)
-      if max_available_ref_date is not None and reference_date != max_available_ref_date:
-          return None
+        # the file is one that is assumed to be updated weekly and so we can
+        # assume that the effective as_of date for this file is the same as
+        # max_available_ref_date (the newest date)
+        if max_available_ref_date is not None and reference_date != max_available_ref_date:
+            return None
 
     # until all hubs implement our new time-series target data standard, we condition on
     # hub_config.target_data_file_name, which acts as a flag indicating whether the hub implements the new standard or

src/hub_predtimechart/hub_config.py

@@ -7,17 +7,25 @@
 import pandas as pd
 import polars as pl
 import yaml
-from jsonschema import ValidationError, validate, FormatChecker
+from hubdata import HubConnection
+from jsonschema import FormatChecker, ValidationError, validate
 
-from hub_predtimechart.hub_config import HubConfig
 from hub_predtimechart.ptc_schema import ptc_config_schema
 
 
-class HubConfigPtc(HubConfig):
+class HubConfigPtc(HubConnection):
     """
-    A HubConfig subclass that adds various visualization-related variables from a hub.
+    A `hubdata.HubConnection` subclass that adds various visualization-related variables from a hub. Note that this
+    class only works with local filesystems, and therefore only accepts a Path for `hub_path`.
 
     Instance variables:
+
+    Via HubConnection:
+    - hub_path: str or Path pointing to a hub's root directory as passed to `hubdata.connect_hub()`
+    - tasks: the hub's `tasks.json` contents as a dict
+    - model_metadata_schema: "" `model-metadata-schema.json` ""
+
+    Via this class:
     - rounds_idx: as loaded from `ptc_config_file`
     - reference_date_col_name: ""
     - horizon_col_name: ""
@@ -36,13 +44,16 @@ class HubConfigPtc(HubConfig):
     """
 
 
-    def __init__(self, hub_dir: Path, ptc_config_file: Path):
+    def __init__(self, hub_path: Path, ptc_config_file: Path):
         """
-        :param hub_dir: as defined in HubConfig.__init__()
+        :param hub_path: Path pointing to a hub's root directory as passed to `hubdata.connect_hub()`
         :param ptc_config_file: location of `predtimechart-config.yml` (or other named) file that matches ptc_schema.py.
-            this file specifies how to process `hub_dir` to get predtimechart output
+            this file specifies how to process `hub_path` to get predtimechart output
         """
-        super().__init__(hub_dir)
+        if not isinstance(hub_path, Path):
+            raise TypeError(f"hub_path was not a Path. hub_path={hub_path!r}, type={type(hub_path).__name__}")
+
+        super().__init__(hub_path)
 
         if not ptc_config_file.exists():
             raise RuntimeError(f"predtimechart config file not found: {ptc_config_file}")
@@ -67,8 +78,8 @@ def __init__(self, hub_dir: Path, ptc_config_file: Path):
 
         # set model_id_to_metadata
         self.model_id_to_metadata: dict[str, dict] = {}
-        for model_metadata_file in (list((self.hub_dir / 'model-metadata').glob('*.yml')) +
-                                    list((self.hub_dir / 'model-metadata').glob('*.yaml'))):
+        for model_metadata_file in (list((self.hub_path / 'model-metadata').glob('*.yml')) +
+                                    list((self.hub_path / 'model-metadata').glob('*.yaml'))):
             with open(model_metadata_file) as fp:
                 model_metadata = yaml.safe_load(fp)
                 model_id = f"{model_metadata['team_abbr']}-{model_metadata['model_abbr']}"
@@ -90,8 +101,8 @@ def model_output_file_for_ref_date(self, model_id: str, reference_date: str) ->
         Returns a Path to the model output file corresponding to `model_id` and `reference_date`. Returns None if none
         found.
         """
-        poss_output_files = [self.hub_dir / 'model-output' / model_id / f"{reference_date}-{model_id}.csv",
-                             self.hub_dir / 'model-output' / model_id / f"{reference_date}-{model_id}.parquet"]
+        poss_output_files = [self.hub_path / 'model-output' / model_id / f"{reference_date}-{model_id}.csv",
+                             self.hub_path / 'model-output' / model_id / f"{reference_date}-{model_id}.parquet"]
         for poss_output_file in poss_output_files:
             if poss_output_file.exists():
                 return poss_output_file
@@ -104,7 +115,7 @@ def get_target_data_df(self) -> pl.DataFrame:
         Loads the target data csv file from the hub repo for now, file path for target data is hard coded to 'target-data'.
         Raises FileNotFoundError if target data file does not exist.
         """
-        target_data_file_path = self.hub_dir / 'target-data' / self.get_target_data_file_name()
+        target_data_file_path = self.hub_path / 'target-data' / self.get_target_data_file_name()
         try:
             # the override schema handles the 'US' location (the only location that doesn't parse as Int64)
             # todo hard-coded column names
@@ -165,8 +176,10 @@ def _validate_hub_ptc_compatibility(hub_config_ptc: HubConfigPtc):
     if not hub_config_ptc.model_tasks:
         raise ValidationError(f"no applicable model_task entries were found")
 
-    # validate: model metadata must contain a boolean `designated_model` field
-    if 'designated_model' not in hub_config_ptc.model_metadata_schema['required']:
+    # validate: model metadata must be present and must contain a boolean `designated_model` field
+    if hub_config_ptc.model_metadata_schema is None:
+        raise ValidationError(f"model metadata schema not found")
+    elif 'designated_model' not in hub_config_ptc.model_metadata_schema['required']:
         raise ValidationError(f"'designated_model' not found in model metadata schema's 'required' section")
 
     # validate: all model_task entries have the same task_ids. frozenset lets us make a set of sets