feat: support larger integer for station identifiers, up to 18 or 19 digits or so.

Fixes #17

Author: jmp75

src/efts_io/_ncdf_stf2.py

@@ -106,6 +106,7 @@ def write_nc_stf2(
     data_qual: Optional[xr.DataArray] = None,
     overwrite: bool = True,  # noqa: FBT001, FBT002
     # loc_info: Optional[Dict[str, Any]] = None,
+    intdata_type: str = "i4",
 ) -> None:
     from efts_io.conventions import (  # noqa: I001
         X_VARNAME,
@@ -163,8 +164,6 @@ def _check_optional_var_attr(dataset: xr.Dataset, var_id: str) -> None:
     for var_id in (AREA_VARNAME, X_VARNAME, Y_VARNAME, ELEVATION_VARNAME):
         _check_optional_var_attr(dataset, var_id)
 
-    intdata_type = "i4"
-
     var_type = var_type.value
     data_type = data_type.value
 
@@ -226,6 +225,20 @@ def _check_optional_var_attr(dataset: xr.Dataset, var_id: str) -> None:
     station_var[:] = station
 
     #  station_id
+
+    # we check that station_id can be safely stored as int32
+    # I add this deliberately as a check to avoid possibly silent data corruption as observed in
+    # https://github.com/csiro-hydroinformatics/efts-io/issues/17
+    if intdata_type == "i4":
+        max_station_id = np.max(station_id)
+        min_station_id = np.min(station_id)
+        if not np.issubdtype(type(max_station_id), np.integer) or not np.issubdtype(type(min_station_id), np.integer):
+            raise TypeError("station_id values must be integers to be stored in STF2.0 format.")
+        if max_station_id > np.iinfo(np.int32).max or min_station_id < np.iinfo(np.int32).min:
+            raise OverflowError(
+                f"station_id values must be in the int32 range [{np.iinfo(np.int32).min}, {np.iinfo(np.int32).max}] to be stored in STF2.0 format.",
+            )
+
     station_id_var = ncfile.createVariable(STATION_ID_VARNAME, intdata_type, (STATION_DIMNAME,), fill_value=-9999)
     station_id_var.setncattr(LONG_NAME_ATTR_KEY, "station or node identification code")
     station_id_var[:] = station_id

src/efts_io/wrapper.py

@@ -187,6 +187,8 @@ def __init__(self, data: Union[str, xr.Dataset]) -> None:
         else:
             self.data = data
 
+        self.stf2_int_datatype = "i4"  # default integer type for STF2 saving
+
     @property
     def title(self) -> str:
         """Get or set the title attribute of the dataset."""
@@ -326,6 +328,18 @@ def writeable_to_stf2(self) -> bool:
 
         return exportable_to_stf2(self.data)
 
+    @property
+    def stf2_int_datatype(self) -> str:
+        """The type of integer to save to in the STF 2.x netcdf convention: 'i4' or 'i8'."""
+        return self._stf2_int_datatype
+
+    @stf2_int_datatype.setter
+    def stf2_int_datatype(self, value: str) -> None:
+        """The type of integer to save to in the STF 2.x netcdf convention: 'i4' or 'i8'."""
+        if value not in ("i4", "i8"):
+            raise ValueError("stf2_int_datatype must be either 'i4' or 'i8'")
+        self._stf2_int_datatype = value
+
     def save_to_stf2(
         self,
         path: str,
@@ -359,6 +373,7 @@ def save_to_stf2(
             data_qual=data_qual,  # : Optional[xr.DataArray] = None,
             overwrite=True,  # :bool=True,
             # loc_info=loc_info, # : Optional[Dict[str, Any]] = None,
+            intdata_type=self.stf2_int_datatype,
         )
 
     def create_data_variables(self, data_var_def: Dict[str, Dict[str, Any]]) -> None:

tests/test_create.py

@@ -1,6 +1,7 @@
 # import netCDF4
 import numpy as np
 import pandas as pd
+import pytest
 from efts_io._ncdf_stf2 import StfDataType, StfVariable
 from efts_io.wrapper import EftsDataSet, xr_efts
 
@@ -132,6 +133,18 @@ def _create_test_ds():
 def test_repro_issue_16():
     """Try to repro as closely as possible the issue reported in #16."""
     station_ids = [1, 2, 3]
+    _saving_to_stf2(station_ids)
+
+
+def test_large_station_integers():
+    """Try to repro as closely as possible the issue reported in #17."""
+    station_ids = [1, 2, 123456789123]
+    with pytest.raises(OverflowError):
+        _saving_to_stf2(station_ids, intdata_type="i4")
+    _saving_to_stf2(station_ids, intdata_type="i8")
+
+
+def _saving_to_stf2(station_ids, intdata_type="i4"):
     xr_ds = xr_efts(
         issue_times=pd.date_range("2023-10-01", periods=31, freq="D"),
         station_ids=station_ids,
@@ -155,6 +168,7 @@ def test_repro_issue_16():
         longitudes=[10.0, 11.0, 12.0],
     )
     eds = EftsDataSet(xr_ds)
+    eds.stf2_int_datatype = intdata_type
     eds.create_data_variables(
         {
             "rain_obs": {