Merge pull request #14 from OpenGeoscience/lfs

annehaley · web-flow · commit 6b23c5b610a8 · 2025-12-15T12:51:40.000-05:00
Store files in Git LFS
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+# Track all files in uvdat_flood_sim/data/ with Git LFS
+uvdat_flood_sim/data/** filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@ __pycache__
 downloads
 outputs
 venv
+dist/
diff --git a/README.md b/README.md
@@ -26,6 +26,26 @@ This is a proof of concept for the Charles River in Boston that could be transla
 
 6. Ground Water Percentile: Select the 25th, 50th, 75th, or 90th percentile value for ground water
 
+## Prerequisites
+
+This project uses [Git LFS](https://git-lfs.github.com/) to store large data files. Before cloning the repository, install Git LFS:
+
+```bash
+# On Ubuntu/Debian
+sudo apt-get install git-lfs
+
+# On macOS with Homebrew
+brew install git-lfs
+
+# On Windows, download from https://git-lfs.github.com/
+```
+
+After installation, initialize Git LFS:
+
+```bash
+git lfs install
+```
+
 ## Example usage
 
 Install requirements with `pip install -e .`.
diff --git a/uvdat_flood_sim/constants.py b/uvdat_flood_sim/constants.py
@@ -1,33 +1,19 @@
 from datetime import timedelta
+import importlib.resources
 from pathlib import Path
 
 
-DOWNLOADS_FOLDER = Path('downloads')
+DATA_FOLDER = importlib.resources.files('uvdat_flood_sim.data')
 
-DOWNSCALING_MODEL_URL = 'https://data.kitware.com/api/v1/item/68c463cb7d52b0d5b570f348/download'
-DOWNSCALING_MODEL_PATH = DOWNLOADS_FOLDER / 'downscaling_model.pkl'
-
-HYDROLOGICAL_MODEL_URL = 'https://data.kitware.com/api/v1/item/68c463bd7d52b0d5b570f345/download'
-HYDROLOGICAL_MODEL_PATH = DOWNLOADS_FOLDER / 'hydrological_model.npy'
-
-RATING_CURVE_URL = 'https://data.kitware.com/api/v1/item/68c463bd7d52b0d5b570f342/download'
-RATING_CURVE_PATH = DOWNLOADS_FOLDER / 'rating_curve.csv'
-
-HAND_URL = 'https://data.kitware.com/api/v1/item/68e59bc1e5c7892678d7f036/download'
-HAND_PATH = DOWNLOADS_FOLDER / 'boston_hand.tif'
-
-PERCENTILES_URL = 'https://data.kitware.com/api/v1/item/68e7be67e5c7892678d7f03c/download'
-PERCENTILES_PATH = DOWNLOADS_FOLDER / 'flood_param_percentiles.json'
+DOWNSCALING_MODEL_PATH = DATA_FOLDER / 'downscaling_model.pkl'
+HYDROLOGICAL_MODEL_PATH = DATA_FOLDER / 'hydrological_model.npy'
+RATING_CURVE_PATH = DATA_FOLDER / 'rating_curve.csv'
+HAND_PATH = DATA_FOLDER / 'boston_hand.tif'
+PERCENTILES_PATH = DATA_FOLDER / 'flood_param_percentiles.json'
 
 CESM_DATA = {
-    '2031-2050': dict(
-        filename=DOWNLOADS_FOLDER / 'boston_cesm2-001_2031_2050.npy',
-        url='https://data.kitware.com/api/v1/item/68c463cb7d52b0d5b570f34b/download',
-    ),
-    '2041-2060': dict(
-        filename=DOWNLOADS_FOLDER / 'boston_cesm2-001_2041_2060.npy',
-        url='https://data.kitware.com/api/v1/file/6900c8808c59593f4ceaecf2/download',
-    )
+    '2031-2050': DATA_FOLDER / 'boston_cesm2-001_2031_2050.npy',
+    '2041-2060': DATA_FOLDER / 'boston_cesm2-001_2041_2060.npy',
 }
 
 HYDROGRAPHS = dict(
diff --git a/uvdat_flood_sim/data/boston_cesm2-001_2031_2050.npy b/uvdat_flood_sim/data/boston_cesm2-001_2031_2050.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f85f6f6e70d87ce0274f71cb1c320f27cf30be868a07b384520bae547b5d96c4
+size 700928
diff --git a/uvdat_flood_sim/data/boston_cesm2-001_2041_2060.npy b/uvdat_flood_sim/data/boston_cesm2-001_2041_2060.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc7a093764d8404d35b2c1a8df98808b511b5bc8a91d88946c9e8f12520cb0e5
+size 700928
diff --git a/uvdat_flood_sim/data/boston_hand.tif b/uvdat_flood_sim/data/boston_hand.tif
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e51f5bf2eaef40f79cee3eff7a5de8dec1113f887aec2cb31246289c75eca75
+size 9160882
diff --git a/uvdat_flood_sim/data/downscaling_model.pkl b/uvdat_flood_sim/data/downscaling_model.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3d4d2dad103b9e0795d33d3dac93452a5d8fccb8683ca4b48e0496ba20f327b
+size 75313340
diff --git a/uvdat_flood_sim/data/flood_param_percentiles.json b/uvdat_flood_sim/data/flood_param_percentiles.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d0ec87f43cb4a49a25c1b68a895bfcf970b6e7eda4e40df715ef59c6dbee453
+size 7910
diff --git a/uvdat_flood_sim/data/hydrological_model.npy b/uvdat_flood_sim/data/hydrological_model.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f7c8c0c9594aebe13f35e1a438cf15f381328cfb83b00b8dfde8336926021c
+size 184
diff --git a/uvdat_flood_sim/data/rating_curve.csv b/uvdat_flood_sim/data/rating_curve.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64bd2c2046b9c89f1d0540dc8d5227ae2d9611483a07fc6e518969a0afce1e14
+size 14884
diff --git a/uvdat_flood_sim/downscaling_prediction.py b/uvdat_flood_sim/downscaling_prediction.py
@@ -1,12 +1,12 @@
 # https://github.com/augustposch/IDEEA_2025mar/blob/main/notebooks/machine_learning.ipynb
 
+import math
 import pickle
+
 import numpy
-import math
 from scipy.stats import genextreme as gev
 
-from .utils import download_file
-from .constants import CESM_DATA, DOWNSCALING_MODEL_URL, DOWNSCALING_MODEL_PATH
+from .constants import CESM_DATA, DOWNSCALING_MODEL_PATH
 
 
 def annual_precipitation_maxima(daily):
@@ -17,15 +17,9 @@ def annual_precipitation_maxima(daily):
 
 
 def downscale_boston_cesm(cesm_id, annual_probability):
-    cesm = CESM_DATA[cesm_id] # CESM data is a projection of 20 years of daily weather conditions, spatially coarse over a large region
-    for url, path in [
-        (DOWNSCALING_MODEL_URL, DOWNSCALING_MODEL_PATH),
-        (cesm.get('url'), cesm.get('filename')),
-    ]:
-        download_file(url, path)
-
-    cesm_data = numpy.load(cesm.get('filename'), allow_pickle=True)
-    with open(DOWNSCALING_MODEL_PATH, 'rb') as m:
+    cesm_file = CESM_DATA[cesm_id] # CESM data is a projection of 20 years of daily weather conditions, spatially coarse over a large region
+    cesm_data = numpy.load(cesm_file, allow_pickle=True)
+    with DOWNSCALING_MODEL_PATH.open('rb') as m:
         model = pickle.load(m)
     prediction = model.predict(cesm_data) # Model predicts local watershed precipitation based on each day's weather
     apm = annual_precipitation_maxima(prediction) # Obtain annual precipitation maxima (APM) for each of the 20 years
diff --git a/uvdat_flood_sim/hydrodynamic_prediction.py b/uvdat_flood_sim/hydrodynamic_prediction.py
@@ -1,16 +1,10 @@
 # https://github.com/augustposch/hand_hydrodynamic_2025aug/blob/main/notebooks/hydrodynamic_hand.ipynb
 
-import rasterio
 import numpy
 import pandas
+import rasterio
 
-from .utils import download_file
-from .constants import (
-    RATING_CURVE_URL,
-    RATING_CURVE_PATH,
-    HAND_URL,
-    HAND_PATH,
-)
+from .constants import RATING_CURVE_PATH, HAND_PATH
 
 
 def discharge_to_stage_height(discharge, rating_curve):
@@ -60,10 +54,8 @@ def create_hydrograph(q, unitless_hydrograph):
 
 
 def generate_flood_from_discharge(q, unitless_hydrograph): # q needs to be total flood volume, e.g. cubic feet per day for a 1-day flood
-    download_file(RATING_CURVE_URL, RATING_CURVE_PATH)
     rating_curve = pandas.read_csv(RATING_CURVE_PATH, sep=None, engine='python', skiprows=[1])
 
-    download_file(HAND_URL, HAND_PATH)
     with rasterio.open(HAND_PATH) as f:
         hand_array = f.read(1)
 
diff --git a/uvdat_flood_sim/hydrological_prediction.py b/uvdat_flood_sim/hydrological_prediction.py
@@ -2,8 +2,12 @@
 
 import numpy
 
-from .utils import download_file
-from .constants import HYDROLOGICAL_MODEL_URL, HYDROLOGICAL_MODEL_PATH, WATERSHED_AREA_SQ_M, CUBIC_METERS_TO_CUBIC_FEET, SECONDS_PER_DAY
+from .constants import (
+    HYDROLOGICAL_MODEL_PATH,
+    WATERSHED_AREA_SQ_M,
+    CUBIC_METERS_TO_CUBIC_FEET,
+    SECONDS_PER_DAY,
+)
 
 
 def calculate_discharge_from_precipitation(
@@ -12,7 +16,6 @@ def calculate_discharge_from_precipitation(
     soil_moisture: float, # Soil moisture in physical units (never seen by user)
     ground_water: float, # Ground water in physical units (never seen by user)
 ):
-    download_file(HYDROLOGICAL_MODEL_URL, HYDROLOGICAL_MODEL_PATH)
     model = numpy.load(HYDROLOGICAL_MODEL_PATH)
     (insc, coeff, sq, smsc, sub, crak, k) = model
 
diff --git a/uvdat_flood_sim/run.py b/uvdat_flood_sim/run.py
@@ -5,11 +5,10 @@
 import numpy
 from numpy.typing import NDArray
 
-from .constants import PERCENTILES_URL, PERCENTILES_PATH, HYDROGRAPHS, SECONDS_PER_DAY
+from .constants import PERCENTILES_PATH, HYDROGRAPHS, SECONDS_PER_DAY
 from .downscaling_prediction import downscale_boston_cesm
 from .hydrological_prediction import calculate_discharge_from_precipitation
 from .hydrodynamic_prediction import generate_flood_from_discharge
-from .utils import download_file
 
 logger = logging.getLogger('uvdat_flood_sim')
 
@@ -36,8 +35,7 @@ def run_sim(
     :param gw_percentile: Ground water percentile.
     :return: Array with 2 spatial dimensions and 1 time dimension.
     """
-    download_file(PERCENTILES_URL, PERCENTILES_PATH)
-    with open(PERCENTILES_PATH) as f:
+    with PERCENTILES_PATH.open('r') as f:
         percentiles = json.load(f)
 
     if hydrograph is None and hydrograph_name is None:
diff --git a/uvdat_flood_sim/utils.py b/uvdat_flood_sim/utils.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# Track all files in uvdat_flood_sim/data/ with Git LFS`
	`2`	`+uvdat_flood_sim/data/** filter=lfs diff=lfs merge=lfs -text`
-Original file line number
+Diff line change
 downloads
 outputs
 venv
 +dist/
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:f85f6f6e70d87ce0274f71cb1c320f27cf30be868a07b384520bae547b5d96c4`
	`3`	`+size 700928`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:bc7a093764d8404d35b2c1a8df98808b511b5bc8a91d88946c9e8f12520cb0e5`
	`3`	`+size 700928`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:8e51f5bf2eaef40f79cee3eff7a5de8dec1113f887aec2cb31246289c75eca75`
	`3`	`+size 9160882`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:f3d4d2dad103b9e0795d33d3dac93452a5d8fccb8683ca4b48e0496ba20f327b`
	`3`	`+size 75313340`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:7d0ec87f43cb4a49a25c1b68a895bfcf970b6e7eda4e40df715ef59c6dbee453`
	`3`	`+size 7910`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:89f7c8c0c9594aebe13f35e1a438cf15f381328cfb83b00b8dfde8336926021c`
	`3`	`+size 184`