Merge pull request #29 from Aquaveo/fix-herbie-download-2

Improve herbie downloads memory management

Author: swainn

loone_data_prep/forecast_scripts/create_forecast_LOWs.py

@@ -138,6 +138,10 @@ def _download_herbie_variable(fast_herbie_object: FastHerbie, variable_key: str,
     # Extract point data
     dsi = ds.herbie.pick_points(point_df, method="nearest")
 
+    # Close and delete the original dataset to free up resources
+    ds.close()
+    del ds
+
     # Get actual variable name
     if variable_name == "10u":
         var_name_actual = "u10"  # Map 10u to u10
@@ -156,6 +160,10 @@ def _download_herbie_variable(fast_herbie_object: FastHerbie, variable_key: str,
     elif "step" in df.columns and "time" in dsi.coords:
         df["datetime"] = dsi.time.values[0] + df["step"]
 
+    # Close and delete the intermediate dataset to free memory
+    dsi.close()
+    del dsi, time_series
+
     # Retain necessary columns
     df = df[["datetime", var_name_actual]].drop_duplicates()
 

loone_data_prep/forecast_scripts/weather_forecast.py

@@ -146,6 +146,10 @@ def _download_herbie_variable(fast_herbie_object: FastHerbie, variable_key: str,
 
     # Extract point data
     dsi = ds.herbie.pick_points(points, method="nearest")
+    
+    # Close and delete the original dataset to free up resources
+    ds.close()
+    del ds
 
     # Extract the correct variable name dynamically
     if variable_name == "10u":
@@ -173,6 +177,9 @@ def _download_herbie_variable(fast_herbie_object: FastHerbie, variable_key: str,
     # Print extracted data
     # print(df)
 
+    # Clean up intermediate datasets to free memory
+    del dsi, time_series
+    
     return df