Merge pull request #63 from ecmwf/feat/add_filters

add flexible filtering

Author: Oisin-M

hat/extract_timeseries/extractor.py

@@ -52,6 +52,9 @@ def create_mask_from_coords(coords_config, df, gridx, gridy, shape):
 
 def process_inputs(station_config, grid_config):
     df = pd.read_csv(station_config["file"])
+    filters = station_config.get("filter")
+    if filters is not None:
+        df = df.query(filters)
     station_names = df[station_config["name"]].values
 
     index_config = station_config.get("index", None)

hat/station_mapping/mapper.py

@@ -27,6 +27,9 @@ def get_grid_inputs(grid_config):
 
 def get_station_inputs(station_config):
     df = pd.read_csv(station_config["file"])
+    filters = station_config.get("filter")
+    if filters is not None:
+        df = df.query(filters)
     coord_x = station_config["coords"]["x"]
     coord_y = station_config["coords"]["y"]
     station_coords1 = df[coord_x].values

notebooks/workflow/station_mapping.ipynb

@@ -20,6 +20,7 @@
     "config = {\n",
     "    \"station\": {\n",
     "        \"file\": \"stations.csv\",\n",
+    "        \"filter\": \"(StationLon >= 0) and (drainage_area_provided >= 0)\",\n",
     "        \"coords\": {\n",
     "            \"x\": \"StationLat\",\n",
     "            \"y\": \"StationLon\"\n",
@@ -63,7 +64,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "hat",
    "language": "python",
    "name": "python3"
   },

notebooks/workflow/timeseries_extraction.ipynb

@@ -20,6 +20,7 @@
     "config = {\n",
     "    \"station\": {\n",
     "        \"file\": \"mapped_stations.csv\",\n",
+    "        \"filter\": \"(StationLon >= 0) and (drainage_area_provided >= 0)\",\n",
     "        # EITHER USE INDEX OR COORDS, NOT BOTH\n",
     "        # INDEX is preferred but requires you to have the exact same grid\n",
     "        # as the upstream area used in station mapping\n",