corrected processing for new bahamas data which is now one file

Author: JakobDeutloff

.pre-commit-config.yaml

@@ -31,11 +31,6 @@ repos:
         description: python linter
     -   id: ruff-format
         description: python formatter
--   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.3.0
-    hooks:
-    -   id: black-jupyter
-        description: formating ipynb notebook
 -   repo: https://github.com/compilerla/conventional-pre-commit
     rev: v3.4.0
     hooks:

process_config.yaml

@@ -1,8 +1,8 @@
 date: '20240827'
 flightletter: a
 flightname: HALO-{date}{flightletter}
-bahamas: ipns://latest.orcestra-campaign.org/products/HALO/position_attitude/HALO-{date}{flightletter}.zarr
+bahamas: ipns://latest.orcestra-campaign.org/products/HALO/position_attitude.zarr
 radar: ipns://latest.orcestra-campaign.org/raw/HALO/radar/HALO-{date}{flightletter}/mom/*.nc
 radiometer: ipns://latest.orcestra-campaign.org/raw/HALO/radiometer/HALO-{date}{flightletter}
 sea_land_mask: /work/bm1183/m301049/orcestra/sea_land_mask.nc
-save_dir: /work/bm1183/m301049/orcestra/Hamp_Processed
+save_dir: /work/bm1183/m301049/orcestra/Hamp_Processed_vn2

scripts/find_amplifier_faults.py

@@ -3,6 +3,9 @@
 import yaml
 from src.ipfs_helpers import read_nc
 from src.plots_functions import plot_radiometers
+import matplotlib as mpl
+
+mpl.use("WebAgg")
 
 # %% load processed data
 with open("process_config.yaml", "r") as file:

scripts/process_hamp.py

@@ -25,8 +25,6 @@
 
 
 # %% define function
-
-
 def postprocess_hamp(date, flightletter, version):
     """
     Postprocess raw data from HAMP.
@@ -67,14 +65,16 @@ def postprocess_hamp(date, flightletter, version):
     # load raw data
     print(f"Loading raw data for {date}")
     ds_radar_raw = read_mf_nc(paths["radar"]).load()
-    ds_bahamas = (
-        xr.open_dataset(paths["bahamas"], engine="zarr")
-        .reset_coords(["lat", "lon", "alt"])
-        .resample(time="0.25s")
-        .mean()
-    )
+
     if date == "20240929":  # only flight that crossed 0 UTC
         date_2 = "20240930"
+        ds_bahamas = (
+            xr.open_dataset(paths["bahamas"], engine="zarr")
+            .sel(time=slice(date, date_2))
+            .reset_coords(["lat", "lon", "alt"])
+            .resample(time="0.25s")
+            .mean()
+        )
         ds_iwv_raw_29 = read_nc(f"{paths['radiometer']}/KV/{date[2:]}.IWV.NC")
         ds_iwv_raw_30 = read_nc(f"{paths['radiometer']}/KV/{date_2[2:]}.IWV.NC")
         ds_iwv_raw = xr.combine_by_coords(
@@ -102,6 +102,13 @@ def postprocess_hamp(date, flightletter, version):
             )
 
     else:
+        ds_bahamas = (
+            xr.open_dataset(paths["bahamas"], engine="zarr")
+            .sel(time=date)
+            .reset_coords(["lat", "lon", "alt"])
+            .resample(time="0.25s")
+            .mean()
+        )
         ds_iwv_raw = read_nc(f"{paths['radiometer']}/KV/{date[2:]}.IWV.NC")
         radiometers = ["183", "11990", "KV"]
         ds_radiometers_raw = {}
@@ -118,7 +125,6 @@ def postprocess_hamp(date, flightletter, version):
         plane_pitch=ds_bahamas["pitch"],
         plane_roll=ds_bahamas["roll"],
         plane_altitude=ds_bahamas["alt"],
-        source=ds_bahamas.attrs["source"],
     )
     ds_iwv_lev1 = format_iwv(ds_iwv_raw).pipe(
         add_georeference,
@@ -127,7 +133,6 @@ def postprocess_hamp(date, flightletter, version):
         plane_pitch=ds_bahamas["pitch"],
         plane_roll=ds_bahamas["roll"],
         plane_altitude=ds_bahamas["alt"],
-        source=ds_bahamas.attrs["source"],
     )
     ds_radiometers_lev1 = {}
     for radio in radiometers:
@@ -146,7 +151,6 @@ def postprocess_hamp(date, flightletter, version):
         plane_pitch=ds_bahamas["pitch"],
         plane_roll=ds_bahamas["roll"],
         plane_altitude=ds_bahamas["alt"],
-        source=ds_bahamas.attrs["source"],
     )
 
     sea_land_mask = xr.open_dataarray(paths["sea_land_mask"]).load()
@@ -203,9 +207,11 @@ def postprocess_hamp(date, flightletter, version):
 
 # %% run postprocessing
 flights = pd.read_csv("flights.csv", index_col=0)
-version = "1.0"
+version = "1.1"
 for date, flightletter in zip(flights.index, flights["flightletter"]):
     postprocess_hamp(str(date), flightletter, version)
 
+# %% test postprocessing
+postprocess_hamp("20240929", "a", "1.1")
 
 # %%

scripts/testplot.py

@@ -42,7 +42,7 @@
     fig.savefig(f"Plots/{date}_hamp_filtered.png", dpi=300)
 
 # %% plot single flight
-date = "20240916"
+date = "20240929"
 flightletter = "a"
 radar = xr.open_dataset(
     f"{cfg['save_dir']}/radar/HALO-{date}{flightletter}_radar.zarr", engine="zarr"

src/process.py

@@ -421,7 +421,7 @@ def _add_roll_mask(ds):
     return ds
 
 
-def add_georeference(ds, lat, lon, plane_pitch, plane_roll, plane_altitude, source):
+def add_georeference(ds, lat, lon, plane_pitch, plane_roll, plane_altitude):
     """Add georeference information to dataset."""
     ds = ds.assign(
         plane_altitude=plane_altitude.sel(time=ds.time, method="nearest").assign_coords(
@@ -436,7 +436,6 @@ def add_georeference(ds, lat, lon, plane_pitch, plane_roll, plane_altitude, sour
             time=ds.time
         ),
     )
-    ds.attrs["georeference source"] = source
     return ds
 
 
@@ -643,23 +642,23 @@ def add_metadata_radar(ds, flight_id):
     """
 
     ds.attrs["flight_id"] = flight_id
-    ds.attrs["title"] = (
-        "MIRA Cloud Radar Moments from the HALO Microwave Package (Level 3)"
-    )
+    ds.attrs[
+        "title"
+    ] = "MIRA Cloud Radar Moments from the HALO Microwave Package (Level 3)"
     ds.attrs["summary"] = (
         "This dataset contains measurements from the MIRA cloud radar onboard the HALO aircraft during the ORCESTRA campaign. "
         "The measurements are processed and quality controlled. The processing includes formatting the data, adding georeference information, "
         "filtering for noise, clutter and valid radar states, and adding masks for calibration, ground reflections, and roll segments."
     )
     ds.attrs["creator_name"] = "Jakob Deutloff, Lukas Kluft, Clara Bayley"
-    ds.attrs["creator_email"] = (
-        "jakob.deutloff@uni-hamburg.de, lukas.kluft@mpimet.mpg.de, clara.bayley@mpimet.mpg.de"
-    )
+    ds.attrs[
+        "creator_email"
+    ] = "jakob.deutloff@uni-hamburg.de, lukas.kluft@mpimet.mpg.de, clara.bayley@mpimet.mpg.de"
     ds.attrs["project"] = "ORCESTRA, PERCUSION"
     ds.attrs["platform"] = "HALO"
-    ds.attrs["history"] = (
-        "The processing software is available at https://github.com/orcestra-campaign/hamp_processing"
-    )
+    ds.attrs[
+        "history"
+    ] = "The processing software is available at https://github.com/orcestra-campaign/hamp_processing"
     ds.attrs["license"] = "CC-BY-4.0"
     ds.attrs["featureType"] = "trajectoryProfile"
     ds.attrs["references"] = "10.5194/amt-12-1815-2019, 10.5194/essd-13-5545-2021"
@@ -698,20 +697,20 @@ def add_metadata_radiometer(ds, flight_id):
         "adding georeference information, and filtering for altitudes above 4800 m and adding a land-sea mask."
     )
     ds.attrs["creator_name"] = "Jakob Deutloff, Lukas Kluft, Clara Bayley"
-    ds.attrs["creator_email"] = (
-        "jakob.deutloff@uni-hamburg.de, lukas.kluft@mpimet.mpg.de, clara.bayley@mpimet.mpg.de"
-    )
+    ds.attrs[
+        "creator_email"
+    ] = "jakob.deutloff@uni-hamburg.de, lukas.kluft@mpimet.mpg.de, clara.bayley@mpimet.mpg.de"
     ds.attrs["project"] = "ORCESTRA, PERCUSION"
     ds.attrs["platform"] = "HALO"
-    ds.attrs["history"] = (
-        "The processing software is available at https://github.com/orcestra-campaign/hamp_processing"
-    )
+    ds.attrs[
+        "history"
+    ] = "The processing software is available at https://github.com/orcestra-campaign/hamp_processing"
     ds.attrs["license"] = "CC-BY-4.0"
     ds.attrs["featureType"] = "trajectoryProfile"
     ds.attrs["references"] = "10.5194/amt-12-1815-2019, 10.5194/essd-13-5545-2021"
-    ds.attrs["keywords"] = (
-        "Radiometer, Microwave, HALO, ORCESTRA, PERCUSION, Tropical Atlantic"
-    )
+    ds.attrs[
+        "keywords"
+    ] = "Radiometer, Microwave, HALO, ORCESTRA, PERCUSION, Tropical Atlantic"
 
     ds.attrs.pop("Comment", None)
     ds.attrs.pop("Radiometer_Location", None)
@@ -739,30 +738,30 @@ def add_metadata_iwv(ds, flight_id):
     """
 
     ds.attrs["flight_id"] = flight_id
-    ds.attrs["title"] = (
-        "Integrated Water Vapor Retrieval from the K-Band and W-Band Microwave Radiometers"
-    )
+    ds.attrs[
+        "title"
+    ] = "Integrated Water Vapor Retrieval from the K-Band and W-Band Microwave Radiometers"
     ds.attrs["summary"] = (
         "This dataset contains retrievals of integrated water vapor (IWV) from the KV-band microwave radiometer onboard the HALO aircraft during the ORCESTRA campaign. "
         "The retrieval is based on a linear regression model and should not be used for quantitative analysis. "
         "The purpose of this data was to provide an estimate of IWV during the flights. The processing includes formatting the data, "
         "adding georeference information, filtering for altitudes above 4800 m and adding a land-sea mask."
     )
     ds.attrs["creator_name"] = "Jakob Deutloff, Lukas Kluft, Clara Bayley"
-    ds.attrs["creator_email"] = (
-        "jakob.deutloff@uni-hamburg.de, lukas.kluft@mpimet.mpg.de, clara.bayley@mpimet.mpg.de"
-    )
+    ds.attrs[
+        "creator_email"
+    ] = "jakob.deutloff@uni-hamburg.de, lukas.kluft@mpimet.mpg.de, clara.bayley@mpimet.mpg.de"
     ds.attrs["project"] = "ORCESTRA, PERCUSION"
     ds.attrs["platform"] = "HALO"
-    ds.attrs["history"] = (
-        "The processing software is available at https://github.com/orcestra-campaign/hamp_processing"
-    )
+    ds.attrs[
+        "history"
+    ] = "The processing software is available at https://github.com/orcestra-campaign/hamp_processing"
     ds.attrs["license"] = "CC-BY-4.0"
     ds.attrs["featureType"] = "trajectoryProfile"
     ds.attrs["references"] = "10.5194/amt-7-4539-2014"
-    ds.attrs["keywords"] = (
-        "Radiometer, Microwave, Integrated Water Vapor, Retrieval, HALO, ORCESTRA, PERCUSION, Tropical Atlantic"
-    )
+    ds.attrs[
+        "keywords"
+    ] = "Radiometer, Microwave, Integrated Water Vapor, Retrieval, HALO, ORCESTRA, PERCUSION, Tropical Atlantic"
 
     ds.attrs.pop("Comment", None)
     ds.attrs.pop("Radiometer_Location", None)