amplifier faults masked

JakobDeutloff · JakobDeutloff · commit 83b79d3ca5dd · 2025-02-18T17:07:54.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -166,7 +166,7 @@ cython_debug/
 /planet_data
 /Radiometer_Data
 /Radar_Data
-/Data
+/Data/arts_calibration
 /Plots
 /arts_calibration
 /.vscode
diff --git a/Data/error_files/HALO-20240818.yaml b/Data/error_files/HALO-20240818.yaml
@@ -0,0 +1 @@
+'KV': [["17:01:00", "17:11:30"]]
diff --git a/Data/error_files/HALO-20240822.yaml b/Data/error_files/HALO-20240822.yaml
@@ -0,0 +1 @@
+"90": [["18:23:30", "18:23:40"]]
diff --git a/Data/error_files/HALO-20240825.yaml b/Data/error_files/HALO-20240825.yaml
@@ -0,0 +1 @@
+"90": [["17:33:20", "17:33:40"]]
diff --git a/Data/error_files/HALO-20240829.yaml b/Data/error_files/HALO-20240829.yaml
@@ -0,0 +1 @@
+"90": [["20:00:00", "20:00:20"]]
diff --git a/Data/error_files/HALO-20240831.yaml b/Data/error_files/HALO-20240831.yaml
@@ -0,0 +1 @@
+"90": [["14:05:10", "14:05:25"]]
diff --git a/Data/error_files/HALO-20240909.yaml b/Data/error_files/HALO-20240909.yaml
@@ -0,0 +1 @@
+"90": [["13:01:00", "13:08:00"]]
diff --git a/Data/error_files/HALO-20240912.yaml b/Data/error_files/HALO-20240912.yaml
@@ -0,0 +1 @@
+"119": [["12:09:00", "12:20:30"], ["15:23:00", "15:30:30"]]
diff --git a/Data/error_files/HALO-20240914.yaml b/Data/error_files/HALO-20240914.yaml
@@ -0,0 +1,2 @@
+"KV": [["18:02:00", "18:10:30"]]
+"90": [["13:01:00", "13:10:30"]]
diff --git a/Data/error_files/HALO-20240916.yaml b/Data/error_files/HALO-20240916.yaml
@@ -0,0 +1 @@
+"KV": [["12:28:00", "12:30:30"]]
diff --git a/Data/error_files/HALO-20240919.yaml b/Data/error_files/HALO-20240919.yaml
@@ -0,0 +1 @@
+"119": [["14:46:00", "14:50:30"]]
diff --git a/Data/error_files/HALO-20240921.yaml b/Data/error_files/HALO-20240921.yaml
@@ -0,0 +1 @@
+"90": [["12:57:31", "12:57:38"]]
diff --git a/Data/error_files/HALO-20240926.yaml b/Data/error_files/HALO-20240926.yaml
@@ -0,0 +1,4 @@
+'119': [['12:37:00', '12:40:30'],
+        ['17:48:00','18:00:30'],
+        ['18:34:00','18:40:30'],
+        ['19:12:00', '19:20:30']]
diff --git a/Data/error_files/HALO-20240928.yaml b/Data/error_files/HALO-20240928.yaml
@@ -0,0 +1 @@
+"119": [["16:39:00", "16:40:00"]]
diff --git a/Data/error_files/HALO-20241105.yaml b/Data/error_files/HALO-20241105.yaml
@@ -0,0 +1 @@
+"119": [["16:16:00", "16:20:30"]]
diff --git a/Data/error_files/HALO-20241107.yaml b/Data/error_files/HALO-20241107.yaml
@@ -0,0 +1,2 @@
+"183": [["15:21:00", "15:30:30"]]
+"119": [["14:47:30", "14:50:30"]]
diff --git a/Data/error_files/HALO-20241110.yaml b/Data/error_files/HALO-20241110.yaml
@@ -0,0 +1 @@
+"119": [["10:28:30", "10:30:30"]]
diff --git a/Data/error_files/HALO-20241112.yaml b/Data/error_files/HALO-20241112.yaml
@@ -0,0 +1 @@
+"119": [["11:44:30", "12:03:00"]]
diff --git a/Data/error_files/HALO-20241116.yaml b/Data/error_files/HALO-20241116.yaml
@@ -0,0 +1 @@
+"119": [["17:04:00", "17:10:30"]]
diff --git a/Data/error_files/HALO-20241119.yaml b/Data/error_files/HALO-20241119.yaml
@@ -0,0 +1 @@
+"90": [["10:02:18", "10:02:23"]]
diff --git a/scripts/find_amplifier_faults.py b/scripts/find_amplifier_faults.py
@@ -1,10 +1,10 @@
 # %%
-import matplotlib.pyplot as plt
 import xarray as xr
 import yaml
-from datetime import datetime
+from src.ipfs_helpers import read_nc
+from src.plots_functions import plot_radiometers
 
-# %% create HAMP post-processed data
+# %% load processed data
 with open("process_config.yaml", "r") as file:
     cfg = yaml.safe_load(file)
 
@@ -15,82 +15,20 @@
 )
 iwv = xr.open_dataset(f"{cfg['save_dir']}/full_iwv.zarr", engine="zarr")
 
-# %% load raw radiometer data
+# %% load raw data
+date = "20241119"
+flightletter = "a"
 ds_rad_raw = {}
 for radio in ["11990", "KV", "183"]:
-    ds_rad_raw[radio] = xr.open_dataset(
-        f"Data/Radiometer_Data/HALO-{cfg['date']}a/{radio}/{cfg['date'][2:]}.LV0.NC"
+    ds_rad_raw[radio] = read_nc(
+        f"{cfg['radiometer'].format(date=date, flightletter=flightletter)}/{radio}/{date[2:]}.LV0.NC"
     )
 
-# %% plot radiometer data
 
-
-def define_module(radio):
-    if (radio == "K") | (radio == "V"):
-        module = "KV"
-        is_90 = False
-    elif radio == "183":
-        module = "183"
-        is_90 = False
-    elif radio == "90":
-        module = "11990"
-        is_90 = True
-    elif radio == "119":
-        module = "11990"
-        is_90 = False
-    else:
-        raise ValueError("Invalid radiometer frequency")
-    return module, is_90
-
-
-freqs = {
-    "K": slice(22, 32),
-    "V": slice(50, 58),
-    "183": slice(183, 191),
-    "119": slice(120, 128),
-    "90": 90,
-}
-radio = "119"
-module, is_90 = define_module(radio)
-fig, axes = plt.subplots(3, 2, figsize=(12, 8), sharex="col", width_ratios=[10, 0.2])
-ds_rad_raw[module].sel(frequency=freqs[radio])["gain"].plot.line(
-    ax=axes[2, 0], x="time"
+# %% plot TBs and gain
+plot_radiometers(
+    radiometers.sel(time=date),
+    ds_rad_raw,
 )
-axes[1, 1].remove()
-axes[2, 1].remove()
-plt.show()
-
-# %% filter radiometers
-ds = 1
-date = cfg["date"]
-# read error file
-with open(f"Data/error_files/HALO-{date}a.yaml") as file:
-    error_file = yaml.safe_load(file)
-
-# filter data
-date_format = "%H:%M:%S"
-variables_2d = [
-    var for var in ds.radiometers.data_vars if ds.radiometers[var].ndim == 2
-]
-for radio in ["K", "V", "90", "119", "183"]:
-    if radio in error_file.keys():
-        for timeframe in error_file[radio]["times"]:
-            start_time = datetime.strptime(timeframe[0], date_format).time()
-            end_time = datetime.strptime(timeframe[1], date_format).time()
-            mask = (ds.radiometers.time.dt.time < start_time) | (
-                ds.radiometers.time.dt.time > end_time
-            )
-            for var in variables_2d:
-                value = ds.radiometers[var]
-                for f in ds.radiometers.sel(frequency=freqs[radio]).frequency.values:
-                    value.loc[{"frequency": f}] = value.loc[{"frequency": f}].where(
-                        mask
-                    )
-                ds.radiometers.assign({var: value})
-
-# %% control plot
-
-plt.show()
-
 
 # %%
diff --git a/scripts/process_hamp.py b/scripts/process_hamp.py
@@ -12,6 +12,7 @@
     filter_radiometer,
     add_masks_radar,
     add_masks_radiometer,
+    add_masks_iwv,
     add_metadata_radar,
     add_metadata_radiometer,
     add_metadata_iwv,
@@ -164,7 +165,7 @@ def postprocess_hamp(date, flightletter, version):
     ds_iwv_lev2 = (
         filter_radiometer(ds_iwv_lev1)
         .pipe(add_metadata_iwv, flight_id=f"{date}{flightletter}")
-        .pipe(add_masks_radiometer, sea_land_mask)
+        .pipe(add_masks_iwv, sea_land_mask)
     )
 
     print(f"Saving data for {date}")
@@ -196,11 +197,9 @@ def postprocess_hamp(date, flightletter, version):
     )
 
 
-# %%
-postprocess_hamp("20240929", "a", "1.0")
-
 # %% run postprocessing
 flights = pd.read_csv("flights.csv", index_col=0)
+flights = flights.loc[20241119:]
 version = "1.0"
 for date, flightletter in zip(flights.index, flights["flightletter"]):
     postprocess_hamp(str(date), flightletter, version)
diff --git a/scripts/testplot.py b/scripts/testplot.py
@@ -4,22 +4,23 @@
 import yaml
 import pandas as pd
 
-# %% load data
+# %% load config
 flights = pd.read_csv("flights.csv", index_col=0)
 
 with open("process_config.yaml", "r") as file:
     cfg = yaml.safe_load(file)
 
+# %% load full data
 radar = xr.open_dataset(f"{cfg['save_dir']}/full_radar.zarr", engine="zarr")
 radiometers = xr.open_dataset(
     f"{cfg['save_dir']}/full_radiometer.zarr",
     engine="zarr",
 )
 iwv = xr.open_dataset(f"{cfg['save_dir']}/full_iwv.zarr", engine="zarr")
 
-# %%
-for date in [20240929]:
-    if date == 20240929:  # only flight that crossed 0 UTC
+# %% plot all flights
+for date in flights.index:
+    if date == 20240928:  # only flight that crossed 0 UTC
         fig = testplot_hamp(
             radar.sel(time=slice("20240929", "20240930")),
             radiometers.sel(time=slice("20240929", "20240930")),
@@ -36,7 +37,32 @@
             ground_filter=True,
             roll_filter=True,
             calibration_filter=True,
+            amplifier_faults=True,
         )
-    fig.savefig(f"Plots/{date}_hamp.png", dpi=300)
+    fig.savefig(f"Plots/{date}_hamp_filtered.png", dpi=300)
+
+# %% plot single flight
+date = "20240916"
+flightletter = "a"
+radar = xr.open_dataset(
+    f"{cfg['save_dir']}/radar/HALO-{date}{flightletter}_radar.zarr", engine="zarr"
+)
+radiometers = xr.open_dataset(
+    f"{cfg['save_dir']}/radiometer/HALO-{date}{flightletter}_radio.zarr", engine="zarr"
+)
+iwv = xr.open_dataset(
+    f"{cfg['save_dir']}/iwv/HALO-{date}{flightletter}_iwv.zarr", engine="zarr"
+)
+
+# %%
+fig = testplot_hamp(
+    radar,
+    radiometers,
+    iwv,
+    ground_filter=True,
+    roll_filter=True,
+    calibration_filter=True,
+    amplifier_faults=True,
+)
 
 # %%
diff --git a/src/config.json b/src/config.json
@@ -3,5 +3,6 @@
     "valid_radar_state": 13,
     "noise_threshold": 100,
     "altitude_threshold": 4800,
-    "plane_variables": ["lat", "lon", "plane_altitude", "plane_roll", "plane_pitch"]
+    "plane_variables": ["lat", "lon", "plane_altitude", "plane_roll", "plane_pitch"],
+    "path_error_file": "/home/m/m301049/hamp_processing/Data/error_files/"
 }
diff --git a/src/plots_functions.py b/src/plots_functions.py
@@ -59,6 +59,7 @@ def testplot_hamp(
     ground_filter=True,
     roll_filter=True,
     calibration_filter=True,
+    amplifier_faults=True,
 ):
     plt.close("all")
     fig, axes = plt.subplots(
@@ -76,6 +77,8 @@ def testplot_hamp(
         data = data.where(ds_radar["mask_roll"])
     if calibration_filter:
         data = data.where(ds_radar["mask_calibration"])
+    if amplifier_faults:
+        ds_radiometers = ds_radiometers.where(ds_radiometers["mask_amplifier_fault"])
 
     col = data.plot.pcolormesh(
         ax=axes[0, 0],
@@ -129,6 +132,71 @@ def testplot_hamp(
     return fig
 
 
+def define_module(radio):
+    if (radio == "K") | (radio == "V"):
+        module = "KV"
+    elif radio == "183":
+        module = "183"
+    elif radio == "90":
+        module = "11990"
+    elif radio == "119":
+        module = "11990"
+    else:
+        raise ValueError("Invalid radiometer frequency")
+    return module
+
+
+def plot_radiometers(
+    ds_radiometers,
+    ds_radiometers_raw,
+):
+    plt.close("all")
+    fig, axes = plt.subplots(
+        5,
+        2,
+        figsize=(12, 7),
+        sharex="col",
+        width_ratios=[10, 1],
+    )
+
+    freqs = {
+        "K": slice(22, 32),
+        "V": slice(50, 58),
+        "183": slice(183, 191),
+        "119": slice(120, 128),
+        "90": 90,
+    }
+    for i, radio in enumerate(freqs.keys()):
+        ds_radiometers["TBs"].sel(frequency=freqs[radio]).plot.line(
+            ax=axes[i, 0], x="time", add_legend=False
+        )
+        if radio != "90":
+            handles = list(axes[i, 0].get_lines())
+            labels = ds_radiometers.sel(frequency=freqs[radio]).frequency.values
+            axes[i, 1].legend(handles, labels)
+        axes[i, 1].axis("off")
+        axes[i, 0].set_ylabel("TB / K")
+        axes[i, 0].set_xlabel("")
+        axes[i, 0].set_title(f"{radio}")
+        axes[i, 0].spines[["top"]].set_visible(False)
+        ax2 = axes[i, 0].twinx()
+        ax2.spines[["top"]].set_visible(False)
+        ax2.plot(
+            ds_radiometers_raw[define_module(radio)].time,
+            ds_radiometers_raw[define_module(radio)].sel(frequency=freqs[radio])[
+                "gain"
+            ],
+            color="grey",
+        )
+        ax2.set_ylabel("Gain")
+
+    axes[4, 0].set_xlabel("Time")
+    fig.tight_layout()
+    plt.show()
+
+    return fig
+
+
 def plot_regression(regression_coeffs, TB_arts, TB_hamp, date):
     fig, axes = plt.subplots(5, 5, figsize=(15, 15))
     TB_hamp = TB_hamp[TB_hamp.index.date == date]
diff --git a/src/process.py b/src/process.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+"119": [["12:09:00", "12:20:30"], ["15:23:00", "15:30:30"]]`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+"KV": [["18:02:00", "18:10:30"]]`
	`2`	`+"90": [["13:01:00", "13:10:30"]]`