Update Processing

.gitignore

@@ -166,11 +166,11 @@ cython_debug/
 /planet_data
 /Radiometer_Data
 /Radar_Data
-/Data
+/Data/arts_calibration
+/Plots
 /arts_calibration
 /.vscode
 
 # other or unknown
 **/.DS_Store
 **/config.yaml
-**/process_config.yaml

.pre-commit-config.yaml

@@ -31,11 +31,6 @@ repos:
         description: python linter
     -   id: ruff-format
         description: python formatter
--   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.3.0
-    hooks:
-    -   id: black-jupyter
-        description: formating ipynb notebook
 -   repo: https://github.com/compilerla/conventional-pre-commit
     rev: v3.4.0
     hooks:

config_ipns.yaml

@@ -0,0 +1,9 @@
+date: '20240926'
+flightletter: a
+flightname: HALO-{date}{flightletter}
+iwv: ipns://latest.orcestra-campaign.org/products/HALO/iwv/HALO-{date}{flightletter}.zarr
+path_dropsondes: ipns://latest.orcestra-campaign.org/products/HALO/dropsondes/Level_3/PERCUSION_Level_3.zarr
+path_saveplots: ''
+position_attitude: ipns://latest.orcestra-campaign.org/products/HALO/position_attitude/HALO-{date}{flightletter}.zarr
+radar: ipns://latest.orcestra-campaign.org/products/HALO/radar/moments/HALO-{date}{flightletter}.zarr
+radiometer: ipns://latest.orcestra-campaign.org/products/HALO/radiometer/HALO-{date}{flightletter}.zarr

error_files/HALO-20240818.yaml

@@ -0,0 +1 @@
+'KV': [["17:01:00", "17:11:30"]]

error_files/HALO-20240822.yaml

@@ -0,0 +1 @@
+"90": [["18:23:30", "18:23:40"]]

error_files/HALO-20240825.yaml

@@ -0,0 +1 @@
+"90": [["17:33:20", "17:33:40"]]

error_files/HALO-20240829.yaml

@@ -0,0 +1 @@
+"90": [["20:00:00", "20:00:20"]]

error_files/HALO-20240831.yaml

@@ -0,0 +1 @@
+"90": [["14:05:10", "14:05:25"]]

error_files/HALO-20240909.yaml

@@ -0,0 +1 @@
+"90": [["13:01:00", "13:08:00"]]

error_files/HALO-20240912.yaml

@@ -0,0 +1 @@
+"119": [["12:09:00", "12:20:30"], ["15:23:00", "15:30:30"]]

error_files/HALO-20240914.yaml

@@ -0,0 +1,2 @@
+"KV": [["18:02:00", "18:10:30"]]
+"90": [["13:01:00", "13:10:30"]]

error_files/HALO-20240916.yaml

@@ -0,0 +1 @@
+"KV": [["12:28:00", "12:30:30"]]

error_files/HALO-20240919.yaml

@@ -0,0 +1 @@
+"119": [["14:46:00", "14:50:30"]]

error_files/HALO-20240921.yaml

@@ -0,0 +1 @@
+"90": [["12:57:31", "12:57:38"]]

error_files/HALO-20240926.yaml

@@ -0,0 +1,4 @@
+'119': [['12:37:00', '12:40:30'],
+        ['17:48:00','18:00:30'],
+        ['18:34:00','18:40:30'],
+        ['19:12:00', '19:20:30']]

error_files/HALO-20240928.yaml

@@ -0,0 +1 @@
+"119": [["16:39:00", "16:40:00"]]

error_files/HALO-20241105.yaml

@@ -0,0 +1 @@
+"119": [["16:16:00", "16:20:30"]]

error_files/HALO-20241107.yaml

@@ -0,0 +1,2 @@
+"183": [["15:21:00", "15:30:30"]]
+"119": [["14:47:30", "14:50:30"]]

error_files/HALO-20241110.yaml

@@ -0,0 +1 @@
+"119": [["10:28:30", "10:30:30"]]

error_files/HALO-20241112.yaml

@@ -0,0 +1 @@
+"119": [["11:44:30", "12:03:00"]]

error_files/HALO-20241116.yaml

@@ -0,0 +1 @@
+"119": [["17:04:00", "17:10:30"]]

error_files/HALO-20241119.yaml

@@ -0,0 +1 @@
+"90": [["10:02:18", "10:02:23"]]

flights.csv

@@ -0,0 +1,33 @@
+,flightletter,location
+20240809,b,transfer
+20240811,a,sal
+20240813,a,sal
+20240816,a,sal
+20240818,a,sal
+20240821,a,sal
+20240822,a,sal
+20240825,a,sal
+20240827,a,sal
+20240829,a,sal
+20240831,a,sal
+20240903,a,sal
+20240906,a,transfer
+20240907,a,barbados
+20240909,a,barbados
+20240912,a,barbados
+20240914,a,barbados
+20240916,a,barbados
+20240919,a,barbados
+20240921,a,barbados
+20240923,a,barbados
+20240924,a,barbados
+20240926,a,barbados
+20240928,a,barbados
+20240929,a,transfer
+20241105,a,op
+20241107,a,op
+20241110,a,op
+20241112,b,op
+20241114,b,op
+20241116,a,op
+20241119,a,op

process_config.yaml

@@ -0,0 +1,8 @@
+date: '20240827'
+flightletter: a
+flightname: HALO-{date}{flightletter}
+bahamas: ipns://latest.orcestra-campaign.org/products/HALO/position_attitude.zarr
+radar: ipns://latest.orcestra-campaign.org/raw/HALO/radar/HALO-{date}{flightletter}/mom/*.nc
+radiometer: ipns://latest.orcestra-campaign.org/raw/HALO/radiometer/HALO-{date}{flightletter}
+sea_land_mask: /work/bm1183/m301049/orcestra/sea_land_mask.nc
+save_dir: /work/bm1183/m301049/orcestra/Hamp_Processed_vn2

scripts/arts_calibration/analyse_bt_diffs.py

@@ -2,7 +2,8 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 import xarray as xr
-from src import readwrite_functions as rwfuncs
+import yaml
+import numpy as np
 
 # %% define frequencies
 freq_k = [22.24, 23.04, 23.84, 25.44, 26.24, 27.84, 31.40]
@@ -41,38 +42,16 @@
     "20240928",
 ]
 
-next_flight_183 = ["202408"]
 
 # %% Read csv BTs
-dates = [
-    "20240811",
-    "20240813",
-    "20240816",
-    "20240818",
-    "20240821",
-    "20240822",
-    "20240825",
-    "20240827",
-    "20240829",
-    "20240831",
-    "20240903",
-    "20240906",
-    "20240907",
-    "20240909",
-    "20240912",
-    "20240914",
-    "20240916",
-    "20240919",
-    "20240921",
-    "20240923",
-    "20240924",
-    "20240926",
-    "20240928",
+flights = pd.read_csv("flights.csv", index_col=0)
+flights_processed = flights[
+    (flights["location"] == "sal") | (flights["location"] == "barbados")
 ]
 
 TB_arts_list = []
 TB_hamp_list = []
-for date in dates:
+for date in flights_processed.index:
     TB_arts_list.append(
         pd.read_csv(f"Data/arts_calibration/HALO-{date}a/TBs_arts.csv", index_col=0)
     )
@@ -81,18 +60,25 @@
     )
 
 # load dropsonde data
-configfile = "config_ipns.yaml"
-cfg = rwfuncs.extract_config_params(configfile)
-ds_dropsonde = xr.open_dataset(cfg["path_dropsondes"], engine="zarr")
+with open("process_config.yaml") as f:
+    cfg = yaml.safe_load(f)
+
+ds_dropsonde = xr.open_dataset(
+    "ipns://latest.orcestra-campaign.org/products/HALO/dropsondes/Level_3/PERCUSION_Level_3.zarr",
+    engine="zarr",
+)
+ds_dropsonde = ds_dropsonde.assign_coords(sonde=np.arange(ds_dropsonde.sonde.size))
 
-# %% restructure data
+# restructure data
 TB_arts = pd.concat(TB_arts_list, axis=1)
 TB_hamp = pd.concat(TB_hamp_list, axis=1)
-launch_time = ds_dropsonde.sel(sonde_id=TB_arts.columns).launch_time.values
+launch_time = ds_dropsonde.sel(
+    sonde=[int(float(x)) for x in TB_arts.columns.values]
+).sonde_time.values
 TB_arts.columns = launch_time
 TB_hamp.columns = launch_time
-TB_arts = TB_arts.T
-TB_hamp = TB_hamp.T
+TB_arts = TB_arts.T.dropna()
+TB_hamp = TB_hamp.T.dropna()
 
 # %% calculate statistics for each flight
 diffs = TB_arts - TB_hamp