Archive and cyclo average CMIP6 data

Author: briochemc

scripts/archive_unarchived_CMIP6_ACCESS_GM_files.sh

@@ -28,7 +28,7 @@ model=ACCESS-ESM1-5
 # model=ACCESS-CM2
 
 experiment=historical
-members=("HI-37","HI-38","HI-39","HI-40","HI-41","HI-42","HI-43","HI-44")
+members=("HI-33","HI-34","HI-35","HI-36")
 decade_start=1850
 decade_end=2020
 # experiment=ssp370

scripts/average_unarchived_CMIP6_ACCESS_GM_variables.py

@@ -112,7 +112,7 @@ def open_my_dataset(paths):
         print(f"\nProcessing {CSIRO_member(member)} as {CMIP6_member(member)}")
 
         # directory to save the data to (as NetCDF)
-        inputdir = f'{gdatadatadir}/{model}/{CSIRO_member(member)}'
+        inputdir = f'{gdatadatadir}/{model}/{experiment}/{CSIRO_member(member)}'
         outputdir = f'{scratchdatadir}/{model}/{experiment}/{CMIP6_member(member)}/{start_time_str}-{end_time_str}'
         print("Creating directory: ", outputdir)
         os.makedirs(outputdir, exist_ok=True)

scripts/check_archive_unarchived_CMIP6_ACCESS_GM_files.py

@@ -6,10 +6,10 @@
 # interactive use only
 model="ACCESS-ESM1-5"
 experiment="historical"
-# year_start = 1850
-year_start = 1940
-# year_end = 2015
-year_end = 1950
+# decade_start = 1850
+decade_start = 1940
+# decade_end = 2015
+decade_end = 1950
 
 
 # Model etc. defined from script input
@@ -19,10 +19,10 @@
 print("Experiment: ", experiment, " (type: ", type(experiment), ")")
 members = sys.argv[3].split(',')
 print("members: ", members, " (type: ", type(members), ")")
-year_start = int(sys.argv[4])
-print("year_start: ", year_start, " (type: ", type(year_start), ")")
-year_end = int(sys.argv[5])
-print("year_end: ", year_end, " (type: ", type(year_end), ")")
+decade_start = int(sys.argv[4])
+print("decade_start: ", decade_start, " (type: ", type(decade_start), ")")
+decade_end = int(sys.argv[5])
+print("decade_end: ", decade_end, " (type: ", type(decade_end), ")")
 
 # 1. Load packages
 
@@ -45,6 +45,20 @@
 # # Load xmip for preprocessing (trying to get consistent metadata for making matrices down the road)
 # from xmip.preprocessing import combined_preprocessing
 
+decades = range(decade_start, decade_end, 10)
+print(f"\nDecades:\n")
+print(*decades)
+
+# Historical runs go from 1850 to 2015, and future scenarios from 2015 to 2100.
+# I want to save data per decade, so I need some logic branching to deal with the 2010s.
+def decade_years(decade, experiment):
+    if decade == 2010:
+        if (experiment == "historical"):
+            return range(2010, 2015)
+        else:
+            return range(2015, 2020)
+    else:
+        return range(decade, decade + 10)
 
 
 
@@ -57,13 +71,11 @@
 # members = ["HI-09", "HI-10", "HI-11", "HI-12"]
 # members = ["HI-05"]
 
-
-
 print("Starting client")
 
 # This `if` statement is required in scripts (not required in Jupyter)
 if __name__ == '__main__':
-    client = Client(n_workers=24, threads_per_worker=1)
+    client = Client(n_workers=40, threads_per_worker=1)
     #, threads_per_worker=1, memory_limit='16GB') # Note: with 1thread/worker cannot plot thetao. Maybe I need to understand why?
     # added threads_per_worker=1 back again because I possibly hitting some random unsafe multithreading issue:
     # https://forum.access-hive.org.au/t/netcdf-not-a-valid-id-errors/389
@@ -72,27 +84,27 @@
     for member in members:
 
         # print ensemble/member
-        inputdir = f'/scratch/p66/pbd562/petrichor/get/{member}/history/ocn'
-        outputdir = f'{gdatadatadir}/{model}/{member}'
-        print(f"\n{member}")
+        if experiment == "historical":
+            inputdir = f'/scratch/p66/pbd562/petrichor/get/{member}/history/ocn'
+        else:
+            inputdir = f'/scratch/p66/pbd562/petrichor/get/{experiment}/{member}/history/ocn'
 
-        for decade in range(year_start, year_end, 10):
+        outputdir = f'{gdatadatadir}/{model}/{experiment}/{member}'
+        print(f"\nProcessing {member}")
 
-            # print(f'  {decade}s')
+        for decade in decades:
 
-            # subset of the files required
-            # paths = [f'{inputdir}/history/ocn/ocean_month.nc-{year}1231' for year in range(year_start, year_end)]
-            # paths = [f'{inputdir}/history/ocn/ocean_month.nc-{year}1231' for year in range(1990, 2000)]
-            if decade == 2010:
-                num_years = 5
-            else:
-                num_years = 10
+            print(f'\nDecade {decade}:\n')
+
+            years = decade_years(decade, experiment)
+            print(*years)
 
-            paths = [f'{inputdir}/ocean_month.nc-{year}1231' for year in range(decade, decade + num_years)]
+            # subset of the files required
+            paths = [f'{inputdir}/ocean_month.nc-{year}1231' for year in years]
 
             # Exit early if cannot find all files
             if not all([os.path.isfile(fname) for fname in paths]):
-                print(f'Missing files for {member} {decade}-{decade + num_years}')
+                print(f'Missing files for {member} {decade}s')
                 continue
 
             # load the data

scripts/check_archive_unarchived_CMIP6_ACCESS_GM_files.sh

@@ -2,10 +2,10 @@
 
 #PBS -P xv83
 #PBS -N archive_GM_check
-#PBS -l ncpus=28
+#PBS -l ncpus=48
 #PBS -l mem=180GB
 #PBS -l jobfs=4GB
-#PBS -l walltime=12:00:00
+#PBS -l walltime=24:00:00
 #PBS -l storage=gdata/xv83+gdata/dk92+gdata/hh5+gdata/xp65+gdata/p73+scratch/p66
 #PBS -l wd
 #PBS -o output/PBS/
@@ -23,20 +23,28 @@ conda info
 echo "Loading python3/3.12.1"
 module load python3/3.12.1
 
-# CHANGE HERE the model, experiment, ensemble, etc.
+# CHANGE HERE the model, experiment, members, decades.
 model=ACCESS-ESM1-5
 # model=ACCESS-CM2
-experiment=historical
-year_start=1850
-# year_start=1990
-year_end=2015
-# year_end=2000
-members=("HI-05","HI-07","HI-08","HI-09","HI-10","HI-11","HI-12")
-# members=("HI-07")
+
+# experiment=historical
+# # members=("HI-05","HI-06","HI-07","HI-08","HI-09","HI-10","HI-11","HI-12","HI-13","HI-14")
+# # members=("HI-15","HI-16","HI-17","HI-18","HI-19","HI-20","HI-21","HI-22","HI-23","HI-24")
+# # members=("HI-25","HI-26","HI-27","HI-28","HI-29","HI-30","HI-31","HI-32","HI-33","HI-34")
+# members=("HI-35","HI-36","HI-37","HI-38","HI-39","HI-40","HI-41","HI-42","HI-43","HI-44")
+# decade_start=1850
+# decade_end=2020
+experiment=ssp370
+# members=("SSP-370-05","SSP-370-06","SSP-370-07","SSP-370-08","SSP-370-09","SSP-370-10","SSP-370-11","SSP-370-12","SSP-370-13","SSP-370-14")
+# members=("SSP-370-15","SSP-370-16","SSP-370-17","SSP-370-18","SSP-370-19","SSP-370-20","SSP-370-21","SSP-370-22","SSP-370-23","SSP-370-24")
+# members=("SSP-370-25","SSP-370-26","SSP-370-27","SSP-370-28","SSP-370-29","SSP-370-30","SSP-370-31","SSP-370-32","SSP-370-33","SSP-370-34")
+members=("SSP-370-35","SSP-370-36","SSP-370-37","SSP-370-38","SSP-370-39","SSP-370-40","SSP-370-41","SSP-370-42","SSP-370-43","SSP-370-44")
+decade_start=2010
+decade_end=2100
 
 
 echo "Running transport-state script"
-python scripts/check_archive_unarchived_CMIP6_ACCESS_GM_files.py $model $experiment $members $year_start $year_end \
+python scripts/check_archive_unarchived_CMIP6_ACCESS_GM_files.py $model $experiment $members $decade_start $decade_end \
 &> output/$PBS_JOBID.$model.monthly.checkarchiveddatavsTilos.out
 
 

scripts/cyclo_average_CMIP6_ACCESS_variables.py

@@ -252,7 +252,7 @@ def sort_members(members):
 if __name__ == '__main__':
     client = Client(n_workers=4, threads_per_worker=1)#, threads_per_worker=1, memory_limit='16GB') # Note: with 1thread/worker cannot plot thetao. Maybe I need to understand why?
 
-    for member in sorted_members[0:1]:
+    for member in sorted_members:
 
         # print ensemble/member
         print(f"\nProcessing member: {member}")

scripts/cyclo_average_CMIP6_ACCESS_variables.sh

@@ -2,10 +2,10 @@
 
 #PBS -P xv83
 #PBS -N CMIP6_ACCESS_preprocessing
-#PBS -l ncpus=28
+#PBS -l ncpus=48
 #PBS -l mem=180GB
 #PBS -l jobfs=4GB
-#PBS -l walltime=12:00:00
+#PBS -l walltime=24:00:00
 #PBS -l storage=gdata/xv83+gdata/oi10+gdata/dk92+gdata/hh5+gdata/rr3+gdata/al33+gdata/fs38+gdata/xp65+gdata/p73
 #PBS -l wd
 #PBS -o output/PBS/

scripts/cyclo_average_unarchived_CMIP6_ACCESS_GM_variables.py

@@ -138,7 +138,10 @@ def climatology(ds, lumpby):
 decades = range(decade_start, year_start + num_years, 10)
 
 # Members to loop through
-members = [1, 3, 4, 5, 6, 7, 8]
+# members = range(1, 41)
+members = range(1, 9)
+# members = range(9, 41)
+# members = [1, 3, 4, 5, 6, 7, 8]
 # members = [1, 3, 4]
 # members = [5, 6, 7, 8]
 
@@ -149,13 +152,13 @@ def climatology(ds, lumpby):
 if __name__ == '__main__':
     client = Client(n_workers=24) #, threads_per_worker=1, memory_limit='16GB') # Note: with 1thread/worker cannot plot thetao. Maybe I need to understand why?
 
-    for member in members[0:1]:
+    for member in members:
 
         # print ensemble/member
         print(f"\nProcessing {CSIRO_member(member)} as {CMIP6_member(member)}")
 
         # directory to save the data to (as NetCDF)
-        inputdir = f'{gdatadatadir}/{model}/{CSIRO_member(member)}'
+        inputdir = f'{gdatadatadir}/{model}/{experiment}/{CSIRO_member(member)}'
         outputdir = f'{scratchdatadir}/{model}/{experiment}/{CMIP6_member(member)}/{start_time_str}-{end_time_str}/cyclo{lumpby}'
         print("Creating directory: ", outputdir)
         os.makedirs(outputdir, exist_ok=True)