Finish 40 ensemble member generation?

Author: briochemc

scripts/archive_unarchived_CMIP6_ACCESS_GM_files.sh

@@ -28,11 +28,11 @@ model=ACCESS-ESM1-5
 # model=ACCESS-CM2
 
 # experiment=historical
-# members=("HI-06","HI-33","HI-34","HI-35","HI-36")
+# members=("HI-43","HI-44")
 # decade_start=1850
 # decade_end=2020
 experiment=ssp370
-members=("SSP-370-31","SSP-370-32","SSP-370-33","SSP-370-34","SSP-370-35","SSP-370-36","SSP-370-37","SSP-370-38","SSP-370-39","SSP-370-40","SSP-370-41","SSP-370-42","SSP-370-43","SSP-370-44")
+members=("SSP-370-43","SSP-370-44")
 decade_start=2010
 decade_end=2100
 

scripts/check_archived_transport.jl

@@ -0,0 +1,21 @@
+# This script test the "completion rate" of archived data
+using Format
+using Test
+
+datadir = "/g/data/xv83/TMIP/data"
+modeldir = joinpath(datadir, "ACCESS-ESM1-5")
+experiments = ["historical", "ssp370"]
+exp_prefix = ["HI", "SSP-370"]
+vars = ["$(str1)_trans_$(str2)" for str1 in ("tx", "ty") for str2 in ("gm", "submeso")]
+
+@testset "archived data" begin
+    @testset "$(member)" for member in 1:40
+        @testset "$(experiment)" for (experiment, exp_prefix) in zip(experiments, exp_prefix)
+            expdir = joinpath(modeldir, experiment)
+            CSIRO_member = "$exp_prefix-$(format(member + 4, width=2, zeropadding=true))"
+            memberdir = joinpath(expdir, CSIRO_member)
+            decades = (experiment == "historical") ? (1850:10:2010) : (2010:10:2090)
+            @test all(isfile(joinpath(memberdir, "month_$(var)_$(decade)s.nc")) for var in vars for decade in decades)
+        end
+    end
+end

scripts/check_cyclo_average_data.jl

@@ -1,44 +1,22 @@
-# This script just prints a tiny table that shows the "completion rate" of making  variables
+# This script test the "completion rate" of making averaged variables
 using Format
-using DataFrames
-using DataFramesMeta
+
 datadir = "/scratch/xv83/TMIP/data"
 modeldir = joinpath(datadir, "ACCESS-ESM1-5")
-decades = [1850, 1990, 2020, 2030, 2090]
+decades = [1850, 1990, 2030, 2090]
 # decades = [1990]
 experiments = [t < 2010 ? "historical" : "ssp370" for t in decades]
 vars = ("umo", "vmo", "mlotst", "tx_trans_gm", "ty_trans_gm", "tx_trans_submeso", "ty_trans_submeso")
 members = map(m -> "r$(m)i1p1f1", 1:40)
-df = DataFrame(experiment = String[], decade = Int64[], member = String[], variable = String[], isfile = Bool[])
-for (decade, experiment) in zip(decades, experiments)
-    experimentdir = joinpath(modeldir, experiment)
-    for member in members
-        dir = joinpath(experimentdir, member, "Jan$(decade)-Dec$(decade + 9)", "cyclomonth")
-        for variable in vars
-            fname = joinpath(dir, "$variable.nc")
-            row = (; experiment, decade, member, variable, isfile = isfile(fname))
-            push!(df, row)
+
+using Test
+files = ["$var.nc" for var in vars]
+@testset "saved data" begin
+    @testset "$(decade) $(experiment)" for (decade, experiment) in zip(decades, experiments)
+        experimentdir = joinpath(modeldir, experiment)
+        @testset "$(member)" for member in members
+            dir = joinpath(experimentdir, member, "Jan$(decade)-Dec$(decade + 9)", "cyclomonth")
+            @test all(isfile(joinpath(dir, file)) for file in files)
         end
     end
-end
-df
-Nmembers = length(members)
-Nvars = length(vars)
-df2 = @chain df begin
-    # @groupby(:decade)
-    # @combine(:count => sum(:isfile))
-    # @by(:decade, :done = 100 * sum(:isfile) / (Nmembers * Nvars))
-    @by([:decade, :variable], :done = 100 * sum(:isfile) / Nmembers)
-end
-# df2 = @chain df begin
-#     # @subset(:isfile)
-#     # @rtransform(:filesize = round(:filesize, sigdigits=2))
-#     # @select(:variable, :filesize)
-#     @groupby(:decade)
-#     @combine(:count => sum(:isfile))
-#     # @transform(:maxsize = maximum(:filesize))
-#     # @rsubset(!isapprox(:filesize, :maxsize, atol = 0.1 * :maxsize), :decade ≠ 2010)
-#     # @rselect(:CSIRO_member, :decade, :variable, :potentially_missing_data = round(Int, pc, 1 - :filesize / :maxsize |> pc))
-#     # @orderby(:CSIRO_member, :decade, :variable)
-# end
-show(df2, allrows = true)
+end

scripts/check_cyclo_matrices.jl

@@ -0,0 +1,61 @@
+# This script just prints a tiny table that shows the "completion rate" of making  variables
+using Format
+using DataFrames
+using DataFramesMeta
+datadir = "/scratch/xv83/TMIP/data"
+modeldir = joinpath(datadir, "ACCESS-ESM1-5")
+decades = [1850, 1990, 2030, 2090]
+# decades = [1990]
+experiments = [t < 2010 ? "historical" : "ssp370" for t in decades]
+files = ("cyclo_matrix_$i.jld2" for i in 1:12)
+members = map(m -> "r$(m)i1p1f1", 1:40)
+df = DataFrame(experiment = String[], decade = Int64[], member = String[], file = String[], isfile = Bool[])
+for (decade, experiment) in zip(decades, experiments)
+    experimentdir = joinpath(modeldir, experiment)
+    for member in members
+        dir = joinpath(experimentdir, member, "Jan$(decade)-Dec$(decade + 9)", "cyclomonth")
+        for file in files
+            filepath = joinpath(dir, file)
+            row = (; experiment, decade, member, file, isfile = isfile(filepath))
+            push!(df, row)
+        end
+    end
+end
+df
+Nmembers = length(members)
+Nfiles = length(files)
+df2 = @chain df begin
+    # @groupby(:decade)
+    # @combine(:count => sum(:isfile))
+    # @by(:decade, :done = 100 * sum(:isfile) / (Nmembers * Nvars))
+    # @by([:decade], :count = "$(sum(:isfile)/Nfiles)/40", :done = (sum(:isfile) == Nmembers * Nfiles) ? "✓" : "")
+    @by([:decade, :member], :done = (sum(:isfile) == Nfiles) ? "✓" : "")
+end
+show(df2, allrows = true)
+
+using Test
+@testset "Matrices" begin
+    @testset "$experiment $decade" for (decade, experiment) in zip(decades, experiments)
+        experimentdir = joinpath(modeldir, experiment)
+        @testset "$member" for member in members
+            dir = joinpath(experimentdir, member, "Jan$(decade)-Dec$(decade + 9)", "cyclomonth")
+            @test all(isfile(joinpath(dir, file)) for file in files)
+            # @testset "$member" for file in files
+            #     filepath = joinpath(dir, file)
+            #     @test isfile(filepath)
+            # end
+        end
+    end
+end
+# df2 = @chain df begin
+#     # @subset(:isfile)
+#     # @rtransform(:filesize = round(:filesize, sigdigits=2))
+#     # @select(:variable, :filesize)
+#     @groupby(:decade)
+#     @combine(:count => sum(:isfile))
+#     # @transform(:maxsize = maximum(:filesize))
+#     # @rsubset(!isapprox(:filesize, :maxsize, atol = 0.1 * :maxsize), :decade ≠ 2010)
+#     # @rselect(:CSIRO_member, :decade, :variable, :potentially_missing_data = round(Int, pc, 1 - :filesize / :maxsize |> pc))
+#     # @orderby(:CSIRO_member, :decade, :variable)
+# end
+println("done")

scripts/cyclo_average_CMIP6_ACCESS_variables.sh

@@ -3,8 +3,8 @@
 #PBS -P xv83
 #PBS -N CMIP6_ACCESS_preprocessing
 #PBS -l ncpus=28
-#PBS -q normal
-#PBS -l mem=180GB
+#PBS -q hugemem
+#PBS -l mem=200GB
 #PBS -l jobfs=4GB
 #PBS -l walltime=24:00:00
 #PBS -l storage=gdata/xv83+gdata/oi10+gdata/dk92+gdata/hh5+gdata/rr3+gdata/al33+gdata/fs38+gdata/xp65+gdata/p73
@@ -27,12 +27,12 @@ module load python3/3.12.1
 # CHANGE HERE the model, experiment, ensemble, etc.
 model=ACCESS-ESM1-5
 # model=ACCESS-CM2
-# experiment=historical
-# year_start=1850
+experiment=historical
+year_start=1850
 # year_start=1990
-experiment=ssp370
+# experiment=ssp370
 # year_start=2030
-year_start=2090
+# year_start=2090
 num_years=10
 lumpby=month
 # lumpby=season

scripts/cyclo_average_unarchived_CMIP6_ACCESS_GM_variables.py

@@ -146,12 +146,13 @@ def climatology(ds, lumpby):
 decades = range(decade_start, year_start + num_years, 10)
 
 # Members to loop through
-members = range(1, 41)
+# members = range(1, 41)
 # members = range(1, 9)
 # members = range(9, 41)
 # members = [1, 3, 4, 5, 6, 7, 8]
 # members = [1, 3, 4]
 # members = [5, 6, 7, 8]
+members = [39, 40]
 
 
 print("Starting client")