Skip to content

Commit aeafaf6

Browse files
navidcynavidcy
authored
Fixes bugs for ECCO2, adds some more love to ECCO2, and tests (#479)
* fixes bugs for ECCO2 + add test for downloading * test are not only for ecco4 * day(1) for daily * add validation for Metadatum * fix docstring * reorganize ecco/en4 tests * add empty line * alignment * add test_ECCO_datasets * define metadata to download * less test * split ecco2 & ecco4/en4 tests * fix tests * cleaner * create the default ECCO directories if not existing * cleanup * fix default_download_directory * resolve conflicts * cleanup * separate tests * separate tests * ECCODaily -> ECCO2Daily * Update ECCO_metadata.jl * update docstring for native_times * end_data -> end_date * code alignment * fix inpainting docs * don't test daily for now * fix bugs * import znode * import Field from Oceananigans; fix docs for inpaint_mask! * try again with only the montly datasets * fixes * add dates in test_ocean_sea_ice_model * fix MultiYearJRA55 test * bump patch release * fix tests * updates * remove empty line * add debug message for inpainting * restructure tests to minimize re-inpainting * drop debug message * merge main * generalize test to work with Daily datasets * reduce chat :) * split ecco2 tests
1 parent cdbe169 commit aeafaf6

19 files changed

+436
-311
lines changed

‎.buildkite/pipeline.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,28 @@ steps:
4444
slurm_cpus_per_task: 8
4545
slurm_ntasks: 1
4646

47+
- label: "CPU ECCO2Monthly tests"
48+
key: "cpu_ecco2_monthly_tests"
49+
env:
50+
TEST_GROUP: "ecco2_monthly"
51+
commands:
52+
- "julia --project -e 'using Pkg; Pkg.test()'"
53+
agents:
54+
slurm_mem: 32G
55+
slurm_cpus_per_task: 8
56+
slurm_ntasks: 1
57+
58+
- label: "CPU ECCO2Daily tests"
59+
key: "cpu_ecco2_daily_tests"
60+
env:
61+
TEST_GROUP: "ecco2_daily"
62+
commands:
63+
- "julia --project -e 'using Pkg; Pkg.test()'"
64+
agents:
65+
slurm_mem: 32G
66+
slurm_cpus_per_task: 8
67+
slurm_ntasks: 1
68+
4769
- label: "CPU ECCO4/EN4 tests"
4870
key: "cpu_ecco4_en4_tests"
4971
env:
@@ -91,6 +113,34 @@ steps:
91113
slurm_ntasks: 1
92114
slurm_gpus_per_task: 1
93115

116+
- label: "GPU ECCO2Monthly tests"
117+
key: "gpu_ecco2_monthly_tests"
118+
env:
119+
TEST_GROUP: "ecco2_monthly"
120+
GPU_TEST: "true"
121+
commands:
122+
- "julia --project -e 'using Pkg; Pkg.test()'"
123+
agents:
124+
slurm_mem: 8G
125+
slurm_gpus: 1
126+
slurm_cpus_per_task: 8
127+
slurm_ntasks: 1
128+
slurm_gpus_per_task: 1
129+
130+
- label: "GPU ECCO2Daily tests"
131+
key: "gpu_ecco2_daily_tests"
132+
env:
133+
TEST_GROUP: "ecco2_daily"
134+
GPU_TEST: "true"
135+
commands:
136+
- "julia --project -e 'using Pkg; Pkg.test()'"
137+
agents:
138+
slurm_mem: 8G
139+
slurm_gpus: 1
140+
slurm_cpus_per_task: 8
141+
slurm_ntasks: 1
142+
slurm_gpus_per_task: 1
143+
94144
- label: "GPU ECCO4/EN4 tests"
95145
key: "gpu_ecco4_en4_tests"
96146
env:

‎Project.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name = "ClimaOcean"
22
uuid = "0376089a-ecfe-4b0e-a64f-9c555d74d754"
33
license = "MIT"
44
authors = ["Climate Modeling Alliance and contributors"]
5-
version = "0.6.4"
5+
version = "0.6.5"
66

77
[deps]
88
Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"

‎src/Bathymetry.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,10 +241,10 @@ function interpolate_bathymetry_in_passes(native_z, target_grid;
241241
"Grid info:", '\n',
242242
"target grid", '\n',
243243
"├── minimum x-spacings ", @sprintf("%.3e", resxt), " m", '\n',
244-
"└── minimum x-spacings ", @sprintf("%.3e", resyt), " m", '\n',
244+
"└── minimum y-spacings ", @sprintf("%.3e", resyt), " m", '\n',
245245
"bathymetry grid", '\n',
246246
"├── minimum x-spacings ", @sprintf("%.3e", resxn), " m", '\n',
247-
"└── minimum x-spacings ", @sprintf("%.3e", resyn), " m")
247+
"└── minimum y-spacings ", @sprintf("%.3e", resyn), " m")
248248
end
249249
return target_z
250250
end

‎src/DataWrangling/ECCO/ECCO_mask.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ using ClimaOcean.DataWrangling: dataset_mask, dataset_immersed_grid
44
import ClimaOcean.DataWrangling: default_set_dataset_mask
55

66
# Defaults
7-
ECCO_mask(arch::AbstractArchitecture=CPU()) = dataset_mask(Metadata(:temperature, dataset=ECCO4Monthly()), arch)
8-
ECCO_immersed_grid(arch::AbstractArchitecture=CPU()) = dataset_immersed_grid(Metadata(:temperature, dataset=ECCO4Monthly()), arch)
7+
ECCO_mask(arch::AbstractArchitecture=CPU()) = dataset_mask(Metadatum(:temperature, dataset=ECCO4Monthly()), arch)
8+
ECCO_immersed_grid(arch::AbstractArchitecture=CPU()) = dataset_immersed_grid(Metadatum(:temperature, dataset=ECCO4Monthly()), arch)
99

1010
default_set_dataset_mask(metadata::Metadata{<:ECCO4Monthly}) = ClimaOcean.DataWrangling.ECCO._set_ECCO4_mask!
1111
default_set_dataset_mask(metadata::Metadata{<:Union{ECCO2Monthly, ECCO2Daily}}) = ClimaOcean.DataWrangling.ECCO._set_ECCO2_mask!

‎src/DataWrangling/ECCO/ECCO_metadata.jl‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,20 @@ function ECCOMetadatum(name;
3737
return Metadatum(name; date, dir, dataset=ECCO4Monthly())
3838
end
3939

40-
default_download_directory(::Union{<:ECCO2Monthly, <:ECCO2Daily, <:ECCO4Monthly}) = download_ECCO_cache
40+
function default_download_directory(::ECCO2Monthly)
41+
path = joinpath(download_ECCO_cache, "v2", "monthly")
42+
return mkpath(path)
43+
end
44+
45+
function default_download_directory(::ECCO2Daily)
46+
path = joinpath(download_ECCO_cache, "v2", "daily")
47+
return mkpath(path)
48+
end
49+
50+
function default_download_directory(::ECCO4Monthly)
51+
path = joinpath(download_ECCO_cache, "v4")
52+
return mkpath(path)
53+
end
4154

4255
datasetstr(md::ECCOMetadata) = string(md.dataset)
4356

@@ -56,32 +69,32 @@ Base.size(::Metadatum{<:ECCO2Monthly}) = (1440, 720, 50, 1)
5669
Base.size(::Metadatum{<:ECCO4Monthly}) = (720, 360, 50, 1)
5770

5871
# The whole range of dates in the different dataset datasets
59-
all_dates(::ECCO4Monthly, name) = DateTime(1992, 1, 1) : Month(1) : DateTime(2023, 12, 1)
60-
all_dates(::ECCO2Monthly, name) = DateTime(1992, 1, 1) : Month(1) : DateTime(2023, 12, 1)
61-
all_dates(::ECCO2Daily, name) = DateTime(1992, 1, 4) : Day(1) : DateTime(2023, 12, 31)
72+
all_dates(::ECCO4Monthly, name) = DateTime(1992, 1, 1) : Month(1) : DateTime(2017, 12, 1)
73+
all_dates(::ECCO2Monthly, name) = DateTime(1992, 1, 1) : Month(1) : DateTime(2024, 12, 1)
74+
all_dates(::ECCO2Daily, name) = DateTime(1992, 1, 1) : Day(1) : DateTime(2024, 12, 31)
6275

6376
# Fallback, actually, we do not really need the name for ECCO since all
6477
# variables have the same frequency and the same time-range, differently from JRA55
6578
all_dates(dataset::Union{<:ECCO4Monthly, <:ECCO2Monthly, <:ECCO2Daily}) = all_dates(dataset, :temperature)
6679

67-
# File name generation specific to each Dataset dataset
80+
# File name generation specific to each dataset
6881
function metadata_filename(metadata::Metadatum{<:ECCO4Monthly})
6982
shortname = short_name(metadata)
70-
yearstr = string(Dates.year(metadata.dates))
71-
monthstr = string(Dates.month(metadata.dates), pad=2)
83+
yearstr = string(Dates.year(metadata.dates))
84+
monthstr = string(Dates.month(metadata.dates), pad=2)
7285
return shortname * "_" * yearstr * "_" * monthstr * ".nc"
7386
end
7487

7588
function metadata_filename(metadata::Metadatum{<:Union{ECCO2Daily, ECCO2Monthly}})
76-
shortname = short_name(metadata)
77-
yearstr = string(Dates.year(metadata.dates))
78-
monthstr = string(Dates.month(metadata.dates), pad=2)
79-
postfix = variable_is_three_dimensional(metadata) ? ".1440x720x50." : ".1440x720."
89+
shortname = short_name(metadata)
90+
yearstr = string(Dates.year(metadata.dates))
91+
monthstr = string(Dates.month(metadata.dates), pad=2)
92+
postfix = variable_is_three_dimensional(metadata) ? ".1440x720x50." : ".1440x720."
8093

8194
if metadata.dataset isa ECCO2Monthly
8295
return shortname * postfix * yearstr * monthstr * ".nc"
8396
elseif metadata.dataset isa ECCO2Daily
84-
daystr = string(Dates.day(metadata.dates), pad=2)
97+
daystr = variable_is_three_dimensional(metadata) ? string(Dates.day(metadata.dates), pad=2) : ""
8598
return shortname * postfix * yearstr * monthstr * daystr * ".nc"
8699
end
87100
end

‎src/DataWrangling/EN4/EN4_mask.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ using ClimaOcean.DataWrangling: dataset_mask, dataset_immersed_grid
44
import ClimaOcean.DataWrangling: default_set_dataset_mask
55

66
# Defaults
7-
EN4_mask(arch::AbstractArchitecture=CPU()) = dataset_mask(Metadata(:temperature, dataset=EN4Monthly()), arch)
8-
EN4_immersed_grid(arch::AbstractArchitecture=CPU()) = dataset_immersed_grid(Metadata(:temperature, dataset=EN4Monthly()), arch)
7+
EN4_mask(arch::AbstractArchitecture=CPU()) = dataset_mask(Metadatum(:temperature, dataset=EN4Monthly()), arch)
8+
EN4_immersed_grid(arch::AbstractArchitecture=CPU()) = dataset_immersed_grid(Metadatum(:temperature, dataset=EN4Monthly()), arch)
99

1010
default_set_dataset_mask(metadata::Metadata{<:EN4Monthly}) = ClimaOcean.DataWrangling.EN4._set_EN4_mask!
1111

‎src/DataWrangling/dataset.jl‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,16 @@ end
4242
"""
4343
Field(metadata::Metadatum;
4444
architecture = CPU(),
45-
inpainting = nothing,
45+
inpainting = default_inpainting(metadata),
4646
mask = nothing,
4747
horizontal_halo = (7, 7),
48-
cache_inpainted_data = false)
48+
cache_inpainted_data = true)
4949
5050
Return a `Field` on `architecture` described by `metadata` with `horizontal_halo` size.
5151
If not `nothing`, the `inpainting` method is used to fill the cells
5252
within the specified `mask`. `mask` is set to `dataset_mask` for non-nothing
53-
`inpainting`.
53+
`inpainting`. Keyword argument `cache_inpainted_data` dictates whether the inpainted
54+
data is cached to avoid recomputing it; default: `true`.
5455
"""
5556
function Field(metadata::Metadatum;
5657
architecture = CPU(),

‎src/DataWrangling/masking.jl‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
using Oceananigans
22
using Oceananigans.BoundaryConditions
3-
using Oceananigans.Fields: OneField
4-
using Oceananigans.Grids: peripheral_node
5-
using Oceananigans.Utils: launch!
6-
using Oceananigans.Fields: instantiated_location, interior, CenterField
73
using Oceananigans.Architectures: architecture, device, GPU
8-
using ClimaOcean.DataWrangling: Field
4+
using Oceananigans.Fields: Field, OneField, instantiated_location, interior, CenterField
5+
using Oceananigans.Grids: peripheral_node, znode
6+
using Oceananigans.Utils: launch!
97
using KernelAbstractions: @kernel, @index
108

119
import ClimaOcean: stateindex
@@ -195,7 +193,7 @@ end
195193
end
196194

197195
"""
198-
inpaint_mask!(field, mask; max_iter = Inf)
196+
inpaint_mask!(field, mask; inpainting=NearestNeighborInpainting(Inf))
199197
200198
Inpaint `field` within `mask`, using values outside `mask`.
201199
In other words, regions where `mask[i, j, k] == 1` is inpainted
@@ -210,6 +208,7 @@ Arguments
210208
- `inpainting`: The inpainting algorithm to use. The only option is
211209
`NearestNeighborInpainting(maxiter)`, where an average
212210
of the valid surrounding values is used `maxiter` times.
211+
Default: `NearestNeighborInpainting(Inf)`.
213212
"""
214213
function inpaint_mask!(field, mask; inpainting=NearestNeighborInpainting(Inf))
215214

@@ -231,19 +230,19 @@ end
231230
function default_set_dataset_mask end
232231

233232
"""
234-
dataset_mask(metadata, architecture = CPU();
233+
dataset_mask(metadata::Metadatum, architecture = CPU();
235234
data_field = Field(metadata; architecture, inpainting=nothing),
236235
minimum_value = Float32(-1e5),
237236
maximum_value = Float32(1e5))
238237
239238
A boolean field where `true` represents a missing value in the dataset.
240239
"""
241-
function dataset_mask(metadata, architecture = CPU();
240+
function dataset_mask(metadata::Metadatum, architecture = CPU();
242241
data_field = Field(metadata; architecture, inpainting=nothing),
243242
minimum_value = Float32(-1e5),
244243
maximum_value = Float32(1e5))
245244

246-
mask = Field{location(data_field)...}(data_field.grid, Bool)
245+
mask = Field{location(data_field)...}(data_field.grid, Bool)
247246

248247
_set_mask! = default_set_dataset_mask(metadata)
249248

@@ -261,7 +260,7 @@ by the first non-missing value from the bottom up.
261260
"""
262261
function dataset_immersed_grid(metadata, architecture = CPU())
263262

264-
mask = dataset_mask(metadata, architecture)
263+
mask = dataset_mask(first(metadata), architecture)
265264
grid = mask.grid
266265
bottom = Field{Center, Center, Nothing}(grid)
267266

‎src/DataWrangling/metadata.jl‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ function Metadata(variable_name;
4545
end_date = nothing)
4646

4747
if !isnothing(start_date) && !isnothing(end_date)
48-
@info "Slicing date range within $start_date and $end_date"
4948
dates = compute_native_date_range(dates, start_date, end_date)
5049
end
5150

@@ -58,15 +57,15 @@ const Metadatum{V} = Metadata{V, <:AnyDateTime} where V
5857
"""
5958
Metadatum(variable_name;
6059
dataset,
61-
date=first_date(dataset, variable_name),
62-
dir=default_download_directory(dataset))
60+
date = first_date(dataset, variable_name),
61+
dir = default_download_directory(dataset))
6362
6463
A specialized constructor for a [`Metadata`](@ref) object with a single date, representative of a snapshot in time.
6564
"""
6665
function Metadatum(variable_name;
6766
dataset,
68-
date=first_date(dataset, variable_name),
69-
dir=default_download_directory(dataset))
67+
date = first_date(dataset, variable_name),
68+
dir = default_download_directory(dataset))
7069

7170
date isa Union{CFTime.AbstractCFDateTime, Dates.AbstractDateTime} ||
7271
throw(ArgumentError("date must be Union{Dates.AbstractDateTime, CFTime.AbstractCFDateTime}"))
@@ -121,12 +120,15 @@ function short_name end
121120
"""
122121
native_times(metadata; start_time=first(metadata).dates)
123122
124-
Extract the time values from the given `metadata` and calculate the time difference
125-
from the `start_time` and return as an array of time differences in seconds.
123+
Extract the time values from the given `metadata`, calculate the time difference
124+
from the `start_time`, and return an array of time differences in seconds.
126125
127-
Arguments
128-
=========
126+
Argument
127+
========
129128
- `metadata`: The metadata containing the date information.
129+
130+
Keyword Argument
131+
================
130132
- `start_time`: The start time for calculating the time difference. Defaults to the first date in the metadata.
131133
"""
132134
function native_times(metadata; start_time=first(metadata).dates)

‎test/runtests.jl‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,12 @@ if test_group == :init || test_group == :all
2929
#### Download Dataset data
3030
####
3131

32-
# Metadata for tests
33-
32+
# Download few datasets for tests
3433
for dataset in test_datasets
34+
time_resolution = dataset isa ECCO2Daily ? Day(1) : Month(1)
35+
end_date = start_date + 2 * time_resolution
36+
dates = start_date:time_resolution:end_date
37+
3538
temperature_metadata = Metadata(:temperature; dataset, dates)
3639
salinity_metadata = Metadata(:salinity; dataset, dates)
3740

@@ -45,6 +48,14 @@ if test_group == :JRA55 || test_group == :all
4548
include("test_jra55.jl")
4649
end
4750

51+
if test_group == :ecco2_monthly || test_group == :all
52+
include("test_ecco2_monthly.jl")
53+
end
54+
55+
if test_group == :ecco2_daily || test_group == :all
56+
include("test_ecco2_daily.jl")
57+
end
58+
4859
if test_group == :ecco4_en4 || test_group == :all
4960
include("test_ecco4_en4.jl")
5061
end

0 commit comments

Comments
 (0)