Ensure consistent download_dataset for Copernicus data (#522)

* fix a couple of errors

* correction

* bugfix

* disambiguate

* make sure we download individual files

* comment

* comment

* cleanup unused grid

* add metadatum

---------

Co-authored-by: Navid C. Constantinou <[email protected]>

Author: simone-silvestri

ext/ClimaOceanPythonCallExt.jl

@@ -7,7 +7,7 @@ using Oceananigans
 using Oceananigans.DistributedComputations: @root
 
 using Dates: DateTime
-using ClimaOcean.DataWrangling.Copernicus: CopernicusMetadata
+using ClimaOcean.DataWrangling.Copernicus: CopernicusMetadata, CopernicusMetadatum
 
 import ClimaOcean.DataWrangling: download_dataset
 
@@ -25,7 +25,17 @@ function install_copernicusmarine()
     return cli
 end
 
-function download_dataset(meta::CopernicusMetadata, grid=nothing; skip_existing = true, additional_kw...)
+# Download each date individually, instead of downloading the entire dataset at once.
+# This is useful for a possible extension of the temporal horizon of the dataset.
+function download_dataset(metadata::CopernicusMetadata; kwargs...)
+    paths = Array{String}(undef, length(metadata))
+    for (m, metadatum) in enumerate(metadata)
+        paths[m] = download_dataset(metadatum; kwargs...)
+    end
+    return paths
+end
+
+function download_dataset(meta::CopernicusMetadatum; skip_existing = true, additional_kw...)
     output_directory = meta.dir
     output_filename = ClimaOcean.DataWrangling.metadata_filename(meta)
     output_path = joinpath(output_directory, output_filename)

src/DataWrangling/Copernicus/Copernicus.jl

@@ -6,7 +6,7 @@ using NCDatasets
 using Printf
 
 using Oceananigans.Fields: Center
-using ClimaOcean.DataWrangling: Metadata, Metadatum, metadata_path
+using ClimaOcean.DataWrangling: Metadata, Metadatum
 using Dates: DateTime, Day, Month
 
 import Oceananigans.Fields:
@@ -45,6 +45,8 @@ dataset_name(::GLORYSStatic) = "GLORYSStatic"
 dataset_name(::GLORYSDaily) = "GLORYSDaily"
 dataset_name(::GLORYSMonthly) = "GLORYSMonthly"
 
+Base.size(::CopernicusDataset, variable) = (4320, 2040, 50)
+
 all_dates(::GLORYSStatic, var) = [nothing]
 all_dates(::GLORYSDaily, var) = range(DateTime("1993-01-01"), stop=DateTime("2021-06-30"), step=Day(1))
 all_dates(::GLORYSMonthly, var) = range(DateTime("1993-01-01"), stop=DateTime("2024-12-01"), step=Month(1))
@@ -57,10 +59,11 @@ copernicusmarine_dataset_id(::GLORYSMonthly) = "cmems_mod_glo_phy_my_0.083deg_P1
 struct CMEMSHourlyAnalysis <: CopernicusDataset end
 copernicusmarine_dataset_id(::CMEMSHourlyAnalysis) = "cmems_mod_glo_phy_anfc_0.083deg_PT1H-m"
 
-CopernicusMetadata{D} = Metadata{<:CopernicusDataset, D}
-CopernicusMetadatum = Metadatum{<:CopernicusDataset}
+const CopernicusMetadata{D} = Metadata{<:CopernicusDataset, D}
+const CopernicusMetadatum = Metadatum{<:CopernicusDataset}
 
 Base.size(::CopernicusMetadatum) = (4320, 2040, 50, 1)
+
 reversed_vertical_axis(::CopernicusDataset) = true
 
 available_variables(::CopernicusDataset) = copernicus_dataset_variable_names