JuliaClimate · juliohm · Jan 7, 2025 · Jan 3, 2025 · Jan 7, 2025 · Jan 7, 2025
diff --git a/Project.toml b/Project.toml
@@ -1,10 +1,9 @@
 name = "CDSAPI"
 uuid = "8a7b9de3-9c00-473e-88b4-7eccd7ef2fea"
 authors = ["Micky Yun Chan <[email protected]> and contributors"]
-version = "1.0.1"
+version = "2.0.0"
 
 [deps]
-Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 
@@ -15,11 +14,9 @@ julia = "1.3"
 
 [extras]
 GRIB = "b16dfd50-4035-11e9-28d4-9dfe17e6779b"
-GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
 NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9"
-Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
 
 [targets]
-test = ["Test", "GRIB", "ZipFile", "NetCDF", "Tar", "GZip"]
+test = ["Test", "GRIB", "ZipFile", "NetCDF"]
diff --git a/src/CDSAPI.jl b/src/CDSAPI.jl
@@ -2,57 +2,63 @@ module CDSAPI
 
 using HTTP
 using JSON
-using Base64
 
 """
-    retrieve(name, params, filename; max_sleep = 120.)
+    retrieve(name, params, filename; wait=1.0)
 
 Retrieves data for `name` from the Climate Data Store
 with the specified `params` and stores it in the current
 directory as `filename`.
 
 The client periodically requests the status of the retrieve request.
-`max_sleep` is the maximum time (in seconds) between the status updates.
+`wait` is the maximum time (in seconds) between status updates.
 """
-function retrieve(name, params, filename; max_sleep = 120.)
+function retrieve(name, params, filename; wait=1.0)
     creds = Dict()
-    open(joinpath(homedir(),".cdsapirc")) do f
+    open(joinpath(homedir(), ".cdsapirc")) do f
         for line in readlines(f)
-            key, val = strip.(split(line,':', limit=2))
+            key, val = strip.(split(line, ':', limit=2))
             creds[key] = val
         end
     end
 
-    apikey = string("Basic ", base64encode(creds["key"]))
-    response = HTTP.request(
-        "POST",
-        creds["url"] * "/resources/$name",
-        ["Authorization" => apikey],
-        body=JSON.json(params),
-        verbose=1)
-
-    resp_dict = JSON.parse(String(response.body))
-    data = Dict("state" => "queued")
-    sleep_seconds = 1.
-
-    while data["state"] != "completed"
-        data = HTTP.request("GET", creds["url"] * "/tasks/" * string(resp_dict["request_id"]),  ["Authorization" => apikey])
+    response = HTTP.request("POST",
+        creds["url"] * "/retrieve/v1/processes/$name/execute",
+        ["PRIVATE-TOKEN" => creds["key"]],
+        body=JSON.json(Dict("inputs" => params))
+    )
+    body = JSON.parse(String(response.body))
+    data = Dict("status" => "queued")
+
+    while data["status"] != "successful"
+        data = HTTP.request("GET",
+            creds["url"] * "/retrieve/v1/jobs/" * string(body["jobID"]),
+            ["PRIVATE-TOKEN" => creds["key"]]
+        )
         data = JSON.parse(String(data.body))
-        println("request queue status ", data["state"])
-
-        if data["state"] == "failed"
-            error("Request to dataset $name failed. Check " *
-                  "https://cds.climate.copernicus.eu/cdsapp#!/yourrequests " *
-                  "for more information (after login).")
+        @info "request status" data["status"]
+
+        if data["status"] == "failed"
+            throw(ErrorException("""
+            Request to dataset $name failed.
+            Check https://cds.climate.copernicus.eu/requests
+            for more information (after login).
+            """
+            ))
         end
 
-        sleep_seconds = min(1.5 * sleep_seconds,max_sleep)
-        if data["state"] != "completed"
-            sleep(sleep_seconds)
+        if data["status"] != "successful"
+            sleep(wait)
         end
     end
 
-    HTTP.download(data["location"], filename)
+    response = HTTP.request("GET",
+        creds["url"] * "/retrieve/v1/jobs/" * string(body["jobID"]) * "/results",
+        ["PRIVATE-TOKEN" => creds["key"]]
+    )
+    body = JSON.parse(String(response.body))
+    HTTP.download(body["asset"]["value"]["href"], filename)
+
     return data
 end
 
@@ -88,7 +94,7 @@ function py2ju(dictstr)
     # if there's no pair after the last comma
     if findnext(":", dictstr_cpy, lastcomma_pos) == nothing
         # remove the comma
-        dictstr_cpy = dictstr_cpy[firstindex(dictstr_cpy):(lastcomma_pos - 1)] * dictstr_cpy[(lastcomma_pos + 1):lastindex(dictstr_cpy)]
+        dictstr_cpy = dictstr_cpy[firstindex(dictstr_cpy):(lastcomma_pos-1)] * dictstr_cpy[(lastcomma_pos+1):lastindex(dictstr_cpy)]
     end
 
     # removes trailing comma from a list

diff --git a/test/py2ju.jl b/test/py2ju.jl
@@ -1,6 +1,6 @@
 @testset "Py2Ju" begin
     pydict_str = """{
-                'format': 'grib',
+                'data_format': 'grib',
                 'product_type': 'monthly_averaged_reanalysis',
                 'variable': 'divergence',
                 'pressure_level': '1',
@@ -12,14 +12,14 @@
                 ],
                 'time': '00:00',
             }"""
-    julia_dict = Dict("format"=> "grib",
-                    "month" => "06",
-                    "time" => "00:00",
-                    "year" => "2020",
-                    "pressure_level" => "1",
-                    "area" => Any[90, -180, -90, 180],
-                    "product_type" => "monthly_averaged_reanalysis",
-                    "variable" => "divergence")
+    julia_dict = Dict("data_format" => "grib",
+        "month" => "06",
+        "time" => "00:00",
+        "year" => "2020",
+        "pressure_level" => "1",
+        "area" => Any[90, -180, -90, 180],
+        "product_type" => "monthly_averaged_reanalysis",
+        "variable" => "divergence")
     py2ju_result = CDSAPI.py2ju(pydict_str)
 
     @test typeof(py2ju_result) <: Dict

diff --git a/test/retrieve.jl b/test/retrieve.jl
@@ -1,11 +1,11 @@
 @testset "Retrieve" begin
-    datadir = joinpath(@__DIR__,"data")
+    datadir = joinpath(@__DIR__, "data")
 
     @testset "ERA5 monthly preasure data" begin
         filepath = joinpath(datadir, "era5.grib")
         response = CDSAPI.retrieve("reanalysis-era5-pressure-levels-monthly-means",
             CDSAPI.py2ju("""{
-                'format': 'grib',
+                'data_format': 'grib',
                 'product_type': 'monthly_averaged_reanalysis',
                 'variable': 'divergence',
                 'pressure_level': '1',
@@ -20,7 +20,6 @@
             filepath)
 
         @test typeof(response) <: Dict
-        @test response["content_type"] == "application/x-grib"
         @test isfile(filepath)
 
         GribFile(filepath) do datafile
@@ -33,22 +32,22 @@
         rm(filepath)
     end
 
-    @testset "Europe water quantity data" begin
-        filepath = joinpath(datadir, "ewq.zip")
-        response = CDSAPI.retrieve("sis-water-quantity-swicca",
+    @testset "Sea ice type data" begin
+        filepath = joinpath(datadir, "sea_ice_type.zip")
+        response = CDSAPI.retrieve("satellite-sea-ice-edge-type",
             CDSAPI.py2ju("""{
-                'variable': 'river_flow',
-                'time_aggregation': 'annual_maximum',
-                'horizontal_aggregation': 'catchments',
-                'emissions_scenario': 'rcp_2_6',
-                'period': '2071_2100',
-                'return_period': '100',
-                'format': 'zip',
+                'variable': 'sea_ice_type',
+                'region': 'northern_hemisphere',
+                'cdr_type': 'cdr',
+                'year': '1979',
+                'month': '01',
+                'day': '02',
+                'version': '3_0',
+                'data_format': 'zip',
             }"""),
             filepath)
 
         @test typeof(response) <: Dict
-        @test response["content_type"] == "application/zip"
         @test isfile(filepath)
 
         # extract contents
@@ -59,44 +58,11 @@
         close(zip_reader)
 
         # test file contents
-        @test ncgetatt(ewq_file, "Global", "time_coverage_start") == "20710101"
-        @test ncgetatt(ewq_file, "Global", "time_coverage_end") == "21001231"
-        @test ncgetatt(ewq_file, "Global", "invar_experiment_name") == "rcp26"
+        @test ncgetatt(ewq_file, "Global", "time_coverage_start") == "19790102T000000Z"
+        @test ncgetatt(ewq_file, "Global", "time_coverage_end") == "19790103T000000Z"
 
         # cleanup
         rm(filepath)
         rm(ewq_file)
     end
-
-    @testset "European energy sector cimate" begin
-        filepath = joinpath(datadir, "ees.tar.gz")
-        response = CDSAPI.retrieve("sis-european-energy-sector",
-            CDSAPI.py2ju("""{
-                'variable': 'precipitation',
-                'time_aggregation': '1_year_average',
-                'vertical_level': '0_m',
-                'bias_correction': 'bias_adjustment_based_on_gamma_distribution',
-                'format': 'tgz',
-            }"""),
-            filepath)
-
-        @test typeof(response) <: Dict
-        @test response["content_type"] == "application/gzip"
-        @test isfile(filepath)
-
-        # extract contents
-        gzip_io = GZip.open(filepath)
-        eesfile_dir = Tar.extract(gzip_io, joinpath(datadir, "ees"))
-        ees_file = joinpath(eesfile_dir, readdir(eesfile_dir)[1])
-        close(gzip_io)
-
-        # test file contents
-        @test ncgetatt(ees_file, "Global", "frequency") == "year"
-        @test ncgetatt(ees_file, "tp", "long_name") == "precip total"
-
-        # cleanup
-        rm(filepath)
-        rm(ees_file)
-        rm(eesfile_dir)
-    end
 end
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,5 +1,5 @@
 using CDSAPI
-using ZipFile, GZip, Tar
+using ZipFile
 using GRIB, NetCDF
 
 using Test