diff --git a/README.md b/README.md index 427dda9..b7d514c 100644 --- a/README.md +++ b/README.md @@ -26,37 +26,51 @@ Suppose that the `Show API request` button generated the following Python code: ```python #!/usr/bin/env python import cdsapi -c = cdsapi.Client() -c.retrieve("insitu-glaciers-elevation-mass", -{ -"variable": "all", -"product_type": "elevation_change", -"file_version": "20170405", -"format": "tgz" -}, -"download.tar.gz") + +dataset = "reanalysis-era5-single-levels" +request = { + "product_type": ["reanalysis"], + "variable": [ + "10m_u_component_of_wind", + "10m_v_component_of_wind" + ], + "year": ["2024"], + "month": ["12"], + "day": ["06"], + "time": ["16:00"], + "data_format": "netcdf", + "download_format": "unarchived", + "area": [58, 6, 55, 9] +} + +client = cdsapi.Client() +client.retrieve(dataset, request).download() ``` -You can obtain the same results in Julia with the following code: +You can obtain the same results in Julia: ```julia using CDSAPI -CDSAPI.retrieve("insitu-glaciers-elevation-mass", -CDSAPI.py2ju(""" -{ -"variable": "all", -"product_type": "elevation_change", -"file_version": "20170405", -"format": "tgz" -} -"""), -"download.tar.gz") +dataset = "reanalysis-era5-single-levels" +request = """{ + "product_type": ["reanalysis"], + "variable": [ + "10m_u_component_of_wind", + "10m_v_component_of_wind" + ], + "year": ["2024"], + "month": ["12"], + "day": ["06"], + "time": ["16:00"], + "data_format": "netcdf", + "download_format": "unarchived", + "area": [58, 6, 55, 9] +}""" # <- notice the multiline string. + +CDSAPI.retrieve(dataset, request, "download.nc") ``` -We've copied/pasted the code and called the `py2ju` function on the second argument of the `retrieve` function. -The `py2ju` function simply converts the string containing a Python dictionary to an actual Julia dictionary. - Besides the downloaded file, the `retrieve` function also returns a dictionary with metadata: ``` diff --git a/src/CDSAPI.jl b/src/CDSAPI.jl index ea97b39..9f84818 100644 --- a/src/CDSAPI.jl +++ b/src/CDSAPI.jl @@ -6,14 +6,19 @@ using JSON """ retrieve(name, params, filename; wait=1.0) -Retrieves data for `name` from the Climate Data Store -with the specified `params` and stores it in the current -directory as `filename`. +Retrieves dataset with given `name` from the Climate Data Store +with the specified `params` (JSON string) and stores it in the +given `filename`. -The client periodically requests the status of the retrieve request. -`wait` is the maximum time (in seconds) between status updates. +The client periodically checks the status of the request and one +can specify the maximum time in seconds to `wait` between updates. """ -function retrieve(name, params, filename; wait=1.0) +retrieve(name, params::AbstractString, filename; wait=1.0) = + retrieve(name, JSON.parse(params), filename; wait) + +# CDSAPI.parse can be used to convert the request params into a +# Julia dictionary for additional manipulation before retrieval +function retrieve(name, params::AbstractDict, filename; wait=1.0) creds = Dict() open(joinpath(homedir(), ".cdsapirc")) do f for line in readlines(f) @@ -44,12 +49,11 @@ function retrieve(name, params, filename; wait=1.0) throw(e) end - body = JSON.parse(String(response.body)) - data = Dict("status" => "queued") + data = JSON.parse(String(response.body)) + endpoint = Dict(response.headers)["location"] while data["status"] != "successful" - data = HTTP.request("GET", - creds["url"] * "/retrieve/v1/jobs/" * string(body["jobID"]), + data = HTTP.request("GET", endpoint, ["PRIVATE-TOKEN" => creds["key"]] ) data = JSON.parse(String(data.body)) @@ -70,7 +74,7 @@ function retrieve(name, params, filename; wait=1.0) end response = HTTP.request("GET", - creds["url"] * "/retrieve/v1/jobs/" * string(body["jobID"]) * "/results", + endpoint * "/results", ["PRIVATE-TOKEN" => creds["key"]] ) body = JSON.parse(String(response.body)) @@ -80,45 +84,12 @@ function retrieve(name, params, filename; wait=1.0) end """ - py2ju(dictstr) - -Takes a Python dictionary as string and converts it into Julia's `Dict` - -# Examples -```julia-repl -julia> str = \"""{ - 'format': 'zip', - 'variable': 'surface_air_temperature', - 'product_type': 'climatology', - 'month': '08', - 'origin': 'era_interim', - }\"""; - -julia> CDSAPI.py2ju(str) -Dict{String,Any} with 5 entries: - "format" => "zip" - "month" => "08" - "product_type" => "climatology" - "variable" => "surface_air_temperature" - "origin" => "era_interim" - -``` -""" -function py2ju(dictstr) - dictstr_cpy = replace(dictstr, "'" => "\"") - lastcomma_pos = findlast(",", dictstr_cpy).start - - # if there's no pair after the last comma - if findnext(":", dictstr_cpy, lastcomma_pos) == nothing - # remove the comma - dictstr_cpy = dictstr_cpy[firstindex(dictstr_cpy):(lastcomma_pos-1)] * dictstr_cpy[(lastcomma_pos+1):lastindex(dictstr_cpy)] - end + parse(string) - # removes trailing comma from a list - rx = r",[ \n\r\t]*\]" - dictstr_cpy = replace(dictstr_cpy, rx => "]") +Equivalent to `JSON.parse(string)`. +""" +parse(string) = JSON.parse(string) - return JSON.parse(dictstr_cpy) -end +@deprecate py2ju(string) parse(string) end # module diff --git a/test/py2ju.jl b/test/py2ju.jl deleted file mode 100644 index a8b987c..0000000 --- a/test/py2ju.jl +++ /dev/null @@ -1,27 +0,0 @@ -@testset "Py2Ju" begin - pydict_str = """{ - 'data_format': 'grib', - 'product_type': 'monthly_averaged_reanalysis', - 'variable': 'divergence', - 'pressure_level': '1', - 'year': '2020', - 'month': '06', - 'area': [ - 90, -180, -90, - 180, - ], - 'time': '00:00', - }""" - julia_dict = Dict("data_format" => "grib", - "month" => "06", - "time" => "00:00", - "year" => "2020", - "pressure_level" => "1", - "area" => Any[90, -180, -90, 180], - "product_type" => "monthly_averaged_reanalysis", - "variable" => "divergence") - py2ju_result = CDSAPI.py2ju(pydict_str) - - @test typeof(py2ju_result) <: Dict - @test py2ju_result == julia_dict -end diff --git a/test/retrieve.jl b/test/retrieve.jl deleted file mode 100644 index fc26799..0000000 --- a/test/retrieve.jl +++ /dev/null @@ -1,81 +0,0 @@ -@testset "Retrieve" begin - datadir = joinpath(@__DIR__, "data") - - @testset "ERA5 monthly preasure data" begin - filepath = joinpath(datadir, "era5.grib") - response = CDSAPI.retrieve("reanalysis-era5-pressure-levels-monthly-means", - CDSAPI.py2ju("""{ - 'data_format': 'grib', - 'product_type': 'monthly_averaged_reanalysis', - 'variable': 'divergence', - 'pressure_level': '1', - 'year': '2020', - 'month': '06', - 'area': [ - 90, -180, -90, - 180, - ], - 'time': '00:00', - }"""), - filepath) - - @test typeof(response) <: Dict - @test isfile(filepath) - - GribFile(filepath) do datafile - data = Message(datafile) - @test data["name"] == "Divergence" - @test data["level"] == 1 - @test data["year"] == 2020 - @test data["month"] == 6 - end - rm(filepath) - end - - @testset "Sea ice type data" begin - filepath = joinpath(datadir, "sea_ice_type.zip") - response = CDSAPI.retrieve("satellite-sea-ice-edge-type", - CDSAPI.py2ju("""{ - 'variable': 'sea_ice_type', - 'region': 'northern_hemisphere', - 'cdr_type': 'cdr', - 'year': '1979', - 'month': '01', - 'day': '02', - 'version': '3_0', - 'data_format': 'zip', - }"""), - filepath) - - @test typeof(response) <: Dict - @test isfile(filepath) - - # extract contents - zip_reader = ZipFile.Reader(filepath) - ewq_fileio = zip_reader.files[1] - ewq_file = joinpath(datadir, ewq_fileio.name) - write(ewq_file, read(ewq_fileio)) - close(zip_reader) - - # test file contents - @test ncgetatt(ewq_file, "Global", "time_coverage_start") == "19790102T000000Z" - @test ncgetatt(ewq_file, "Global", "time_coverage_end") == "19790103T000000Z" - - # cleanup - rm(filepath) - rm(ewq_file) - end - - @testset "Bad requests errors are catched" begin - goodname = "reanalysis-era5-single-levels" - badname = "bad-dataset" - badrequest = """{ - "this": "is", - "a": "bad", - "re": ["quest"] - }""" - - @test_throws ArgumentError CDSAPI.retrieve(goodname, badrequest, "unreachable") - @test_throws ArgumentError CDSAPI.retrieve(badname, badrequest, "unreachable") - end -end diff --git a/test/runtests.jl b/test/runtests.jl index 610582e..217f4a2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,14 +4,85 @@ using GRIB, NetCDF using Test -# list of tests -testfiles = [ - "py2ju.jl", - "retrieve.jl", -] +datadir = joinpath(@__DIR__, "data") @testset "CDSAPI.jl" begin - for testfile in testfiles - include(testfile) + @testset "ERA5 monthly preasure data" begin + filename = joinpath(datadir, "era5.grib") + response = CDSAPI.retrieve( + "reanalysis-era5-pressure-levels-monthly-means", + """{ + "data_format": "grib", + "product_type": "monthly_averaged_reanalysis", + "variable": "divergence", + "pressure_level": "1", + "year": "2020", + "month": "06", + "area": [90,-180,-90,180], + "time": "00:00" + }""", + filename + ) + + @test typeof(response) <: Dict + @test isfile(filename) + + GribFile(filename) do datafile + data = Message(datafile) + @test data["name"] == "Divergence" + @test data["level"] == 1 + @test data["year"] == 2020 + @test data["month"] == 6 + end + rm(filename) + end + + @testset "Sea ice type data" begin + filename = joinpath(datadir, "sea_ice_type.zip") + response = CDSAPI.retrieve( + "satellite-sea-ice-edge-type", + """{ + "variable": "sea_ice_type", + "region": "northern_hemisphere", + "cdr_type": "cdr", + "year": "1979", + "month": "01", + "day": "02", + "version": "3_0", + "data_format": "zip" + }""", + filename + ) + + @test typeof(response) <: Dict + @test isfile(filename) + + # extract contents + zip_reader = ZipFile.Reader(filename) + ewq_fileio = zip_reader.files[1] + ewq_file = joinpath(datadir, ewq_fileio.name) + write(ewq_file, read(ewq_fileio)) + close(zip_reader) + + # test file contents + @test ncgetatt(ewq_file, "Global", "time_coverage_start") == "19790102T000000Z" + @test ncgetatt(ewq_file, "Global", "time_coverage_end") == "19790103T000000Z" + + # cleanup + rm(filename) + rm(ewq_file) + end + + @testset "Bad requests errors are catched" begin + goodname = "reanalysis-era5-single-levels" + badname = "bad-dataset" + badrequest = """{ + "this": "is", + "a": "bad", + "re": ["quest"] + }""" + + @test_throws ArgumentError CDSAPI.retrieve(goodname, badrequest, "unreachable") + @test_throws ArgumentError CDSAPI.retrieve(badname, badrequest, "unreachable") end end