adds scoped values credential to support multiple endpoints. (#58)

* adds scoped values credential to support multiple endpoints.

* fix naming

Co-authored-by: Júlio Hoffimann <[email protected]>

* clearer check

Co-authored-by: Júlio Hoffimann <[email protected]>

* rename CDScredentials

Co-authored-by: Júlio Hoffimann <[email protected]>

* WIP: for now adding compatibility for 1.11 only.

* fix: better docs of credentials, moved the check inside the credentials.
and various renames

* better docstring

Co-authored-by: Júlio Hoffimann <[email protected]>

* add logic for ScopedValues compat

* fix wrong string interpolation of `$HOME`

* fix: logic error. auth[] is valid only if both entries are non empty.

* fix: even better logic. clearer.

* add ScopedValues as a direct Dependency

* fix: leftover

* added README entry about multiple token use with examples.

* export with from ScopedValues, so it is available when `using CDSAPI`

* added tests

* better ScopedValue versioning

Co-authored-by: Júlio Hoffimann <[email protected]>

* Documentation fixes

Co-authored-by: Júlio Hoffimann <[email protected]>

* don't export with

Co-authored-by: Júlio Hoffimann <[email protected]>

* added better explaination of the various priorities for default credentials

* split the auth scoped value into key and url.
Properly let the various cred inputs overwrite each other.

* update README with new syntax for scoped values

* clarified the readme and removed references to `credentialsfromfile`
renamed the scoped values to the uppercase variant

* Update README.md

Co-authored-by: Júlio Hoffimann <[email protected]>

* Update README.md

Co-authored-by: Júlio Hoffimann <[email protected]>

* default url as based, fix tests, fix leftover

* typos and whitespace

Co-authored-by: Júlio Hoffimann <[email protected]>

* revert default url in scoped value.
Enforce presence of key and url in file.

* Apply suggestions from code review

Co-authored-by: Júlio Hoffimann <[email protected]>

* Final adjustments

---------

Co-authored-by: Júlio Hoffimann <[email protected]>

Author: ghyatzo

Project.toml

@@ -7,11 +7,13 @@ version = "2.0.4"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+ScopedValues = "7e506255-f358-4e82-b7e4-beb19740aa63"
 
 [compat]
 Dates = "1.11"
 HTTP = "1"
 JSON = "0.21"
+ScopedValues = "1.3.0"
 julia = "1"
 
 [extras]

README.md

@@ -16,13 +16,26 @@ Please install the package with Julia's package manager:
 ] add CDSAPI
 ```
 
-## Usage
+## Basic usage
 
-Make sure your `~/.cdsapirc` file exists. Instructions on how to create the file for your user account can be found
+The package will attempt to use CDS credentials using three different methods with the following priority:
+
+    1. direct credentials provided through the scoped values `CDSAPI.KEY` and `CDSAPI.URL`
+    2. environmental variables `CDSAPI_URL` and `CDSAPI_KEY`
+    3. default credential file in home directory `~/.cdsapirc`
+
+A valid credential file is a text file with two lines:
+```
+url: https://yourendpoint
+key: your-personal-api-token
+```
+
+Instructions on how to create the file for your user account can be found
 [here](https://cds.climate.copernicus.eu/how-to-api).
 
-Suppose that the `Show API request` button generated the following Python code:
+For the following example to work, make sure your `~/.cdsapirc` file exists or the env vars `CDSAPI_URL` and `CDSAPI_KEY` are set.
 
+Suppose that the `Show API request` button generated the following Python code:
 ```python
 #!/usr/bin/env python
 import cdsapi
@@ -48,7 +61,6 @@ client.retrieve(dataset, request).download()
 ```
 
 You can obtain the same results in Julia:
-
 ```julia
 using CDSAPI
 
@@ -82,6 +94,28 @@ Dict{String,Any} with 6 entries:
   "content_length"     => 193660
   "state"              => "completed"
 ```
+# Multiple credentials
+
+In case you want to use multiple credentials for different requests, pass the desired values to the corresponding scoped values `CDSAPI.URL` and `CDSAPI.KEY`:
+```julia
+using CDSAPI
+
+dataset = "reanalysis-era5-single-levels"
+request = """ #= some request =# """
+
+customkey = "an-example-of-key"
+customurl = "http://my-custom-endpoint"
+
+# overwrite KEY and use URL from other methods
+CDSAPI.with(CDSAPI.KEY => customkey) do
+    CDSAPI.retrieve(dataset, request, "download.nc")
+end
+
+# overwrite URL and use KEY from other methods
+CDSAPI.with(CDSAPI.URL => customurl) do
+    CDSAPI.retrieve(dataset, request, "download.nc")
+end
+```
 
 [build-img]: https://img.shields.io/github/actions/workflow/status/JuliaClimate/CDSAPI.jl/CI.yml?branch=master&style=flat-square
 [build-url]: https://github.com/JuliaClimate/CDSAPI.jl/actions

src/CDSAPI.jl

@@ -4,6 +4,78 @@ using HTTP
 using JSON
 using Dates
 
+using ScopedValues
+
+const URL = ScopedValue("")
+const KEY = ScopedValue("")
+
+"""
+    credentials()
+
+Attempt to find CDS credentials using different methods:
+
+    1. direct credentials provided through the scoped values `KEY` and `URL`
+    2. environmental variables `CDSAPI_URL` and `CDSAPI_KEY`
+    3. credential file in home directory `~/.cdsapirc`
+
+A credential file is a text file with two lines:
+
+url: https://yourendpoint
+key: your-personal-api-token
+"""
+function credentials()
+    # attempt to retrieve url/key from dotfile
+    dotrc = joinpath(homedir(), ".cdsapirc")
+    if isfile(dotrc)
+        url, key = credentialsfromfile(dotrc)
+    else
+        url = key = ""
+    end
+
+    # overwrite with env values
+    url = get(ENV, "CDSAPI_URL", url)
+    key = get(ENV, "CDSAPI_KEY", key)
+
+    # overwrite with scoped values
+    url = isempty(URL[]) ? url : URL[]
+    key = isempty(KEY[]) ? key : KEY[]
+
+    if isempty(url) || isempty(key)
+        error("""
+        Missing credentials. Either add the CDSAPI_URL and CDSAPI_KEY env variables
+        or create a .cdsapirc file (default location: '$(homedir())').
+        """)
+    end
+
+    return url, key
+end
+
+"""
+    credentials(file)
+
+Parse the CDS credentials from a provided `file`.
+"""
+function credentialsfromfile(file)
+    creds = Dict()
+    open(realpath(file)) do f
+        for line in readlines(f)
+            key, val = strip.(split(line, ':', limit=2))
+            creds[key] = val
+        end
+    end
+
+    if !(haskey(creds, "url") && haskey(creds, "key"))
+        error("""
+        The credentials' file must have both a `url` value and a `key` value in the following format:
+
+        url: https://yourendpoint
+        key: your-personal-api-token
+        """)
+    end
+
+    return get(creds, "url", ""), get(creds, "key", "")
+end
+
 """
     retrieve(name, params, filename; wait=1.0)
 
@@ -20,18 +92,12 @@ retrieve(name, params::AbstractString, filename; wait=1.0) =
 # CDSAPI.parse can be used to convert the request params into a
 # Julia dictionary for additional manipulation before retrieval
 function retrieve(name, params::AbstractDict, filename; wait=1.0)
-    creds = Dict()
-    open(joinpath(homedir(), ".cdsapirc")) do f
-        for line in readlines(f)
-            key, val = strip.(split(line, ':', limit=2))
-            creds[key] = val
-        end
-    end
+    url, key = credentials()
 
     try
         response = HTTP.request("POST",
-            creds["url"] * "/retrieve/v1/processes/$name/execute",
-            ["PRIVATE-TOKEN" => creds["key"]],
+            url * "/retrieve/v1/processes/$name/execute",
+            ["PRIVATE-TOKEN" => key],
             body=JSON.json(Dict("inputs" => params))
         )
     catch e
@@ -56,7 +122,7 @@ function retrieve(name, params::AbstractDict, filename; wait=1.0)
     laststatus = nothing
     while data["status"] != "successful"
         data = HTTP.request("GET", endpoint,
-            ["PRIVATE-TOKEN" => creds["key"]]
+            ["PRIVATE-TOKEN" => key]
         )
         data = JSON.parse(String(data.body))
 
@@ -81,7 +147,7 @@ function retrieve(name, params::AbstractDict, filename; wait=1.0)
 
     response = HTTP.request("GET",
         endpoint * "/results",
-        ["PRIVATE-TOKEN" => creds["key"]]
+        ["PRIVATE-TOKEN" => key]
     )
     body = JSON.parse(String(response.body))
 

test/runtests.jl

@@ -85,4 +85,26 @@ datadir = joinpath(@__DIR__, "data")
         @test_throws ArgumentError CDSAPI.retrieve(goodname, badrequest, "unreachable")
         @test_throws ArgumentError CDSAPI.retrieve(badname, badrequest, "unreachable")
     end
+
+    @testset "Credentials with scoped values" begin
+        filename = joinpath(datadir, "sea_ice_type.zip")
+        dataset = "satellite-sea-ice-edge-type"
+        request = """{
+                "variable": "sea_ice_type",
+                "region": "northern_hemisphere",
+                "cdr_type": "cdr",
+                "year": "1979",
+                "month": "01",
+                "day": "02",
+                "version": "3_0",
+                "data_format": "zip"
+            }"""
+
+        url, key = CDSAPI.credentials() # grab the default ones
+        CDSAPI.with(CDSAPI.URL => url, CDSAPI.KEY => key) do
+            response = CDSAPI.retrieve(dataset, request, filename)
+        end
+        
+        rm(filename)
+    end
 end