Added retry capability for get APIs (#24)

* Added retry capability to some APIs (e.g. `get`)

Author: mdpradeep

Project.toml

@@ -12,6 +12,7 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Swagger = "2d69052b-6a58-5cd9-a030-a48559c324ac"
+Retry = "20febd7b-183b-5ae2-ac4a-720e7ce64774"
 
 [compat]
 HTTP = "0.8"

src/Kuber.jl

@@ -3,6 +3,7 @@ module Kuber
 using JSON
 using Swagger
 using HTTP
+using Retry, HTTP.IOExtras
 
 include("api/Kubernetes.jl")
 using .Kubernetes

src/helpers.jl

@@ -61,11 +61,11 @@ show(io::IO, ctx::KuberContext) = print("Kubernetes namespace ", ctx.namespace,
 get_server(ctx::KuberContext) = ctx.client.root
 get_ns(ctx::KuberContext) = ctx.namespace
 
-function set_server(ctx::KuberContext, uri::String=DEFAULT_URI, reset_api_versions::Bool=false; kwargs...)
+function set_server(ctx::KuberContext, uri::String=DEFAULT_URI, reset_api_versions::Bool=false; max_tries=1, kwargs...)
     rtfn = (default,data)->kuber_type(ctx, default, data)
     ctx.client = Swagger.Client(uri; get_return_type=rtfn, kwargs...)
     ctx.client.headers["Connection"] = "close"
-    reset_api_versions && set_api_versions!(ctx)
+    reset_api_versions && set_api_versions!(ctx; max_tries=max_tries)
     ctx.client
 end
 
@@ -113,9 +113,16 @@ function override_pref(name, server_pref, override)
     server_pref
 end
 
-function fetch_misc_apis_versions(ctx::KuberContext; override=nothing, verbose::Bool=false)
+function fetch_misc_apis_versions(ctx::KuberContext; override=nothing, verbose::Bool=false, max_tries=1)
     apis = ctx.apis
-    vers = getAPIVersions(ApisApi(ctx.client))
+    vers = @repeat max_tries try
+        getAPIVersions(ApisApi(ctx.client))
+    catch e
+        @retry if isa(e, IOError)
+            @debug("Retrying getAPIVersions")
+            sleep(2)
+        end
+    end
     api_groups = vers.groups
     for apigrp in api_groups
         name = apigrp.name
@@ -150,9 +157,16 @@ function fetch_misc_apis_versions(ctx::KuberContext; override=nothing, verbose::
     apis
 end
 
-function fetch_core_version(ctx::KuberContext; override=nothing, verbose::Bool=false)
+function fetch_core_version(ctx::KuberContext; override=nothing, verbose::Bool=false, max_tries=1)
     apis = ctx.apis
-    api_vers = getCoreAPIVersions(CoreApi(ctx.client))
+    api_vers = @repeat max_tries try
+        getCoreAPIVersions(CoreApi(ctx.client))
+    catch e
+        @retry if isa(e, IOError)
+            @debug("Retrying getCoreAPIVersions")
+            sleep(2)
+        end
+    end
     name = "Core"
     pref_vers = override_pref(name, api_vers.versions[1], override)
     verbose && @info("Core versions", supported=join(api_vers.versions, ", "), preferred=pref_vers)
@@ -194,7 +208,7 @@ function build_model_api_map(ctx::KuberContext)
     modelapi
 end
 
-function set_api_versions!(ctx::KuberContext; override=nothing, verbose::Bool=false)
+function set_api_versions!(ctx::KuberContext; override=nothing, verbose::Bool=false, max_tries=1)
     empty!(ctx.apis)
     empty!(ctx.modelapi)
 
@@ -203,8 +217,8 @@ function set_api_versions!(ctx::KuberContext; override=nothing, verbose::Bool=fa
     build_model_api_map(ctx)
 
     # fetch apis and map the types
-    fetch_core_version(ctx; override=override, verbose=verbose)
-    fetch_misc_apis_versions(ctx; override=override, verbose=verbose)
+    fetch_core_version(ctx; override=override, verbose=verbose, max_tries=max_tries)
+    fetch_misc_apis_versions(ctx; override=override, verbose=verbose, max_tries=max_tries)
     build_model_api_map(ctx)
 
     # add custom models

src/simpleapi.jl

@@ -58,34 +58,62 @@ function list(ctx::KuberContext, O::Symbol, apiversion::Union{String,Nothing}=no
     end
 end
 
-function get(ctx::KuberContext, O::Symbol, name::String, apiversion::Union{String,Nothing}=nothing; kwargs...)
-    isempty(ctx.apis) && set_api_versions!(ctx)
+function get(ctx::KuberContext, O::Symbol, name::String, apiversion::Union{String,Nothing}=nothing; max_tries::Integer=1, kwargs...)
+    isempty(ctx.apis) && set_api_versions!(ctx; max_tries=max_tries)
 
     apictx = _get_apictx(ctx, O, apiversion)
     try
         apicall = eval(Symbol("read$O"))
-        return apicall(apictx, name; kwargs...)
+        @repeat max_tries try
+            return apicall(apictx, name; kwargs...)
+        catch e
+            @retry if isa(e, IOError)
+                @debug("Retrying ", "read$O")
+                sleep(2)
+            end
+        end
     catch ex
         isa(ex, UndefVarError) || rethrow()
         apicall = eval(Symbol("readNamespaced$O"))
-        return apicall(apictx, name, ctx.namespace; kwargs...)
+        @repeat max_tries try
+            return apicall(apictx, name, ctx.namespace; kwargs...)
+        catch e
+            @retry if isa(e, IOError)
+                @debug("Retrying ", "readNamespaced$O")
+                sleep(2)
+            end
+        end
     end
 end
 
-function get(ctx::KuberContext, O::Symbol, apiversion::Union{String,Nothing}=nothing; label_selector=nothing, namespace::Union{String,Nothing}=ctx.namespace)
-    isempty(ctx.apis) && set_api_versions!(ctx)
+function get(ctx::KuberContext, O::Symbol, apiversion::Union{String,Nothing}=nothing; label_selector=nothing, namespace::Union{String,Nothing}=ctx.namespace, max_tries::Integer=1)
+    isempty(ctx.apis) && set_api_versions!(ctx; max_tries=max_tries)
 
     apictx = _get_apictx(ctx, O, apiversion)
     try
         apiname = "list$O"
         (namespace === nothing) && (apiname *= "ForAllNamespaces")
         apicall = eval(Symbol(apiname))
-        return apicall(apictx; labelSelector=label_selector)
+        @repeat max_tries try
+            return apicall(apictx; labelSelector=label_selector)
+        catch e
+            @retry if isa(e, IOError)
+                @debug("Retrying ", apiname)
+                sleep(2)
+            end
+        end
     catch ex
         isa(ex, UndefVarError) || rethrow()
         (namespace === nothing) && rethrow()
         apicall = eval(Symbol("listNamespaced$O"))
-        return apicall(apictx, namespace; labelSelector=label_selector)
+        @repeat max_tries try
+            return apicall(apictx, namespace; labelSelector=label_selector)
+        catch e
+            @retry if isa(e, IOError)
+                @debug("Retrying ", "listNamespaced$O")
+                sleep(2)
+            end
+        end
     end
 end