Move to pure / impure breakup (#165)

* proof of concept with dynamo

* proof of concept with S3

* kinesis

* lambda

* SQS

* green tests

* remove unnecessary files

* latest elixir travis

* bug fix

* dialyzer fixes

* decode kinesis records

* s3 parsers

* correct tests for new parsers

* delete_all_objects again

* include actual function

* green tests for rds and ec2

* dialyzer fixes

* Don't do virtual style hosts per #168

* note about 0.5 branch

* handle encoding lambda context

Author: benwilson512

.travis.yml

@@ -1,6 +1,6 @@
 language: elixir
 sudo: false
-elixir: 1.2.0
+elixir: 1.2.5
 notifications:
   recipients:
     - [email protected]

README.md

@@ -2,7 +2,45 @@ ExAws
 =====
 [![Build Status](https://travis-ci.org/CargoSense/ex_aws.svg?branch=master)](https://travis-ci.org/CargoSense/ex_aws)
 
-A flexible easy to use set of clients AWS APIs.
+A flexible easy to use set of AWS APIs.
+
+## 1.0.0-beta0 Changes
+
+The `v0.5` branch holds the legacy approach.
+
+ExAws 1.0.0 takes a more data driven approach to querying APIs. The various functions
+that exist inside a service like `S3.list_objects` or `Dynamo.create_table` all
+return a struct which holds the information necessary to make that particular operation.
+
+You then have 4 ways you can choose to execute that operation:
+
+```elixir
+# Normal
+S3.list_buckets |> ExAw.request #=> {:ok, response}
+# With per request configuration overrides
+S3.list_buckets |> ExAw.request(config) #=> {:ok, response}
+
+# Raise on error, return successful responses directly
+S3.list_buckets |> ExAw.request! #=> response
+S3.list_buckets |> ExAw.request!(config) #=> response
+```
+
+Certain operations also support Elixir streams:
+
+```elixir
+S3.list_objects("my-bucket") |> ExAws.stream! #=> #Function<13.52451638/2 in Stream.resource/3>
+S3.list_objects("my-bucket") |> ExAws.stream!(config) #=> #Function<13.52451638/2 in Stream.resource/3>
+```
+
+The ability to return a stream is noticed in the function's documentation.
+
+### Migration
+
+This change greatly simplifies the ExAws code paths, and removes entirely the complex
+meta-programming pervasive to the original approach. However, it does constitute
+a breaking change for anyone who had a client with custom logic.
+
+
 
 ## Highlighted Features
 - Easy configuration.
@@ -28,17 +66,17 @@ If you wish to use instance roles to obtain AWS access keys you will need to add
 ```elixir
 def deps do
   [
-    {:ex_aws, "~> 0.4.10"},
-    {:poison, "~> 1.2"},
-    {:httpoison, "~> 0.7"}
+    {:ex_aws, "~> 1.0.0-beta0"},
+    {:poison, "~> 2.0"},
+    {:httpoison, "~> 0.8"}
   ]
 end
 ```
 Don't forget to add :httpoison to your applications list if that's in fact the http client you choose. `:ex_aws` must always be added to your applications list.
 
 ```elixir
 def application do
-  [applications: [:ex_aws, :httpoison]]
+  [applications: [:ex_aws, :httpoison, :poison]]
 end
 ```
 

lib/ex_aws.ex

@@ -2,7 +2,65 @@ defmodule ExAws do
   @moduledoc File.read!("#{__DIR__}/../README.md")
   use Application
 
+  @doc """
+  Perform an AWS request
 
+  First build an operation from one of the services, and then pass it to this
+  function to perform it.
+
+  This function takes an optional second parameter of configuration overrides.
+  This is useful if you want to have certain configuration changed on a per
+  request basis.
+
+  ## Examples
+
+  ```
+  ExAws.S3.list_buckets |> ExAws.request
+
+  ExAws.S3.list_buckets |> ExAws.request(region: "eu-west-1")
+
+  ExAws.Dynamo.get_object("users", "[email protected]") |> ExAws.request
+  ```
+
+  """
+  @spec request(ExAws.Operation.t) :: term
+  @spec request(ExAws.Operation.t, Keyword.t) :: {:ok, term} | {:error, term}
+  def request(op, config_overrides \\ []) do
+    ExAws.Operation.perform(op, ExAws.Config.build(op.service, config_overrides))
+  end
+
+  @doc """
+  Build a stream
+  """
+  @spec stream!(ExAws.Operation.t) :: Enumerable.t
+  @spec stream!(ExAws.Operation.t, Keyword.t) :: Enumerable.t
+  def stream!(op, config_overrides \\ []) do
+    ExAws.Operation.stream!(op, ExAws.Config.build(op.service, config_overrides))
+  end
+
+  @doc """
+  Perform an AWS request, raise if it fails.
+
+  Same as `request/1,2` except it will either return the successful response from
+  AWS or raise an exception.
+  """
+  @spec request!(ExAws.Operation.t) :: term | no_return
+  @spec request!(ExAws.Operation.t, Keyword.t) :: term | no_return
+  def request!(op, config_overrides \\ []) do
+    case request(op, config_overrides) do
+      {:ok, result} ->
+        result
+
+      error ->
+        raise ExAws.Error, """
+          ExAws Request Error!
+
+          #{inspect error}
+          """
+    end
+  end
+
+  @doc false
   def start(_type, _args) do
     import Supervisor.Spec, warn: false
 

lib/ex_aws/client.ex

@@ -2,7 +2,7 @@ defmodule ExAws.Config do
 
   @moduledoc false
 
-  # Generates the configuration for a client.
+  # Generates the configuration for a service.
   # It starts with the defaults for a given environment
   # and then merges in the common config from the ex_aws config root,
   # and then finally any config specified for the particular service
@@ -12,72 +12,70 @@ defmodule ExAws.Config do
     :region, :security_token
   ]
 
-  def build(client, opts \\ []) do
-    config = client
-    |> ExAws.Config.get
-    |> Map.merge(Enum.into(opts, %{}))
+  def build(service, opts \\ []) do
+    overrides = Map.new(opts)
 
-    %{client | config: config}
+    service
+    |> ExAws.Config.get(overrides)
     |> retrieve_runtime_config
     |> parse_host_for_region
   end
 
-  def get(%{__struct__: client_module, service: service}) do
-    config_root = client_module.config_root
-    unless config_root, do: raise "A valid configuration root is required in your #{service} client"
+  @doc """
+  Builds a complete set of config for an operation.
 
-    defaults = ExAws.Config.Defaults.get
-    config   = config_root |> Keyword.get(service, [])
-    common   = defaults
-    |> Keyword.merge(config_root)
-    |> Keyword.take(@common_config)
+  1) Defaults are pulled from `ExAws.Config.Defaults`
+  2) Common values set via e.g `config :ex_aws` are merged in.
+  3) Keys set on the individual service e.g `config :ex_aws, :s3` are merged in
+  4) Finally, any configuration overrides are merged in
+  """
+  def get(service, overrides) do
+    defaults = ExAws.Config.Defaults.get(service)
+    common_config = Application.get_all_env(:ex_aws) |> Map.new |> Map.take(@common_config)
+    service_config = Application.get_env(:ex_aws, service, []) |> Map.new
 
     defaults
-    |> Keyword.get(service, [])
-    |> Keyword.merge(common)
-    |> Keyword.merge(config)
-    |> Enum.into(%{})
+    |> Map.merge(common_config)
+    |> Map.merge(service_config)
+    |> Map.merge(overrides)
   end
 
-  def retrieve_runtime_config(%{config: config} = client) do
-    new_config = config
-    |> Enum.reduce(%{}, fn
+  def retrieve_runtime_config(config) do
+    Enum.reduce(config, %{}, fn
       {:host, host}, config ->
-        Map.put(config, :host, retrieve_runtime_value(host, client))
+        Map.put(config, :host, retrieve_runtime_value(host, config))
       {k, v}, config ->
-        case retrieve_runtime_value(v, client) do
+        case retrieve_runtime_value(v, config) do
           %{} = result -> Map.merge(config, result)
           value -> Map.put(config, k, value)
         end
     end)
-
-    %{client | config: new_config}
   end
 
   def retrieve_runtime_value({:system, env_key}, _) do
     System.get_env(env_key)
   end
-  def retrieve_runtime_value(:instance_role, client) do
-    client
+  def retrieve_runtime_value(:instance_role, config) do
+    config
     |> ExAws.Config.AuthCache.get
     |> Map.take([:access_key_id, :secret_access_key, :security_token])
   end
-  def retrieve_runtime_value(values, client) when is_list(values) do
+  def retrieve_runtime_value(values, config) when is_list(values) do
     values
-    |> Stream.map(&retrieve_runtime_value(&1, client))
+    |> Stream.map(&retrieve_runtime_value(&1, config))
     |> Enum.find(&(&1))
   end
   def retrieve_runtime_value(value, _), do: value
 
-  def parse_host_for_region(%{config: %{host: {stub, host}, region: region} = config} = client) do
-    %{client | config: Map.put(config, :host, String.replace(host, stub, region))}
+  def parse_host_for_region(%{host: {stub, host}, region: region} = config) do
+    Map.put(config, :host, String.replace(host, stub, region))
   end
-  def parse_host_for_region(%{config: %{host: map, region: region} = config} = client) when is_map(map) do
+  def parse_host_for_region(%{host: map, region: region} = config) when is_map(map) do
     case Map.fetch(map, region) do
-      {:ok, host} -> %{client | config: Map.put(config, :host, host)}
+      {:ok, host} -> Map.put(config, :host, host)
       :error      -> "A host for region #{region} was not found in host map #{inspect(map)}"
     end
   end
-  def parse_host_for_region(client), do: client
+  def parse_host_for_region(config), do: config
 
 end

lib/ex_aws/config.ex

@@ -9,10 +9,10 @@ defmodule ExAws.Config.AuthCache do
     GenServer.start_link(__MODULE__, :ok, opts)
   end
 
-  def get(client) do
+  def get(config) do
     case :ets.lookup(__MODULE__, :aws_instance_auth) do
       [{:aws_instance_auth, auth_config}] -> auth_config
-      [] -> GenServer.call(__MODULE__, {:refresh_config, client})
+      [] -> GenServer.call(__MODULE__, {:refresh_config, config})
     end
   end
 
@@ -23,20 +23,20 @@ defmodule ExAws.Config.AuthCache do
     {:ok, ets}
   end
 
-  def handle_call({:refresh_config, client}, _from, ets) do
-    auth = refresh_config(client, ets)
+  def handle_call({:refresh_config, config}, _from, ets) do
+    auth = refresh_config(config, ets)
     {:reply, auth, ets}
   end
 
-  def handle_info({:refresh_config, client}, ets) do
-    refresh_config(client, ets)
+  def handle_info({:refresh_config, config}, ets) do
+    refresh_config(config, ets)
     {:noreply, ets}
   end
 
-  def refresh_config(client, ets) do
-    auth = ExAws.InstanceMeta.security_credentials(client)
+  def refresh_config(config, ets) do
+    auth = ExAws.InstanceMeta.security_credentials(config)
     :ets.insert(ets, {:aws_instance_auth, auth})
-    Process.send_after(self(), {:refresh_config, client}, refresh_in(auth[:expiration]))
+    Process.send_after(self(), {:refresh_config, config}, refresh_in(auth[:expiration]))
     auth
   end