Merge pull request #18 from robmckinnon/update_elixir_1-9

Update for Elixir 1.9

Author: robmckinnon

.gitignore

@@ -5,3 +5,4 @@
 /docs
 erl_crash.dump
 *.ez
+.elixir_ls

.travis.yml

@@ -6,6 +6,7 @@ elixir:
   - 1.6.6
   - 1.7.4
   - 1.8.1
+  - 1.9.1
 after_script:
-  - mix deps.get --only docs
+  - MIX_ENV=docs mix deps.get --only docs
   - MIX_ENV=docs mix inch.report

README.md

@@ -10,83 +10,95 @@ Create Elixir structs, maps with atom keys, and keyword lists from CSV/TSV data.
 
 You can view [full DataMorph API documentation on hexdocs](https://hexdocs.pm/data_morph/DataMorph.html).
 
+Note, we should never convert user input to atoms. This is because atoms are not
+garbage collected. Once an atom is created, it is never reclaimed.
+
+Generating atoms from user input would mean the user can inject enough
+different names to exhaust our system memory, or we reach the Erlang VM limit
+for the maximum number of atoms which will bring our system down regardless.
+
 ## Installation
 
 Add
 ```elixir
-{:data_morph, "~> 0.0.9"}
+{:data_morph, "~> 0.1.0"}
 ```
 to your deps in `mix.exs` like so:
 
 ```elixir
 defp deps do
   [
-    {:data_morph, "~> 0.0.9"}
+    {:data_morph, "~> 0.1.0"}
   ]
 end
 ```
 
 ## Usage examples
 
-Define a struct and return stream of structs created from a `tsv` string, a `namespace` atom and `name` string.
+Given a CSV file of data like this:
+
+```sh
+(echo "name,iso-code" && echo New Zealand,nz && echo United Kingdom,gb) > tmp.csv
+```
+
+Define a struct and return stream of structs created from a `csv` stream,
+a `namespace` string or atom, and a `name` string or atom.
 
 ```elixir
-"name\tiso\n" <>
-"New Zealand\tnz\n" <>
-"United Kingdom\tgb" \
-|> DataMorph.structs_from_tsv(OpenRegister, "country") \
+File.stream!('./tmp.csv') \
+|> DataMorph.structs_from_csv("my-module", :iso_country) \
 |> Enum.to_list
 # [
-#   %OpenRegister.Country{iso: "nz", name: "New Zealand"},
-#   %OpenRegister.Country{iso: "gb", name: "United Kingdom"}
+#   %MyModule.IsoCountry{iso_code: "nz", name: "New Zealand"},
+#   %MyModule.IsoCountry{iso_code: "gb", name: "United Kingdom"}
 # ]
 ```
 
-Define a struct and return stream of structs created from a `csv` file stream,
-and a `namespace` string and `name` atom.
-
-```sh
-(echo name,iso && echo New Zealand,nz && echo United Kingdom,gb) > tmp.csv
-```
+Return stream of maps created from a `csv` stream.
 
 ```elixir
 File.stream!('./tmp.csv') \
-|> DataMorph.structs_from_csv("open-register", :iso_country) \
+|> DataMorph.maps_from_csv() \
 |> Enum.to_list
 # [
-#   %OpenRegister.IsoCountry{iso: "nz", name: "New Zealand"},
-#   %OpenRegister.IsoCountry{iso: "gb", name: "United Kingdom"}
+#   %{iso_code: "nz", name: "New Zealand"},
+#   %{iso_code: "gb", name: "United Kingdom"}
 # ]
 ```
 
-Define a struct and puts stream of structs created from a stream of `csv`
-on standard input, and a `namespace` atom, and `name` string.
+Return a stream of keyword lists created from a stream of `csv`.
+
 ```sh
 (echo name,iso && echo New Zealand,NZ) | \
-    mix run -e 'IO.puts inspect \
-    IO.stream(:stdio, :line) \
-    |> DataMorph.structs_from_csv(:ex, "ample") \
-    |> Enum.at(0)'
-# %Ex.Ample{iso: "NZ", name: "New Zealand"}
+    mix run -e 'IO.stream(:stdio, :line) \
+    |> DataMorph.keyword_lists_from_csv() \
+    |> IO.inspect'
+# [[name: "New Zealand", iso: "NZ"]]
 ```
 
-Add additional new fields to struct when called again with different `tsv`.
+Add new fields to struct when called twice with different fields in `tsv`.
 
 ```elixir
 "name\tiso\n" <>
 "New Zealand\tnz\n" <>
 "United Kingdom\tgb" \
-|> DataMorph.structs_from_tsv(OpenRegister, "country") \
+|> DataMorph.structs_from_tsv(MyModule, "country") \
 |> Enum.to_list
+# [
+#   %MyModule.Country{iso: "nz", name: "New Zealand"},
+#   %MyModule.Country{iso: "gb", name: "United Kingdom"}
+# ]
 
 "name\tacronym\n" <>
 "New Zealand\tNZ\n" <>
 "United Kingdom\tUK" \
-|> DataMorph.structs_from_tsv(OpenRegister, "country") \
+|> DataMorph.structs_from_tsv("MyModule", :country) \
 |> Enum.to_list
+# warning: redefining module MyModule.Country
+#          (current version defined in memory)
 # [
-#   %OpenRegister.Country{acronym: "NZ", iso: nil, name: "New Zealand"},
-#   %OpenRegister.Country{acronym: "UK", iso: nil, name: "United Kingdom"}
+#   %MyModule.Country{acronym: "NZ", iso: nil, name: "New Zealand"},
+#   %MyModule.Country{acronym: "UK", iso: nil, name: "United Kingdom"}
 # ]
 ```
 

lib/data_morph.ex

@@ -1,6 +1,14 @@
 defmodule DataMorph do
   @moduledoc ~S"""
-  Create Elixir structs, maps with atom keys, and keyword lists from CSV/TSV data.
+  Create Elixir structs, maps with atom keys, and keyword lists from CSV/TSV
+  data.
+
+  Note, we should never convert user input to atoms. This is because atoms are
+  not garbage collected. Once an atom is created, it is never reclaimed.
+
+  Generating atoms from user input would mean the user can inject enough
+  different names to exhaust our system memory, or we reach the Erlang VM limit
+  for the maximum number of atoms which will bring our system down regardless.
 
   ## Examples
 

mix.exs

@@ -1,13 +1,13 @@
 defmodule DataMorph.Mixfile do
   use Mix.Project
 
-  @version "0.0.9"
+  @version "0.1.0"
 
   def project do
     [
       app: :data_morph,
       version: @version,
-      elixir: "~> 1.3 or ~> 1.4 or ~> 1.5 or ~> 1.6 or ~> 1.7 or ~> 1.8",
+      elixir: "~> 1.3 or ~> 1.4 or ~> 1.5 or ~> 1.6 or ~> 1.7 or ~> 1.8 or ~> 1.9",
       description: description(),
       deps: deps(),
       package: package(),
@@ -25,12 +25,12 @@ defmodule DataMorph.Mixfile do
 
   defp deps do
     [
-      {:csv, "~> 2.3"},
+      {:csv, "~> 2.3.1"},
       # Docs dependencies
-      {:ex_doc, "~> 0.19", only: :docs},
-      {:inch_ex, "~> 2.0", only: :docs},
+      {:ex_doc, "~> 0.21.1", only: :docs, runtime: false},
+      {:inch_ex, "~> 2.0", only: :docs, runtime: false},
       # Test dependencies
-      {:mix_test_watch, "~> 0.9", only: :dev}
+      {:mix_test_watch, "~> 0.9", only: :dev, runtime: false}
     ]
   end
 

mix.lock

@@ -1,14 +1,14 @@
 %{
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
-  "csv": {:hex, :csv, "2.3.0", "e30e4bbe633653a5c2da43cab42a61623e0acc80ef40c321bc29f0d698697bd3", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm"},
-  "earmark": {:hex, :earmark, "1.3.1", "73812f447f7a42358d3ba79283cfa3075a7580a3a2ed457616d6517ac3738cb9", [:mix], [], "hexpm"},
-  "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
+  "csv": {:hex, :csv, "2.3.1", "9ce11eff5a74a07baf3787b2b19dd798724d29a9c3a492a41df39f6af686da0e", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm"},
+  "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
+  "ex_doc": {:hex, :ex_doc, "0.21.1", "5ac36660846967cd869255f4426467a11672fec3d8db602c429425ce5b613b90", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
   "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"},
   "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []},
   "inch_ex": {:hex, :inch_ex, "2.0.0", "24268a9284a1751f2ceda569cd978e1fa394c977c45c331bb52a405de544f4de", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
-  "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
-  "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
+  "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
   "mix_test_watch": {:hex, :mix_test_watch, "0.9.0", "c72132a6071261893518fa08e121e911c9358713f62794a90c95db59042af375", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm"},
   "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"},
   "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"},