Merge pull request #354 from Dynamoid/add-warning-about-skipped-conditions

Add :map field type

Author: andrykonchin

README.md

@@ -121,7 +121,8 @@ These fields will not change an existing table: so specifying a new read_capacit
 You'll have to define all the fields on the model and the data type of each field. Every field on the object must be included here; if you miss any they'll be completely bypassed during DynamoDB's initialization and will not appear on the model objects.
 
 By default, fields are assumed to be of type `:string`. Other built-in types are
-`:integer`, `:number`, `:set`, `:array`, `:datetime`, `date`, `:boolean`, `:raw` and `:serialized`.
+`:integer`, `:number`, `:set`, `:array`, `:map`, `:datetime`, `date`, `:boolean`, `:raw` and `:serialized`.
+`array` and `map` match List and Map DynamoDB types respectively.
 `raw` type means you can store Ruby Array, Hash, String and numbers.
 If built-in types do not suit you, you can use a custom field type represented by an arbitrary class, provided that the class supports a compatible serialization interface.
 The primary use case for using a custom field type is to represent your business logic with high-level types, while ensuring portability or backward-compatibility of the serialized representation.
@@ -531,6 +532,18 @@ u.addresses.where(city: 'Chicago').all
 
 But keep in mind Dynamoid -- and document-based storage systems in general -- are not drop-in replacements for existing relational databases. The above query does not efficiently perform a conditional join, but instead finds all the user's addresses and naively filters them in Ruby. For large associations this is a performance hit compared to relational database engines.
 
+**WARNING:** There is a limitation of conditions passed to `where`
+method. Only one condition for some particular field could be specified.
+The last one only will be applyed and others will be ignored. E.g. in
+examples:
+
+```ruby
+User.where('age.gt': 10, 'age.lt': 20)
+User.where(name: 'Mike').where('name.begins_with': 'Ed')
+```
+
+the first one will be ignored and the last one will be used.
+
 #### Limits
 
 There are three types of limits that you can query with:
@@ -901,6 +914,32 @@ Dynamoid.configure do |config|
 end
 ```
 
+## Logging
+
+There is a config option `logger`. Dynamoid writes requests and
+responses to DynamoDB using this logger on the `debug` level. So in
+order to troubleshoot and debug issues just set it:
+
+```ruby
+class User
+  include Dynamoid::Document
+  field name
+end
+
+Dynamoid.config.logger.level = :debug
+Dynamoid.config.endpoint = 'localhost:8000'
+
+User.create(name: 'Alex')
+
+# => D, [2019-05-12T20:01:07.840051 #75059] DEBUG -- : put_item | Request "{\"TableName\":\"dynamoid_users\",\"Item\":{\"created_at\":{\"N\":\"1557680467.608749\"},\"updated_at\":{\"N\":\"1557680467.608809\"},\"id\":{\"S\":\"1227eea7-2c96-4b8a-90d9-77b38eb85cd0\"}},\"Expected\":{\"id\":{\"Exists\":false}}}" | Response "{}"
+
+# => D, [2019-05-12T20:01:07.842397 #75059] DEBUG -- : (231.28 ms) PUT ITEM - ["dynamoid_users", {:created_at=>0.1557680467608749e10, :updated_at=>0.1557680467608809e10, :id=>"1227eea7-2c96-4b8a-90d9-77b38eb85cd0", :User=>nil}, {}]
+```
+
+The first line is a body of HTTP request and response. The second line -
+Dynamoid internal logging of API call (`PUT ITEM` in our case) with
+timing (231.28 ms).
+
 ## Credits
 
 Dynamoid borrows code, structure, and even its name very liberally from the truly amazing [Mongoid](https://github.com/mongoid/mongoid). Without Mongoid to crib from none of this would have been possible, and I hope they don't mind me reusing their very awesome ideas to make DynamoDB just as accessible to the Ruby world as MongoDB.

lib/dynamoid/dumping.rb

@@ -29,6 +29,7 @@ def self.find_dumper(options)
                      when :number     then NumberDumper
                      when :set        then SetDumper
                      when :array      then ArrayDumper
+                     when :map        then MapDumper
                      when :datetime   then DateTimeDumper
                      when :date       then DateDumper
                      when :serialized then SerializedDumper
@@ -42,6 +43,35 @@ def self.find_dumper(options)
       end
     end
 
+    module DeepSanitizeHelper
+      extend self
+
+      def deep_sanitize(value)
+        case value
+        when Hash
+          sanitize_hash(value).transform_values { |v| deep_sanitize(v) }
+        when Array
+          sanitize_array(value).map { |v| deep_sanitize(v) }
+        else
+          value
+        end
+      end
+
+      private
+
+      def sanitize_hash(hash)
+        hash.transform_values { |v| invalid_value?(v) ? nil : v }
+      end
+
+      def sanitize_array(array)
+        array.map { |v| invalid_value?(v) ? nil : v }
+      end
+
+      def invalid_value?(value)
+        (value.is_a?(Set) || value.is_a?(String)) && value.empty?
+      end
+    end
+
     class Base
       def initialize(options)
         @options = options
@@ -168,6 +198,13 @@ def element_options
       end
     end
 
+    # hash -> map
+    class MapDumper < Base
+      def process(value)
+        DeepSanitizeHelper.deep_sanitize(value)
+      end
+    end
+
     # datetime -> integer/string
     class DateTimeDumper < Base
       def process(value)
@@ -221,32 +258,7 @@ def format_date(value, options)
     # any standard Ruby object -> self
     class RawDumper < Base
       def process(value)
-        deep_sanitize(value)
-      end
-
-      private
-
-      def deep_sanitize(value)
-        case value
-        when Hash
-          sanitize_hash(value).transform_values { |v| deep_sanitize(v) }
-        when Array
-          sanitize_array(value).map { |v| deep_sanitize(v) }
-        else
-          value
-        end
-      end
-
-      def sanitize_hash(hash)
-        hash.transform_values { |v| invalid_value?(v) ? nil : v }
-      end
-
-      def sanitize_array(array)
-        array.map { |v| invalid_value?(v) ? nil : v }
-      end
-
-      def invalid_value?(value)
-        (value.is_a?(Set) || value.is_a?(String)) && value.empty?
+        DeepSanitizeHelper.deep_sanitize(value)
       end
     end
 

lib/dynamoid/type_casting.rb

@@ -29,6 +29,7 @@ def self.find_type_caster(options)
                           when :number     then NumberTypeCaster
                           when :set        then SetTypeCaster
                           when :array      then ArrayTypeCaster
+                          when :map        then MapTypeCaster
                           when :datetime   then DateTimeTypeCaster
                           when :date       then DateTypeCaster
                           when :raw        then RawTypeCaster
@@ -204,6 +205,22 @@ def element_options
       end
     end
 
+    class MapTypeCaster < Base
+      def process(value)
+        return nil if value.nil?
+
+        if value.is_a? Hash
+          value
+        elsif value.respond_to? :to_hash
+          value.to_hash
+        elsif value.respond_to? :to_h
+          value.to_h
+        else
+          nil
+        end
+      end
+    end
+
     class DateTimeTypeCaster < Base
       def process(value)
         if !value.respond_to?(:to_datetime)

lib/dynamoid/undumping.rb

@@ -32,6 +32,7 @@ def self.find_undumper(options)
                        when :number     then NumberUndumper
                        when :set        then SetUndumper
                        when :array      then ArrayUndumper
+                       when :map        then MapUndumper
                        when :datetime   then DateTimeUndumper
                        when :date       then DateUndumper
                        when :raw        then RawUndumper
@@ -45,6 +46,35 @@ def self.find_undumper(options)
       end
     end
 
+    module UndumpHashHelper
+      extend self
+
+      def undump_hash(hash)
+        {}.tap do |h|
+          hash.each { |key, value| h[key.to_sym] = undump_hash_value(value) }
+        end
+      end
+
+      private
+
+      def undump_hash_value(val)
+        case val
+        when BigDecimal
+          if Dynamoid::Config.convert_big_decimal
+            val.to_f
+          else
+            val
+          end
+        when Hash
+          undump_hash(val)
+        when Array
+          val.map { |v| undump_hash_value(v) }
+        else
+          val
+        end
+      end
+    end
+
     class Base
       def initialize(options)
         @options = options
@@ -157,6 +187,12 @@ def element_options
       end
     end
 
+    class MapUndumper < Base
+      def process(value)
+        UndumpHashHelper.undump_hash(value)
+      end
+    end
+
     class DateTimeUndumper < Base
       def process(value)
         return value if value.is_a?(Date) || value.is_a?(DateTime) || value.is_a?(Time)
@@ -190,36 +226,11 @@ def process(value)
     class RawUndumper < Base
       def process(value)
         if value.is_a?(Hash)
-          undump_hash(value)
+          UndumpHashHelper.undump_hash(value)
         else
           value
         end
       end
-
-      private
-
-      def undump_hash(hash)
-        {}.tap do |h|
-          hash.each { |key, value| h[key.to_sym] = undump_hash_value(value) }
-        end
-      end
-
-      def undump_hash_value(val)
-        case val
-        when BigDecimal
-          if Dynamoid::Config.convert_big_decimal
-            val.to_f
-          else
-            val
-          end
-        when Hash
-          undump_hash(val)
-        when Array
-          val.map { |v| undump_hash_value(v) }
-        else
-          val
-        end
-      end
     end
 
     class SerializedUndumper < Base

spec/dynamoid/dumping_spec.rb

@@ -1003,6 +1003,106 @@ def self.dynamoid_load(string)
     end
   end
 
+  describe 'Map field' do
+    let(:klass) do
+      new_class do
+        field :settings, :map
+      end
+    end
+
+    it 'stores as a Document' do
+      settings = {
+        Day: "Monday",
+        UnreadEmails: 42,
+        ItemsOnMyDesk: [
+          "Coffee Cup",
+          "Telephone",
+          {
+            Pens: { Quantity: 3 },
+            Pencils: { Quantity: 2 },
+            Erasers: { Quantity: 1 }
+          }
+        ]
+      }
+      obj = klass.create(settings: settings)
+
+      expect(reload(obj).settings).to eql(
+        {
+          Day: "Monday",
+          UnreadEmails: 42,
+          ItemsOnMyDesk: [
+            "Coffee Cup",
+            "Telephone",
+            {
+              Pens: { Quantity: 3 },
+              Pencils: { Quantity: 2 },
+              Erasers: { Quantity: 1 }
+            }
+          ]
+        }
+      )
+      expect(raw_attributes(obj)[:settings]).to eql(
+        {
+          "Day" => "Monday",
+          "UnreadEmails" => 42,
+          "ItemsOnMyDesk" => [
+            "Coffee Cup",
+            "Telephone",
+            {
+              "Pens" => { "Quantity" => 3 },
+              "Pencils" => { "Quantity" => 2 },
+              "Erasers" => { "Quantity" => 1 }
+            }
+          ]
+        }
+      )
+    end
+
+    it 'deeply symbolizes keys' do
+      settings = { 'foo' => { 'bar' => 1 }, 'baz' => [{ 'foobar' => 2 }] }
+      obj = klass.create(settings: settings)
+
+      expect(reload(obj).settings).to eql(foo: { bar: 1 }, baz: [{ foobar: 2 }])
+    end
+
+    describe 'sanitizing' do
+      it 'replaces empty set with nil in Hash' do
+        settings = { 'foo' => [].to_set }
+        obj = klass.create(settings: settings)
+
+        expect(reload(obj).settings).to eql(foo: nil)
+      end
+
+      it 'replaces empty string with nil in Hash' do
+        settings = { 'foo' => '' }
+        obj = klass.create(settings: settings)
+
+        expect(reload(obj).settings).to eql(foo: nil)
+      end
+
+      it 'replaces empty set with nil in nested Array' do
+        settings = { 'foo' => [1, 2, [].to_set] }
+        obj = klass.create(settings: settings)
+
+        expect(reload(obj).settings).to eql(foo: [1, 2, nil])
+      end
+
+      it 'replaces empty string with nil in nested Array' do
+        settings = { 'foo' => [1, 2, ''] }
+        obj = klass.create(settings: settings)
+
+        expect(reload(obj).settings).to eql(foo: [1, 2, nil])
+      end
+
+      it 'processes nested Hash and Array' do
+        settings = { a: 1, b: '', c: [1, 2, '', { d: 3, e: '' }] }
+        obj = klass.create(settings: settings)
+
+        expect(reload(obj).settings).to eql(a: 1, b: nil, c: [1, 2, nil, { d: 3, e: nil }])
+      end
+    end
+  end
+
   describe 'String field' do
     it 'stores as strings' do
       klass = new_class do