Merge pull request #359 from Dynamoid/add-null-and-not-null-predicates

Add support of NULL and NOT_NULL operators

Author: andrykonchin

README.md

@@ -634,14 +634,21 @@ Address.where('latitude.between': [10212, 20000])
 ```
 
 You are able to filter results on the DynamoDB side and specify conditions for non-key fields.
-Following operators are available: `in`, `contains`, `not_contains`:
+Following additional operators are available: `in`, `contains`, `not_contains`, `null`, `not_null`:
 
 ```ruby
 Address.where('city.in': ['London', 'Edenburg', 'Birmingham'])
 Address.where('city.contains': ['on'])
 Address.where('city.not_contains': ['ing'])
+Address.where('postcode.null': false)
+Address.where('postcode.not_null': true)
 ```
 
+**WARNING:** Please take into accout that `NULL` and `NOT_NULL`
+operators check attribute presence in a document, not value.
+So if attribute `postcode`'s value is `NULL`, `NULL` operator will return false
+because attribute exists even if has `NULL` value.
+
 ### Consistent Reads
 
 Querying supports consistent reading. By default, DynamoDB reads are eventually consistent: if you do a write and then a read immediately afterwards, the results of the previous write may not be reflected. If you need to do a consistent read (that is, you need to read the results of a write immediately) you can do so, but keep in mind that consistent reads are twice as expensive as regular reads for DynamoDB.

lib/dynamoid/adapter_plugin/aws_sdk_v3.rb

@@ -22,8 +22,6 @@ class AwsSdkV3
         range_eq:           'EQ'
       }.freeze
 
-      # Don't implement NULL and NOT_NULL because it doesn't make seanse -
-      # we declare schema in models
       FIELD_MAP = {
         eq:           'EQ',
         ne:           'NE',
@@ -35,7 +33,9 @@ class AwsSdkV3
         between:      'BETWEEN',
         in:           'IN',
         contains:     'CONTAINS',
-        not_contains: 'NOT_CONTAINS'
+        not_contains: 'NOT_CONTAINS',
+        null:         'NULL',
+        not_null:     'NOT_NULL',
       }.freeze
       HASH_KEY  = 'HASH'
       RANGE_KEY = 'RANGE'
@@ -356,9 +356,14 @@ def delete_item(table_name, key, options = {})
       # @since 1.0.0
       def delete_table(table_name, options = {})
         resp = client.delete_table(table_name: table_name)
-        UntilPastTableStatus.new(client, table_name, :deleting).call if options[:sync] &&
-                                                                (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-                                                                status == TABLE_STATUSES[:deleting]
+
+        if options[:sync]
+          status = PARSE_TABLE_STATUS.call(resp, :table_description)
+          if status == TABLE_STATUSES[:deleting]
+            UntilPastTableStatus.new(client, table_name, :deleting).call
+          end
+        end
+
         table_cache.delete(table_name)
       rescue Aws::DynamoDB::Errors::ResourceInUseException => e
         Dynamoid.logger.error "Table #{table_name} cannot be deleted as it is in use"
@@ -619,15 +624,15 @@ def result_item_to_hash(item)
       # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
       # @params [String] operator: value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc
       # @params [Object] value: scalar value or array/set
-      def attribute_value_list(operator, value)
-        self.class.attribute_value_list(operator, value)
-      end
-
       def self.attribute_value_list(operator, value)
         # For BETWEEN and IN operators we should keep value as is (it should be already an array)
+        # NULL and NOT_NULL require absence of attribute list
         # For all the other operators we wrap the value with array
+        # https://docs.aws.amazon.com/en_us/amazondynamodb/latest/developerguide/LegacyConditionalParameters.Conditions.html
         if %w[BETWEEN IN].include?(operator)
           [value].flatten
+        elsif %w[NULL NOT_NULL].include?(operator)
+          nil
         else
           [value]
         end

lib/dynamoid/adapter_plugin/aws_sdk_v3/create_table.rb

@@ -62,11 +62,16 @@ def call
           end
           resp = client.create_table(client_opts)
           options[:sync] = true if !options.key?(:sync) && ls_indexes.present? || gs_indexes.present?
-          UntilPastTableStatus.new(client, table_name, :creating).call if options[:sync] &&
-                                                                  (status = PARSE_TABLE_STATUS.call(resp, :table_description)) &&
-                                                                  status == TABLE_STATUSES[:creating]
+
+          if options[:sync]
+            status = PARSE_TABLE_STATUS.call(resp, :table_description)
+            if status == TABLE_STATUSES[:creating]
+              UntilPastTableStatus.new(client, table_name, :creating).call
+            end
+          end
+
           # Response to original create_table, which, if options[:sync]
-          #   may have an outdated table_description.table_status of "CREATING"
+          # may have an outdated table_description.table_status of "CREATING"
           resp
         end
 

lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb

@@ -75,6 +75,8 @@ def scan_filter
               comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]],
               attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze)
             }
+            # nil means operator doesn't require attribute value list
+            conditions.delete(:attribute_value_list) if conditions[:attribute_value_list].nil?
             result[attr] = condition
             result
           end

lib/dynamoid/criteria/chain.rb

@@ -271,6 +271,13 @@ def field_hash(key)
                  { contains: val }
                when 'not_contains'
                  { not_contains: val }
+               # NULL/NOT_NULL operators don't have parameters
+               # So { null: true } means NULL check and { null: false } means NOT_NULL one
+               # The same logic is used for { not_null: BOOL }
+               when 'null'
+                 val ? { null: nil } : { not_null: nil }
+               when 'not_null'
+                 val ? { not_null: nil } : { null: nil }
                end
 
         { name.to_sym => hash }
@@ -314,10 +321,15 @@ def range_query
         opts.merge(query_opts).merge(consistent_opts)
       end
 
+      # TODO casting should be operator aware
+      # e.g. for NULL operator value should be boolean
+      # and isn't related to an attribute own type
       def type_cast_condition_parameter(key, value)
         return value if %i[array set].include?(source.attributes[key.to_sym][:type])
 
-        if !value.respond_to?(:to_ary)
+        if [true, false].include?(value) # Support argument for null/not_null operators
+          value
+        elsif !value.respond_to?(:to_ary)
           options = source.attributes[key.to_sym]
           value_casted = TypeCasting.cast_field(value, options)
           Dumping.dump_field(value_casted, options)

spec/dynamoid/criteria/chain_spec.rb

@@ -341,6 +341,34 @@ def request_params
       expect(model.where(name: 'a', 'job_title.not_contains': 'Consul').all)
         .to contain_exactly(customer2)
     end
+
+    it 'supports null' do
+      model.create_table
+
+      put_attributes(model.table_name, name: 'a', last_name: 'aa', age: 1)
+      put_attributes(model.table_name, name: 'a', last_name: 'bb', age: 2)
+      put_attributes(model.table_name, name: 'a', last_name: 'cc', )
+
+      documents = model.where(name: 'a', 'age.null': true).to_a
+      expect(documents.map(&:last_name)).to contain_exactly('cc')
+
+      documents = model.where(name: 'a', 'age.null': false).to_a
+      expect(documents.map(&:last_name)).to contain_exactly('aa', 'bb')
+    end
+
+    it 'supports not_null' do
+      model.create_table
+
+      put_attributes(model.table_name, name: 'a', last_name: 'aa', age: 1)
+      put_attributes(model.table_name, name: 'a', last_name: 'bb', age: 2)
+      put_attributes(model.table_name, name: 'a', last_name: 'cc', )
+
+      documents = model.where(name: 'a', 'age.not_null': true).to_a
+      expect(documents.map(&:last_name)).to contain_exactly('aa', 'bb')
+
+      documents = model.where('age.not_null': false).to_a
+      expect(documents.map(&:last_name)).to contain_exactly('cc')
+    end
   end
 
   # http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LegacyConditionalParameters.ScanFilter.html?shortFooter=true
@@ -490,6 +518,34 @@ def request_params
       expect(model.where('job_title.not_contains': 'Consul').all)
         .to contain_exactly(customer2)
     end
+
+    it 'supports null' do
+      model.create_table
+
+      put_attributes(model.table_name, id: '1', age: 1)
+      put_attributes(model.table_name, id: '2', age: 2)
+      put_attributes(model.table_name, id: '3')
+
+      documents = model.where('age.null': true).to_a
+      expect(documents.map(&:id)).to contain_exactly('3')
+
+      documents = model.where('age.null': false).to_a
+      expect(documents.map(&:id)).to contain_exactly('1', '2')
+    end
+
+    it 'supports not_null' do
+      model.create_table
+
+      put_attributes(model.table_name, id: '1', age: 1)
+      put_attributes(model.table_name, id: '2', age: 2)
+      put_attributes(model.table_name, id: '3')
+
+      documents = model.where('age.not_null':  true).to_a
+      expect(documents.map(&:id)).to contain_exactly('1', '2')
+
+      documents = model.where('age.not_null':  false).to_a
+      expect(documents.map(&:id)).to contain_exactly('3')
+    end
   end
 
   describe 'Lazy loading' do

spec/spec_helper.rb

@@ -47,5 +47,6 @@
   config.include NewClassHelper
   config.include DumpingHelper
   config.include PersistenceHelper
+  config.include ChainHelper
   config.include ActiveSupport::Testing::TimeHelpers
 end

spec/support/chain_helper.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ChainHelper
+  def put_attributes(table_name, attributes)
+    Dynamoid.adapter.put_item(table_name, attributes)
+  end
+end