Merge pull request #832 from Dynamoid/ak/where-with-filter-expression

Support condition expressions with `#where`

Author: andrykonchin

.github/workflows/ci.yml

@@ -175,13 +175,13 @@ jobs:
             gemfile: rails_8_0
 
         include:
-          - ruby: "jruby-9.3.9.0"
+          - ruby: "jruby-9.3.15.0"
             gemfile: rails_4_2
-          - ruby: "jruby-9.3.9.0"
+          - ruby: "jruby-9.3.15.0"
             gemfile: rails_5_0
-          - ruby: "jruby-9.3.9.0"
+          - ruby: "jruby-9.3.15.0"
             gemfile: rails_5_1
-          - ruby: "jruby-9.3.9.0"
+          - ruby: "jruby-9.3.15.0"
             gemfile: rails_5_2
 
     name: ${{ matrix.gemfile }}, Ruby ${{ matrix.ruby }}

README.md

@@ -719,7 +719,7 @@ users = User.import([{ name: 'Josh' }, { name: 'Nick' }])
 
 ### Querying
 
-Querying can be done in one of three ways:
+Querying can be done in one of the following ways:
 
 ```ruby
 Address.find(address.id)              # Find directly by ID.
@@ -728,6 +728,27 @@ Address.where(city: 'Chicago').all    # Find by any number of matching criteria.
 Address.find_by_city('Chicago')       # The same as above, but using ActiveRecord's older syntax.
 ```
 
+There is also a way to `#where` with a condition expression:
+
+```ruby
+Address.where('city = :c', c: 'Chicago')
+```
+
+A condition expression may contain operators (e.g. `<`, `>=`, `<>`),
+keywords (e.g. `AND`, `OR`, `BETWEEN`) and built-in functions (e.g.
+`begins_with`, `contains`) (see (documentation
+)[https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html]
+for full syntax description).
+
+**Warning:** Values (specified for a String condition expression) are
+sent as is so Dynamoid field types that aren't supported natively by
+DynamoDB (e.g. `datetime` and `date`) require explicit casting.
+
+**Warning:** String condition expressions will be used by DynamoDB only
+at filtering, so conditions on key attributes should be specified as a
+Hash to perform Query operation instead of Scan. Don't use key
+attributes in `#where`'s String condition expressions.
+
 And you can also query on associations:
 
 ```ruby

lib/dynamoid/adapter.rb

@@ -118,7 +118,7 @@ def delete(table, ids, options = {})
     # @param [Hash] query a hash of attributes: matching records will be returned by the scan
     #
     # @since 0.2.0
-    def scan(table, query = {}, opts = {})
+    def scan(table, query = [], opts = {})
       benchmark('Scan', table, query) { adapter.scan(table, query, opts) }
     end
 

lib/dynamoid/adapter_plugin/aws_sdk_v3.rb

@@ -517,7 +517,7 @@ def put_item(table_name, object, options = {})
       # @since 1.0.0
       #
       # @todo Provide support for various other options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#query-instance_method
-      def query(table_name, key_conditions, non_key_conditions = {}, options = {})
+      def query(table_name, key_conditions, non_key_conditions = [], options = {})
         Enumerator.new do |yielder|
           table = describe_table(table_name)
 
@@ -550,7 +550,7 @@ def query_count(table_name, key_conditions, non_key_conditions, options)
       # @since 1.0.0
       #
       # @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method
-      def scan(table_name, conditions = {}, options = {})
+      def scan(table_name, conditions = [], options = {})
         Enumerator.new do |yielder|
           table = describe_table(table_name)
 
@@ -563,7 +563,7 @@ def scan(table_name, conditions = {}, options = {})
         end
       end
 
-      def scan_count(table_name, conditions = {}, options = {})
+      def scan_count(table_name, conditions = [], options = {})
         table = describe_table(table_name)
         options[:select] = 'COUNT'
 

lib/dynamoid/adapter_plugin/aws_sdk_v3/filter_expression_convertor.rb

@@ -20,7 +20,24 @@ def initialize(conditions, name_placeholders, value_placeholders, name_placehold
         private
 
         def build
-          clauses = @conditions.map do |name, attribute_conditions|
+          clauses = []
+
+          @conditions.each do |conditions|
+            if conditions.is_a? Hash
+              clauses << build_for_hash(conditions) unless conditions.empty?
+            elsif conditions.is_a? Array
+              query, placeholders = conditions
+              clauses << build_for_string(query, placeholders)
+            else
+              raise ArgumentError, "expected Hash or Array but actual value is #{conditions}"
+            end
+          end
+
+          @expression = clauses.join(' AND ')
+        end
+
+        def build_for_hash(hash)
+          clauses = hash.map do |name, attribute_conditions|
             attribute_conditions.map do |operator, value|
               # replace attribute names with placeholders unconditionally to support
               # - special characters (e.g. '.', ':', and '#') and
@@ -62,7 +79,21 @@ def build
             end
           end.flatten
 
-          @expression = clauses.join(' AND ')
+          if clauses.empty?
+            nil
+          else
+            clauses.join(' AND ')
+          end
+        end
+
+        def build_for_string(query, placeholders)
+          placeholders.each do |(k, v)|
+            k = k.to_s
+            k = ":#{k}" unless k.start_with?(':')
+            @value_placeholders[k] = v
+          end
+
+          "(#{query})"
         end
 
         def name_placeholder_for(name)

lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb

@@ -69,7 +69,7 @@ def build_request
           limit = [record_limit, scan_limit, batch_size].compact.min
 
           # key condition expression
-          convertor = FilterExpressionConvertor.new(@key_conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence)
+          convertor = FilterExpressionConvertor.new([@key_conditions], name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence)
           key_condition_expression = convertor.expression
           value_placeholders = convertor.value_placeholders
           name_placeholders = convertor.name_placeholders

lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb

@@ -13,7 +13,7 @@ class AwsSdkV3
       class Scan
         attr_reader :client, :table, :conditions, :options
 
-        def initialize(client, table, conditions = {}, options = {})
+        def initialize(client, table, conditions = [], options = {})
           @client = client
           @table = table
           @conditions = conditions

lib/dynamoid/criteria/chain.rb

@@ -95,20 +95,27 @@ def initialize(source)
       #
       # Internally +where+ performs either +Scan+ or +Query+ operation.
       #
+      # Conditions can be specified as an expression as well:
+      #
+      #   Post.where('links_count = :v', v: 2)
+      #
+      # This way complex expressions can be constructed (e.g. with AND, OR, and NOT
+      # keyword):
+      #
+      #   Address.where('city = :c AND (post_code = :pc1 OR post_code = :pc2)', city: 'A', pc1: '001', pc2: '002')
+      #
+      # See documentation for condition expression's syntax and examples:
+      # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html
+      # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.FilterExpression.html
+      #
       # @return [Dynamoid::Criteria::Chain]
       # @since 0.2.0
-      def where(args)
-        detector = NonexistentFieldsDetector.new(args, @source)
-        if detector.found?
-          Dynamoid.logger.warn(detector.warning_message)
+      def where(conditions, placeholders = nil)
+        if conditions.is_a?(Hash)
+          where_with_hash(conditions)
+        else
+          where_with_string(conditions, placeholders)
         end
-
-        @where_conditions.update(args.symbolize_keys)
-
-        # we should re-initialize keys detector every time we change @where_conditions
-        @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name)
-
-        self
       end
 
       # Turns on strongly consistent reads.
@@ -500,6 +507,29 @@ def pluck(*args)
 
       private
 
+      def where_with_hash(conditions)
+        detector = NonexistentFieldsDetector.new(conditions, @source)
+        if detector.found?
+          Dynamoid.logger.warn(detector.warning_message)
+        end
+
+        @where_conditions.update_with_hash(conditions.symbolize_keys)
+
+        # we should re-initialize keys detector every time we change @where_conditions
+        @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name)
+
+        self
+      end
+
+      def where_with_string(query, placeholders)
+        @where_conditions.update_with_string(query, placeholders)
+
+        # we should re-initialize keys detector every time we change @where_conditions
+        @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name)
+
+        self
+      end
+
       # The actual records referenced by the association.
       #
       # @return [Enumerator] an iterator of the found records.
@@ -635,12 +665,12 @@ def query_key_conditions
       end
 
       def query_non_key_conditions
-        opts = {}
+        hash_conditions = {}
 
         # Honor STI and :type field if it presents
         if @source.attributes.key?(@source.inheritance_field) &&
            @key_fields_detector.hash_key.to_sym != @source.inheritance_field.to_sym
-          @where_conditions.update(sti_condition)
+          @where_conditions.update_with_hash(sti_condition)
         end
 
         # TODO: Separate key conditions and non-key conditions properly:
@@ -650,11 +680,17 @@ def query_non_key_conditions
           .reject { |k, _| k.to_s =~ /^#{@key_fields_detector.range_key}\./ }
         keys.each do |key|
           name, condition = field_condition(key, @where_conditions[key])
-          opts[name] ||= []
-          opts[name] << condition
+          hash_conditions[name] ||= []
+          hash_conditions[name] << condition
         end
 
-        opts
+        string_conditions = []
+        @where_conditions.string_conditions.each do |query, placeholders|
+          placeholders ||= {}
+          string_conditions << [query, placeholders]
+        end
+
+        [hash_conditions] + string_conditions
       end
 
       # TODO: casting should be operator aware
@@ -721,16 +757,25 @@ def query_options
       def scan_conditions
         # Honor STI and :type field if it presents
         if sti_condition
-          @where_conditions.update(sti_condition)
+          @where_conditions.update_with_hash(sti_condition)
         end
 
-        {}.tap do |opts|
+        hash_conditions = {}
+        hash_conditions.tap do |opts|
           @where_conditions.keys.map(&:to_sym).each do |key|
             name, condition = field_condition(key, @where_conditions[key])
             opts[name] ||= []
             opts[name] << condition
           end
         end
+
+        string_conditions = []
+        @where_conditions.string_conditions.each do |query, placeholders|
+          placeholders ||= {}
+          string_conditions << [query, placeholders]
+        end
+
+        [hash_conditions] + string_conditions
       end
 
       def scan_options

lib/dynamoid/criteria/where_conditions.rb

@@ -4,24 +4,31 @@ module Dynamoid
   module Criteria
     # @private
     class WhereConditions
+      attr_reader :string_conditions
+
       def initialize
-        @conditions = []
+        @hash_conditions = []
+        @string_conditions = []
+      end
+
+      def update_with_hash(hash)
+        @hash_conditions << hash.symbolize_keys
       end
 
-      def update(hash)
-        @conditions << hash.symbolize_keys
+      def update_with_string(query, placeholders)
+        @string_conditions << [query, placeholders]
       end
 
       def keys
-        @conditions.flat_map(&:keys)
+        @hash_conditions.flat_map(&:keys)
       end
 
       def empty?
-        @conditions.empty?
+        @hash_conditions.empty? && @string_conditions.empty?
       end
 
       def [](key)
-        hash = @conditions.find { |h| h.key?(key) }
+        hash = @hash_conditions.find { |h| h.key?(key) }
         hash[key] if hash
       end
     end