add next_offset pagination strategy

Author: 10xSebastian

README.md

@@ -1386,6 +1386,51 @@ Sequentially:
   GET https://service.example.com/records?from_record_id=xcaoXBmuMyFFEcFDSgNgDQ&limit=100
 ```
 
+##### Pagination strategy: next_offset
+
+The `next_offset` strategy continuously follows in-response offset information to following pages until the last page is reached (indicated by next offset being either empty or 0).
+
+*WARNING*
+
+Loading all pages from a resource paginated with next_offset only can result in very poor performance, as pages can only be loaded sequentially!
+
+```ruby
+# app/models/record.rb
+
+class Search < DHS::Record
+  configuration pagination_strategy: 'next_offset'
+
+  endpoint '{+service}/assets'
+end
+```
+
+```ruby
+# app/controllers/some_controller.rb
+
+Record.all
+
+```
+```
+GET https://service.example.com/assets?limit=100
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 29
+}
+GET https://service.example.com/assets?offset=29
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 39
+}
+GET https://service.example.com/assets?offset=39
+{
+  items: [{...}, ...],
+  limit: 10,
+  next_offset: 0
+}
+```
+
 #### Pagination keys
 
 ##### limit_key

lib/dhs.rb

@@ -39,6 +39,7 @@ module Pagination
     autoload :Page, 'dhs/pagination/page'
     autoload :TotalPages, 'dhs/pagination/total_pages'
     autoload :OffsetPage, 'dhs/pagination/offset_page'
+    autoload :NextOffset, 'dhs/pagination/next_offset'
     autoload :Start, 'dhs/pagination/start'
     autoload :Link, 'dhs/pagination/link'
   end

lib/dhs/concerns/record/pagination.rb

@@ -30,6 +30,8 @@ def pagination_class
           DHS::Pagination::Start
         when :link
           DHS::Pagination::Link
+        when :next_offset
+          DHS::Pagination::NextOffset
         else
           DHS::Pagination::Offset
         end
@@ -43,7 +45,8 @@ def pagination(data)
       def paginated?(raw)
         raw.is_a?(Hash) && (
           raw.dig(*total_key).present? ||
-          raw.dig(*limit_key(:body)).present?
+          raw.dig(*limit_key(:body)).present? ||
+          raw.dig(*pagination_key(:body)).present?
         )
       end
     end

lib/dhs/concerns/record/request.rb

@@ -237,7 +237,7 @@ def load_and_merge_paginated_collection!(data, options)
         if pagination.parallel?
           load_and_merge_parallel_requests!(record, data, pagination, options)
         else
-          load_and_merge_sequential_requests!(record, data, options, data._raw.dig(:next, :href), pagination)
+          load_and_merge_sequential_requests!(record, data, options, pagination)
         end
       end
 
@@ -249,13 +249,16 @@ def load_and_merge_parallel_requests!(record, data, pagination, options)
         end
       end
 
-      def load_and_merge_sequential_requests!(record, data, options, next_link, pagination)
-        warn '[WARNING] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! (https://github.com/DePayFi/dhs#pagination-strategy-link).'
-        while next_link.present?
-          page_data = record.request(
-            options.except(:all).merge(url: next_link)
-          )
-          next_link = page_data._raw.dig(:next, :href)
+      def load_and_merge_sequential_requests!(record, data, options, pagination)
+        warn '[WARNING] You are loading all pages from a resource paginated with sequential pagination.'
+        next_value = pagination.next(data._raw)
+        while next_value.present?
+          page_data = if next_value.is_a?(String) && next_value.match(/^http/)
+             record.request(options.except(:all).merge(url: next_value))
+          else
+            record.request(options.except(:all).merge(params: (options.dig(:params) || {}).merge(next_value) ))
+          end
+          next_value = pagination.next(page_data._raw)
           merge_batch_data_with_parent!(page_data, data, pagination)
         end
       end

lib/dhs/pagination/link.rb

@@ -7,6 +7,10 @@ def total
 
   alias count total
 
+  def next(current)
+    current.dig(:next, :href)
+  end
+
   def pages_left
     pages_left? ? 1 : 0
   end

lib/dhs/pagination/next_offset.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class DHS::Pagination::NextOffset < DHS::Pagination::Base
+
+  DEFAULT_OFFSET = 0
+  
+  def total
+    data._raw.dig(*_record.items_key).count || 0
+  end
+  alias count total
+
+  def parallel?
+    false
+  end
+
+  def pages_left?
+    next_offset = data._raw.dig(*_record.pagination_key(:body))
+    next_offset.present? && !next_offset.zero?
+  end
+
+  def next(current)
+    next_value = current.dig(*_record.pagination_key(:body))
+    return if next_value.blank? || next_value.zero?
+    {
+      _record.pagination_key(:parameter) => current.dig(*_record.pagination_key(:body))
+    }
+  end
+end

lib/dhs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DHS
-  VERSION = '1.3.0'
+  VERSION = '1.4.0'
 end

spec/graphql/main_spec.rb

@@ -88,11 +88,13 @@
   end
 
   before do
+    DHC.config.placeholder('bitquery', 'https://graphql.bitquery.io/')
+
     class Record < DHS::Record
 
       configuration items_key: [:data, :ethereum, :address, 0, :balances]
 
-      endpoint 'https://graphql.bitquery.io/',
+      endpoint '{+bitquery}',
         graphql: {
           query: %{
             query ($network: EthereumNetwork!, $address: String!) {

spec/pagination/next_offset_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe DHS::Record do
+
+  context 'pagination' do
+
+    def stub_api_request(items: [], offset: nil, next_offset:)
+      stub_request(:get, ["http://depay.fi/v2/transactions?limit=100", offset ? "offset=#{offset}" : nil].compact.join('&'))
+        .to_return(body: { items: items, next_offset: next_offset }.to_json)
+    end
+
+    let!(:requests) do
+      stub_api_request(items: (0...100).to_a, next_offset: 99)
+      stub_api_request(items: (100...200).to_a, offset: 99, next_offset: 199)
+      stub_api_request(items: (200...300).to_a, offset: 199, next_offset: 0)
+    end
+
+    before do
+      class Transaction < DHS::Record
+        configuration pagination_strategy: :next_offset, pagination_key: { body: :next_offset, parameter: :offset }
+
+        endpoint 'http://depay.fi/v2/transactions'
+      end
+    end
+
+    it 'fetches all the pages' do
+      transactions = Transaction.all.fetch
+      expect(transactions.to_a).to eq (0...300).to_a
+    end
+  end
+end

spec/record/includes_spec.rb

@@ -167,7 +167,7 @@ def additional_products_request(offset, amount)
           .includes(:users, contracts: :products)
           .find(1)
       end).to output(
-        %r{\[WARNING\] You are loading all pages from a resource paginated with links only. As this is performed sequentially, it can result in very poor performance! \(https://github.com/DePayFi/dhs#pagination-strategy-link\).}
+        %r{\[WARNING\] You are loading all pages from a resource paginated with sequential pagination.}
       ).to_stderr
 
       expect(customer.users.length).to eq amount_of_users