feat: add configurable timeout for API requests

Author: crawlbase

README.md

@@ -34,6 +34,12 @@ You can get a token for free by [creating a Crawlbase account](https://crawlbase
 api = Crawlbase::API.new(token: 'YOUR_TOKEN')
 ```
 
+By default, the timeout for API requests is set to 90 seconds. You can configure a custom timeout by passing a `timeout` option during initialization.
+
+```ruby
+api = Crawlbase::API.new(token: 'YOUR_TOKEN', timeout: 120)
+```
+
 ### GET requests
 
 Pass the url that you want to scrape plus any options from the ones available in the [API documentation](https://crawlbase.com/dashboard/docs).
@@ -357,7 +363,7 @@ The gem is available as open source under the terms of the [MIT License](http://
 
 ## Code of Conduct
 
-Everyone interacting in the Crawlbase project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/crawlbase-source/crawlbase-ruby/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Crawlbase project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/crawlbase-source/crawlbase-ruby/blob/master/CODE_OF_CONDUCT.md).
 
 ---
 

crawlbase.gemspec

@@ -24,6 +24,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "rexml"
+
   spec.add_development_dependency "rspec", "~> 3.2"
   spec.add_development_dependency "webmock", "~> 3.4"
   spec.add_development_dependency "bundler", "~> 2.0"

lib/crawlbase/api.rb

@@ -6,23 +6,26 @@
 
 module Crawlbase
   class API
-    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url
+    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url, :timeout
 
     INVALID_TOKEN = 'Token is required'
     INVALID_URL = 'URL is required'
+    DEFAULT_TIMEOUT = 90
 
     def initialize(options = {})
       raise INVALID_TOKEN if options[:token].nil?
 
       @token = options[:token]
+      @timeout = options.fetch(:timeout, DEFAULT_TIMEOUT)
     end
 
     def get(url, options = {})
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
-
-      response = Net::HTTP.get_response(uri)
+      http = build_http(uri)
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
 
       prepare_response(response, options[:format])
 
@@ -33,10 +36,7 @@ def post(url, data, options = {})
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-
-      http.use_ssl = true
+      http = build_http(uri)
 
       content_type = options[:post_content_type].to_s.include?('json') ? { 'Content-Type': 'text/json' } : nil
 
@@ -57,6 +57,14 @@ def post(url, data, options = {})
 
     private
 
+    def build_http(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = @timeout
+      http.read_timeout = @timeout
+      http
+    end
+
     def base_url
       'https://api.crawlbase.com'
     end

spec/api_spec.rb

@@ -10,6 +10,14 @@
     expect(Crawlbase::API.new(token: 'test').token).to eql('test')
   end
 
+  it 'sets default timeout to 90 seconds' do
+    expect(Crawlbase::API.new(token: 'test').timeout).to eql(90)
+  end
+
+  it 'sets a custom timeout' do
+    expect(Crawlbase::API.new(token: 'test', timeout: 120).timeout).to eql(120)
+  end
+
   describe '#get' do
     it 'sends an get request to Crawlbase API' do
       stub_request(:get, 'https://api.crawlbase.com/?token=test&url=http%3A%2F%2Fhttpbin.org%2Fanything%3Fparam1%3Dx%26params2%3Dy').
@@ -28,6 +36,14 @@
       expect(response.url).to eql('http://httpbin.org/anything?param1=x&params2=y')
       expect(response.body).to eql('body')
     end
+
+    it 'raises a timeout error' do
+      stub_request(:get, 'https://api.crawlbase.com/?token=test_with_timeout&url=http%3A%2F%2Fhttpbin.org%2Fdelay%2F3').to_timeout
+
+      api = Crawlbase::API.new(token: 'test_with_timeout', timeout: 2)
+
+      expect { api.get('http://httpbin.org/delay/3') }.to raise_error(Net::OpenTimeout)
+    end
   end
 
   describe '#post' do
@@ -68,6 +84,14 @@
       expect(response.url).to eql('http://httpbin.org/anything?param1=x&params2=y')
       expect(response.body).to eql('body')
     end
+
+    it 'raises a timeout error' do
+      stub_request(:post, 'https://api.crawlbase.com/?token=test_with_timeout&url=http%3A%2F%2Fhttpbin.org%2Fdelay%2F3').to_timeout
+
+      api = Crawlbase::API.new(token: 'test_with_timeout', timeout: 2)
+
+      expect { api.post('http://httpbin.org/delay/3', {}) }.to raise_error(Net::OpenTimeout)
+    end
   end
 
-end
+end