diff --git a/Gemfile b/Gemfile index 4485607d..a9ef2807 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,9 @@ source "https://rubygems.org" ruby "2.7.7" +# Pin ffi to version compatible with Ruby 2.7.7 and RubyGems 3.1.6 +gem 'ffi', '< 1.17.0' + gem "sprockets" gem "rubyzip", ">= 1.2.2" @@ -17,7 +20,7 @@ gem 'activesupport', '6.0.0' gem 'actionpack', '6.0.0' gem 'rails-html-sanitizer' gem "rack-timeout" -gem "faraday" +gem "faraday", "~> 2.7" gem 'faraday-retry' gem "autoprefixer-rails" gem "flutie" @@ -46,6 +49,8 @@ gem 'airbrake' gem 'parallel' gem 'ruby-progressbar' gem "aws-sdk-s3", require: false +# Faraday adapter using libcurl (HTTP/2 support, bypasses WAF/bot detection) +gem 'faraday-typhoeus' # deployment to server gem 'capistrano', '~> 3.8' diff --git a/Gemfile.lock b/Gemfile.lock index 0752de7a..5001719c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -86,6 +86,7 @@ GEM aws-sigv4 (~> 1.4) aws-sigv4 (1.4.0) aws-eventstream (~> 1, >= 1.0.2) + base64 (0.3.0) builder (3.2.4) bullet (7.0.1) activesupport (>= 3.0.0) @@ -131,18 +132,25 @@ GEM enumerize (2.5.0) activesupport (>= 3.2) erubi (1.10.0) + ethon (0.15.0) + ffi (>= 1.15.0) execjs (2.8.1) factory_bot (6.2.0) activesupport (>= 5.0.0) factory_bot_rails (6.2.0) factory_bot (~> 6.2.0) railties (>= 5.0.0) - faraday (2.1.0) - faraday-net_http (~> 2.0) + faraday (2.8.1) + base64 + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-net_http (2.0.1) + faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) + faraday-typhoeus (1.1.0) + faraday (~> 2.0) + typhoeus (~> 1.4) + ffi (1.16.3) flutie (2.2.0) font-awesome-rails (4.7.0.8) railties (>= 3.2, < 8.0) @@ -355,6 +363,8 @@ GEM thread_safe (0.3.6) tilt (2.0.10) timecop (0.9.4) + typhoeus (1.5.0) + ethon (>= 0.9.0, < 0.16.0) tzinfo (1.2.10) thread_safe (~> 0.1) unf (0.1.4) @@ -395,8 +405,10 @@ DEPENDENCIES enumerize execjs factory_bot_rails - faraday + faraday (~> 2.7) faraday-retry + faraday-typhoeus + ffi (< 1.17.0) flutie font-awesome-rails formulaic diff --git a/app/services/http_client.rb b/app/services/http_client.rb index 8331f7de..7b21616b 100644 --- a/app/services/http_client.rb +++ b/app/services/http_client.rb @@ -23,7 +23,8 @@ def setup_connection(base_url, timeout: 10, open_timeout: 5, retries: 3) f.response :raise_error # 40x, 50x errors -> Faraday::ClientError, Faraday::ServerError # convert JSON string to Ruby hash inside body if header is application/json f.response :json # fails with Faraday::ParsingError - f.adapter Faraday.default_adapter # keep for clarity + # Use Typhoeus adapter with HTTP/2 support to mimic curl behavior + f.adapter :typhoeus, http_version: :httpv2_0 f.options.timeout = timeout # max timeout for request f.options.open_timeout = open_timeout # for connection to open end diff --git a/spec/mailers/notifier_spec.rb b/spec/mailers/notifier_spec.rb index 911aff78..fe7ba306 100644 --- a/spec/mailers/notifier_spec.rb +++ b/spec/mailers/notifier_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Notifier, type: :mailer do - let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.1.0' } } + let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.8.1' } } let(:study_statistics_body) {File.read('spec/support/json_data/study_statistics.json') } let(:mailer_stub) { stub_request(:get, 'https://classic.clinicaltrials.gov/api//info/study_statistics?fmt=json').with(headers: stub_request_headers).to_return(:status => 200, :body => study_statistics_body, :headers => {}) } let(:msg) { described_class.send_msg('test@gmail.com', event) } diff --git a/spec/models/clinical_trials_api_v2_spec.rb b/spec/models/clinical_trials_api_v2_spec.rb index 5764e1ef..3160e390 100644 --- a/spec/models/clinical_trials_api_v2_spec.rb +++ b/spec/models/clinical_trials_api_v2_spec.rb @@ -8,7 +8,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: '[{"nctId": "NCT123456", "title": "Sample Study"}]', headers: {}) @@ -27,7 +27,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: '{"nctId": "NCT123456", "title": "Sample Study"}', headers: {}) @@ -44,7 +44,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: '[{"field": "sampleField"}]', headers: {}) @@ -61,7 +61,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: '{"size": 100}', headers: {}) @@ -78,7 +78,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: '{"fieldValues": ["value1", "value2"]}', headers: {}) @@ -96,7 +96,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: "{\"field\": \"#{field}\", \"values\": [\"value1\", \"value2\"]}", headers: {}) @@ -114,7 +114,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: '{"listSizes": [10, 20]}', headers: {}) @@ -132,7 +132,7 @@ headers: { 'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'User-Agent'=>'Faraday v2.1.0' + 'User-Agent'=>'Faraday v2.8.1' }). to_return(status: 200, body: "{\"field\": \"#{field}\", \"size\": 5}", headers: {}) diff --git a/spec/models/utils/util_updater_spec.rb b/spec/models/utils/util_updater_spec.rb index 7e6fbf92..b146697f 100644 --- a/spec/models/utils/util_updater_spec.rb +++ b/spec/models/utils/util_updater_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' describe Util::Updater do - let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.1.0' } } + let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.8.1' } } let(:ctg_api_body) {File.read('spec/support/json_data/ctg_api_all.json') } let(:api_url) { 'https://classic.clinicaltrials.gov/api//query/study_fields?fields=NCTId,StudyFirstPostDate,LastUpdatePostDate&fmt=json&max_rnk=1000&min_rnk=1'} before do diff --git a/spec/services/http_client_spec.rb b/spec/services/http_client_spec.rb index 5b40d733..8b2c38e2 100644 --- a/spec/services/http_client_spec.rb +++ b/spec/services/http_client_spec.rb @@ -99,7 +99,7 @@ expect { api_client.get(endpoint, params) - }.to raise_error(Faraday::ConnectionFailed) # final error after retries + }.to raise_error(Faraday::TimeoutError) # final error after retries expect(WebMock).to have_requested(:get, url) .with(query: params)