Merge pull request #1239 from ctti-clinicaltrials/dev

akabishau · web-flow · commit 16ec282cdf22 · 2025-10-20T19:01:45.000-04:00
Fix CTGov API Connection Issue
diff --git a/Gemfile b/Gemfile
@@ -2,6 +2,9 @@ source "https://rubygems.org"
 
 ruby "2.7.7"
 
+# Pin ffi to version compatible with Ruby 2.7.7 and RubyGems 3.1.6
+gem 'ffi', '< 1.17.0'
+
 gem "sprockets"
 gem "rubyzip", ">= 1.2.2"
 
@@ -17,7 +20,7 @@ gem 'activesupport', '6.0.0'
 gem 'actionpack', '6.0.0'
 gem 'rails-html-sanitizer'
 gem "rack-timeout"
-gem "faraday"
+gem "faraday", "~> 2.7"
 gem 'faraday-retry'
 gem "autoprefixer-rails"
 gem "flutie"
@@ -46,6 +49,8 @@ gem 'airbrake'
 gem 'parallel'
 gem 'ruby-progressbar'
 gem "aws-sdk-s3", require: false
+# Faraday adapter using libcurl (HTTP/2 support, bypasses WAF/bot detection)
+gem 'faraday-typhoeus'
 
 # deployment to server
 gem 'capistrano', '~> 3.8'
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -86,6 +86,7 @@ GEM
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.3.0)
     builder (3.2.4)
     bullet (7.0.1)
       activesupport (>= 3.0.0)
@@ -131,18 +132,25 @@ GEM
     enumerize (2.5.0)
       activesupport (>= 3.2)
     erubi (1.10.0)
+    ethon (0.15.0)
+      ffi (>= 1.15.0)
     execjs (2.8.1)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.2.0)
       factory_bot (~> 6.2.0)
       railties (>= 5.0.0)
-    faraday (2.1.0)
-      faraday-net_http (~> 2.0)
+    faraday (2.8.1)
+      base64
+      faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-net_http (2.0.1)
+    faraday-net_http (3.0.2)
     faraday-retry (2.2.1)
       faraday (~> 2.0)
+    faraday-typhoeus (1.1.0)
+      faraday (~> 2.0)
+      typhoeus (~> 1.4)
+    ffi (1.16.3)
     flutie (2.2.0)
     font-awesome-rails (4.7.0.8)
       railties (>= 3.2, < 8.0)
@@ -355,6 +363,8 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.10)
     timecop (0.9.4)
+    typhoeus (1.5.0)
+      ethon (>= 0.9.0, < 0.16.0)
     tzinfo (1.2.10)
       thread_safe (~> 0.1)
     unf (0.1.4)
@@ -395,8 +405,10 @@ DEPENDENCIES
   enumerize
   execjs
   factory_bot_rails
-  faraday
+  faraday (~> 2.7)
   faraday-retry
+  faraday-typhoeus
+  ffi (< 1.17.0)
   flutie
   font-awesome-rails
   formulaic
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 # AACT
 
-AACT or Aggregated Analysis of ClinicalTrials.gov is an application for retrieving all of the public data from <a href="clinicaltrials.gov" target="_blank">clinicaltrials.gov</a> (via their API) and make the information available in a relational database.  We do this to make this valuable body of information accessible to the public as a complete aggregated set of data.
+AACT or Aggregated Analysis of ClinicalTrials.gov is an application for retrieving all of the public data from ClinicalTrials.gov (via their API) and makes the information available in a relational database.  We do this to provide this valuable body of information accessible to the public as a complete aggregated set of data.
 
-If you need a copy of the database, but don't want to bother installing & running this app, copies of the database are available for download from the <a href='https://aact.ctti-clinicaltrials.org/snapshots' target='_blank'>AACT website (Download page).</a> We use pg_dump to create a snapshot of the database after each nightly update, so a version is always available with the most current info from clinicaltrials.gov.  
+If you need a copy of the database, but don't want to install and run this app, copies of the database are available for download from the <a href='https://aact.ctti-clinicaltrials.org/downloads' target='_blank'>AACT website (Download page).</a> We use pg_dump to create a snapshot of the database after each nightly update, so a version is always available with the most current info from clinicaltrials.gov.  
 
 ### Database Explanation:  
 
@@ -249,4 +249,4 @@ for example:
   rake compare:v2_model[NCT06225401,design]
 ```
 
-This tool will tell you where there is a mismatch between version 1 and version 2 of the api
+This tool will tell you where there is a mismatch between version 1 and version 2 of the api
diff --git a/app/services/http_client.rb b/app/services/http_client.rb
@@ -23,7 +23,8 @@ def setup_connection(base_url, timeout: 10, open_timeout: 5, retries: 3)
       f.response :raise_error # 40x, 50x errors -> Faraday::ClientError, Faraday::ServerError
       # convert JSON string to Ruby hash inside body if header is application/json
       f.response :json # fails with Faraday::ParsingError
-      f.adapter Faraday.default_adapter # keep for clarity
+      # Use Typhoeus adapter with HTTP/2 support to mimic curl behavior
+      f.adapter :typhoeus, http_version: :httpv2_0
       f.options.timeout = timeout # max timeout for request
       f.options.open_timeout = open_timeout # for connection to open
     end
diff --git a/spec/mailers/notifier_spec.rb b/spec/mailers/notifier_spec.rb
@@ -2,7 +2,7 @@
 require 'spec_helper'
 
 describe Notifier, type: :mailer do
-  let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.1.0' } }
+  let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.8.1' } }
   let(:study_statistics_body) {File.read('spec/support/json_data/study_statistics.json') }
   let(:mailer_stub) { stub_request(:get, 'https://classic.clinicaltrials.gov/api//info/study_statistics?fmt=json').with(headers: stub_request_headers).to_return(:status => 200, :body => study_statistics_body, :headers => {}) }
   let(:msg) { described_class.send_msg('test@gmail.com', event) }
diff --git a/spec/models/clinical_trials_api_v2_spec.rb b/spec/models/clinical_trials_api_v2_spec.rb
@@ -8,7 +8,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: '[{"nctId": "NCT123456", "title": "Sample Study"}]', headers: {})
 
@@ -27,7 +27,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: '{"nctId": "NCT123456", "title": "Sample Study"}', headers: {})
 
@@ -44,7 +44,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: '[{"field": "sampleField"}]', headers: {})
 
@@ -61,7 +61,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: '{"size": 100}', headers: {})
 
@@ -78,7 +78,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: '{"fieldValues": ["value1", "value2"]}', headers: {})
 
@@ -96,7 +96,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: "{\"field\": \"#{field}\", \"values\": [\"value1\", \"value2\"]}", headers: {})
 
@@ -114,7 +114,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: '{"listSizes": [10, 20]}', headers: {})
 
@@ -132,7 +132,7 @@
           headers: {
             'Accept'=>'*/*',
             'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
-            'User-Agent'=>'Faraday v2.1.0'
+            'User-Agent'=>'Faraday v2.8.1'
           }).
         to_return(status: 200, body: "{\"field\": \"#{field}\", \"size\": 5}", headers: {})
 
diff --git a/spec/models/utils/util_updater_spec.rb b/spec/models/utils/util_updater_spec.rb
@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 describe Util::Updater do
-  let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.1.0' } }
+  let(:stub_request_headers) { {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Faraday v2.8.1' } }
   let(:ctg_api_body) {File.read('spec/support/json_data/ctg_api_all.json') }
   let(:api_url) { 'https://classic.clinicaltrials.gov/api//query/study_fields?fields=NCTId,StudyFirstPostDate,LastUpdatePostDate&fmt=json&max_rnk=1000&min_rnk=1'}
   before do
diff --git a/spec/services/http_client_spec.rb b/spec/services/http_client_spec.rb
@@ -99,7 +99,7 @@
 
         expect {
           api_client.get(endpoint, params)
-        }.to raise_error(Faraday::ConnectionFailed) # final error after retries
+        }.to raise_error(Faraday::TimeoutError) # final error after retries
 
         expect(WebMock).to have_requested(:get, url)
                           .with(query: params)
