Merge pull request #1226 from ctti-clinicaltrials/feat-search-results

Studies by Search Term Results

Author: akabishau

app/models/search_term.rb

@@ -0,0 +1,15 @@
+class SearchTerm < ActiveRecord::Base
+  self.table_name = "ctgov.search_terms"
+  
+  before_save :normalize_term
+  has_many :search_term_results
+  validates :term, presence: true, uniqueness: true
+  attribute :group, :string, default: 'other'
+
+  private
+
+  def normalize_term
+    self.term = term.downcase
+    self.group = group.downcase
+  end
+end

app/models/search_term_result.rb

@@ -0,0 +1,10 @@
+class SearchTermResult < ActiveRecord::Base
+  self.table_name = "ctgov.search_term_results"
+  
+  belongs_to :search_term
+  belongs_to :study, foreign_key: :nct_id, primary_key: :nct_id
+  
+  validates :nct_id, presence: true
+  validates :search_term_id, presence: true
+  validates :nct_id, uniqueness: { scope: :search_term_id }
+end

app/models/study_relationship.rb

@@ -49,6 +49,8 @@ def self.blacklist
       ctgov_metadata_snapshots
       ctgov_mapping_snapshots
       ctgov_schema_snapshots
+      search_terms
+      search_term_results
     )
   end
 

app/services/ct_gov/api_client/v2.rb

@@ -14,14 +14,17 @@ def nct_id_path
       ["protocolSection", "identificationModule", "nctId"]
     end
 
-    def fetch_studies(range: nil, nct_ids: nil, page_size: nil)
+    # TODO: Make this method private eventually
+    def fetch_studies(range: nil, search: nil, nct_ids: nil, page_size: nil, nct_ids_only: false)
       page_token = nil
       total_count = 0
       total_fetched = 0
 
       params = { pageSize: page_size, countTotal: true }
       params["query.term"] = range
       params["filter.ids"] = nct_ids
+      params["query.cond"] = search
+      params["fields"] = "NCTId" if nct_ids_only
 
       loop do
         params[:pageToken] = page_token
@@ -64,6 +67,12 @@ def get_studies_by_nct_ids(list:, page_size: 50)
       end
     end
 
+    def search_studies(query:, page_size: nil)
+      fetch_studies(search: query, page_size: page_size, nct_ids_only: true) do |studies|
+        yield studies
+      end
+    end
+
     private
 
     def build_date_range_query(start_date:, end_date: nil)

app/services/ct_gov/search_results_service.rb

@@ -0,0 +1,65 @@
+module CTGov
+  class SearchResultsService
+    def initialize(api_client = CTGov::ApiClient::V2.new)
+      unless api_client.is_a?(CTGov::ApiClient::Base)
+        raise ArgumentError, "Invalid API client. Must inherit from CTGov::ApiClient::Base"
+      end
+      @api_client = api_client
+    end
+
+    def fetch_studies_for(search_term, group: nil, page_size: 1000)
+      puts "Fetching search results for: #{search_term}"
+
+      attributes = { term: search_term.downcase }
+      attributes[:group] = group if group.present?
+      search_term_record = SearchTerm.find_by(term: attributes[:term]) || SearchTerm.create!(attributes)
+
+      @api_client.search_studies(query: search_term, page_size: page_size) do |studies|
+        persist(studies, search_term_record)
+      end
+    end
+
+    def refresh_search_results_for(group)
+      raise ArgumentError, "Group parameter is required" if group.blank?
+
+      terms = SearchTerm.where(group: group.downcase)
+      Rails.logger.info("Refreshing #{terms.count} terms for group: #{group}")
+
+      terms.each do |term|
+        fetch_studies_for(term.term, group: term.group)
+      end
+
+    end
+
+    private
+
+    def persist(studies, search_term)
+      silence_active_record do
+        nct_ids = studies.map { |study| study.dig(*@api_client.nct_id_path) }.compact
+        valid_nct_ids = Study.where(nct_id: nct_ids).pluck(:nct_id)
+
+        study_results = valid_nct_ids.map do |nct_id|
+          {
+            nct_id: nct_id,
+            search_term_id: search_term.id,
+            created_at: Time.current,
+            updated_at: Time.current
+          }
+        end
+
+        # current logic: remove all existing search results for the term and insert fresh results
+        ActiveRecord::Base.transaction do
+          SearchTermResult.where(search_term_id: search_term.id).delete_all
+          return if study_results.empty?
+          SearchTermResult.insert_all(study_results)
+        end
+
+        Rails.logger.info("Imported #{study_results.size} search results for term: #{search_term.term}")
+      end
+    rescue => e
+      Rails.logger.error("Error persisting search results: #{e.message}")
+      raise
+    end
+  end
+
+end

db/migrate/20250110023732_create_search_results_tables.rb

@@ -0,0 +1,22 @@
+class CreateSearchResultsTables < ActiveRecord::Migration[6.0]
+  def change
+    create_table "ctgov.search_terms" do |t|
+      t.string :term, null: false
+      t.string :group, null: true
+      t.timestamps null: false
+    end
+
+    add_index "ctgov.search_terms", :term, unique: true, 
+              name: "index_ctgov_search_terms_on_term"
+
+    create_table "ctgov.search_term_results" do |t|
+      t.string :nct_id, null: false
+      t.references :search_term, foreign_key: { to_table: "ctgov.search_terms" }, null: false
+      t.timestamps null: false
+    end
+
+    add_index "ctgov.search_term_results", [:nct_id, :search_term_id], 
+              unique: true,
+              name: "index_ctgov_search_term_results_on_nct_id_and_search_term_id"
+  end
+end

lib/tasks/search_results.rake

@@ -0,0 +1,32 @@
+# require_relative 'application_record'
+require_relative '../../app/models/search_term_result'
+
+namespace :search_results do
+  desc 'Fetch and persist studies for a search term'
+  task :fetch, [:term, :group] => :environment do |t, args|
+    fail "Term is required. Group is optional" unless args[:term]
+
+    service = CTGov::SearchResultsService.new
+    begin
+      service.fetch_studies_for(args[:term], group: args[:group])
+      puts "✅ Successfully fetched studies for term: #{args[:term]}"
+    rescue => e
+      puts "❌ Error fetching studies: #{e.message}"
+      raise e
+    end
+  end
+
+  desc 'Refresh search results for a group'
+  task :refresh, [:group] => :environment do |t, args|
+    fail "Group is required" unless args[:group]
+
+    service = CTGov::SearchResultsService.new
+    begin
+      service.refresh_search_results_for(args[:group])
+      puts "✅ Successfully refreshed search results for group: #{args[:group]}"
+    rescue => e  
+      puts "❌ Error refreshing search results: #{e.message}"
+      raise e
+    end
+  end
+end