Merge branch 'master' of github.com:ctti-clinicaltrials/aact

Author: micronix

app/models/search_result.rb

@@ -32,10 +32,12 @@ def self.study_values(study)
     study_nct_id = study.nct_id
     id_values = study.id_information.pluck(:id_value).join('|')
     sponsors = study.sponsors
-    grouped = sponsors.group_by(&:lead_or_collaborator)
-    lead = grouped['lead'].first
-    collaborators = grouped['collaborator']
-    collab_names = collaborators.map{|collab| "#{collab.name}[#{collab.agency_class}]"}.join('|') if collaborators
+    unless sponsors.blank?
+      grouped = sponsors.group_by(&:lead_or_collaborator)
+      lead = grouped['lead'].first
+      collaborators = grouped['collaborator']
+      collab_names = collaborators.map{|collab| "#{collab.name}[#{collab.agency_class}]"}.join('|') if collaborators
+    end
     interventions = study.interventions
     intervention_name_type = []
     intervention_details = []
@@ -105,9 +107,9 @@ def self.study_values(study)
       study.why_stopped, #why_stopped
       hcq_query(study) ? 'Yes' : 'No', #hcq
       study.has_dmc ? 'Yes' : 'No', #has_dmc
-      sponsors.pluck(:agency_class).uniq.join('|'), #funded_bys
-      sponsors.pluck(:name).join('|'), #sponsor_collaborators
-      lead ? "#{lead.name}[#{lead.agency_class}]" : nil, #lead_sponsor
+      sponsors.blank? ? '' : sponsors.pluck(:agency_class).uniq.join('|'), #funded_bys
+      sponsors.blank? ? '' : sponsors.pluck(:name).join('|'), #sponsor_collaborators
+      lead.blank? ? nil : "#{lead.name}[#{lead.agency_class}]", #lead_sponsor
       collab_names, #collaborators
       study.study_type, #study_type
       study.phase.try(:split, '/').try(:join, '|'), #phases

app/models/study_json_record.rb

@@ -73,27 +73,47 @@ def self.download_all_studies(url='https://ClinicalTrials.gov/AllAPIJSON.zip')
   end
 
   def self.full
-    start_time = Time.current
+    total_time = 0
     study_download = download_all_studies
     nct_ids = StudyJsonRecord.all.map(&:nct_id)
+    remove_indexes_and_constraints
     clear_out_data_for(nct_ids)
 
     Zip::File.open(study_download.path) do |unzipped_folders|
       original_count = unzipped_folders.size
       @count_down = original_count
       unzipped_folders.each do |file|
         begin
-        contents = file.get_input_stream.read
-        json = JSON.parse(contents)
-        study = json['FullStudy']
-        save_single_study(study)
+          unless file.name =~/contents/i
+            start = Time.now
+
+            contents = file.get_input_stream.read
+            json = JSON.parse(contents)
+            study = json['FullStudy']
+            save_single_study(study)
+
+            duration = start - Time.now
+            total_time += duration
+            
+            puts "#{@count_down -= 1}, took #{htime(duration)}, total time so far #{htime(total_time)}, Study Count: #{Study.count}"
+          end
         rescue Exception => error
           msg="#{error.message} (#{error.class} #{error.backtrace}"
           ErrorLog.error(msg)
           Airbrake.notify(error)
         end  
       end
     end
+    add_indexes_and_constraints
+  end
+
+  def self.htime(seconds)
+    seconds = seconds.to_i
+    hours = seconds / 3600
+    seconds -= hours * 3600
+    minutes = seconds / 60
+    seconds -= minutes * 60
+    "#{hours}:#{'%02i' % minutes}:#{'%02i' % seconds}"
   end
 
   def self.incremental
@@ -1648,7 +1668,9 @@ def save_design_groups(design_groups)
   def save_with_result_group(group, name_of_model='BaselineMeasurement')
     return unless group
 
-    group.each{|i| i[:result_group_id] = @study_result_groups[i[:ctgov_group_code]]}
+    group.map do |i| 
+      i[:result_group_id] = @study_result_groups[i[:ctgov_group_code]].try(:id)
+    end
     name_of_model.safe_constantize.import(group, validate: false)
   end
 

app/models/study_search.rb

@@ -59,10 +59,14 @@ def load_update(days_back=2)
 
   def self.execute(days_back=2)
     queries = all
-    queries.each do |query|
-      print "running query group: #{query.grouping}..."
-      query.load_update(days_back)
-      puts "group is done"
+    begin
+      queries.each do |query|
+        print "running query group: #{query.grouping}..."
+        query.load_update(days_back)
+        puts "group is done"
+      end
+    rescue => e
+      Airbrake.notify(e)
     end
   end
 

app/models/util/db_manager.rb

@@ -48,8 +48,14 @@ def dump_database
     # 3. restore teh db from file
     # 4. verify the study count (permissions are not granted again to prevent bad data from being used)
     # 5. grant connection permissions again
-    def restore_database(schema_type, connection, filename)
-      schema = 'ctgov'
+    def restore_database(schema, connection, filename)
+      if schema =~ /beta/
+        schema = 'ctgov_beta'
+      elsif schema =~ /archive/
+        schema = 'ctgov_archive'
+      else
+        schema = 'ctgov'
+      end
       config = connection.instance_variable_get('@config')
       host, port, username, database, password = config[:host], config[:port], config[:username], config[:database], config[:password]
 
@@ -209,7 +215,7 @@ def add_constraints
           migration.add_foreign_key child_table,  parent_table, column: child_column, primary_key: parent_column, name: "#{child_table}_#{child_column}_fkey"
         rescue => e
           log(e)
-          event.add_problem("#{Time.zone.now}: #{e}")
+          event.add_problem("#{Time.zone.now}: #{e}") if event
         end
       }
     end
@@ -221,15 +227,15 @@ def remove_indexes_and_constraints
           con.remove_foreign_key table_name, column: :nct_id if con.foreign_keys(table_name).map(&:column).include?("nct_id")
         rescue => e
           log(e)
-          event.add_problem("#{Time.zone.now}: #{e}")
+          event.add_problem("#{Time.zone.now}: #{e}") if event
         end
 
         con.indexes(table_name).each{|index|
           begin
             migration.remove_index(index.table, index.columns) if !should_keep_index?(index) and migration.index_exists?(index.table, index.columns)
           rescue => e
             log(e)
-            event.add_problem("#{Time.zone.now}: #{e}")
+            event.add_problem("#{Time.zone.now}: #{e}") if event
           end
         }
       }
@@ -241,7 +247,7 @@ def remove_indexes_and_constraints
           con.remove_foreign_key table, column: column if con.foreign_keys(table).map(&:column).include?(column)
         rescue => e
           log(e)
-          event.add_problem("#{Time.zone.now}: #{e}")
+          event.add_problem("#{Time.zone.now}: #{e}") if event
         end
       }
     end
@@ -253,7 +259,7 @@ def remove_constrains
           con.remove_foreign_key table_name, column: :nct_id if con.foreign_keys(table_name).map(&:column).include?("nct_id")
         rescue => e
           log(e)
-          event.add_problem("#{Time.zone.now}: #{e}")
+          event.add_problem("#{Time.zone.now}: #{e}") if event
         end
       }
 
@@ -264,7 +270,7 @@ def remove_constrains
           con.remove_foreign_key table, column: column if con.foreign_keys(table).map(&:column).include?(column)
         rescue => e
           log(e)
-          event.add_problem("#{Time.zone.now}: #{e}")
+          event.add_problem("#{Time.zone.now}: #{e}") if event
         end
       }
     end
@@ -284,6 +290,7 @@ def self.loadable_tables
         study_json_records
         use_cases
         use_case_attachments
+        verifiers
       )
       table_names=con.tables.reject{|table|blacklist.include?(table)}
     end

app/models/util/table_exporter.rb

@@ -6,7 +6,6 @@ def initialize(tables=[],schema='')
       @schema = schema
       @temp_dir     = "#{Util::FileManager.new.dump_directory}/export"
       @zipfile_name = "#{@temp_dir}/#{Time.zone.now.strftime('%Y%m%d')}_export.zip"
-      @connection   = ActiveRecord::Base.connection.raw_connection
       @table_names  = tables
       create_temp_dir_if_none_exists!
     end
@@ -51,8 +50,9 @@ def create_tempfiles(delimiter)
 
     def export_table_to_csv(file, file_name, path, delimiter)
       table = File.basename(file_name, delimiter == ',' ? '.csv' : '.txt')
-      @connection.copy_data("copy #{table} to STDOUT with delimiter '#{delimiter}' csv header") do
-        while row = @connection.get_copy_data
+      connection  = ActiveRecord::Base.connection.raw_connection
+      connection.copy_data("copy #{table} to STDOUT with delimiter '#{delimiter}' csv header") do
+        while row = connection.get_copy_data
           # convert all \n to ~.  Then when you write to the file, convert last ~ back to \n
           # to prevent it from concatenating all rows into one big long string
           fixed_row=row.gsub(/\"\"/, '').gsub(/\n\s/, '~').gsub(/\n/, '~')

app/models/util/updater.rb

@@ -254,6 +254,7 @@ def run
                  end
         finalize_load if status != false
       rescue StandardError => e
+        Airbrake.notify(e)
         begin
           msg = "#{e.message} (#{e.class} #{e.backtrace}"
           log("#{@load_event.event_type} load failed in run: #{msg}")
@@ -269,17 +270,18 @@ def run
     end
 
     def full
-      if should_restart?
-        log('restarting full load...')
-      else
-        log('begin full load ...')
-        StudyJsonRecord.full
-      end
-      truncate_tables unless should_restart?
-      remove_indexes_and_constraints # Index significantly slow the load process. Will be re-created after data loaded.
-      study_counts[:should_add] = Support::StudyXmlRecord.not_yet_loaded.count
-      study_counts[:should_change] = 0
-      @client.populate_studies
+      start_time=Time.zone.now
+      log("storing study statistics data from ClinicalTrials.gov...")
+      verifier = Verifier.create(source: ClinicalTrialsApi.study_statistics.dig('StudyStatistics', "ElmtDefs", "Study"))
+      
+      log("begin full load, Start Time: #{start_time}...")
+      StudyJsonRecord.full
+      log("took #{time_ago_in_words(start_time)}")
+      
+      log("verififing study statistics match the aact database...")
+      verifier.verify(schema)
+      verifier.write_data_to_file(schema)
+
       MeshTerm.populate_from_file
       MeshHeading.populate_from_file
     end
@@ -318,7 +320,6 @@ def finalize_load
       log('finalizing load...')
 
       load_event.log('add indexes and constraints..')
-      add_indexes_and_constraints if params[:event_type] == 'full'
 
       load_event.log('execute study search...')
       days_back = (Date.today - Date.parse('2013-01-01')).to_i if load_event.event_type == 'full'
@@ -331,9 +332,6 @@ def finalize_load
         set_downcase_terms
       end
 
-      load_event.log('populate admin tables...')
-      # populate_admin_tables
-
       load_event.log('run sanity checks...')
       load_event.run_sanity_checks
 
@@ -354,8 +352,6 @@ def finalize_load
 
       load_event.log('create flat files...')
       create_flat_files(schema)
-
-      # Admin::PublicAnnouncement.clear_load_message
     end
 
     def remove_indexes_and_constraints