wip: save non matching data and errors #216

Author: hernanvicente

lib/autorequire/data_import.rb

@@ -187,17 +187,38 @@ def self.restore_default_users
   end
 
   def self.import_training_documents
-    csv_data = load_training_documents_csv
-    import_stats = initialize_import_stats
+    report = ImportReport.create!(
+      import_type: "training_documents",
+      started_at: Time.current,
+      status: "pending"
+    )
+
+    begin
+      csv_data = load_training_documents_csv
+      import_stats = initialize_import_stats
+
+      valid_csv_rows = filter_rows_with_existing_topics(csv_data, import_stats)
+      azure_files = fetch_azure_files
+      importable_rows = match_csv_with_azure_files(valid_csv_rows, azure_files)
+      unmatched_files = collect_unmatched_files(csv_data, azure_files, importable_rows)
 
-    valid_csv_rows = filter_rows_with_existing_topics(csv_data, import_stats)
-    azure_files = fetch_azure_files
-    importable_rows = match_csv_with_azure_files(valid_csv_rows, azure_files)
+      log_import_summary(valid_csv_rows, azure_files, importable_rows)
 
-    log_import_summary(valid_csv_rows, azure_files, importable_rows)
+      process_document_attachments(importable_rows, import_stats)
 
-    process_document_attachments(importable_rows, import_stats)
-    log_final_results(import_stats)
+      report.update!(
+        completed_at: Time.current,
+        status: "completed",
+        summary_stats: build_summary_stats(import_stats, csv_data, azure_files),
+        unmatched_files: unmatched_files,
+        error_details: import_stats[:error_files]
+      )
+
+      log_final_results(import_stats)
+    rescue => e
+      report.update!(status: "failed", error_details: [ { error: e.message } ])
+      raise
+    end
   end
 
   private
@@ -287,6 +308,29 @@ def self.handle_attachment_error(topic, filename, error, stats)
     puts "Error with file: #{filename} for topic #{topic.title} - #{error.message}"
   end
 
+  def self.collect_unmatched_files(csv_data, azure_files, importable_rows)
+    csv_file_names = csv_data.map { |row| row["File_Name"] }
+    azure_file_names = azure_files.map { |file| file[:name] }
+    matched_file_names = importable_rows.map { |row| row["File_Name"] }
+
+    {
+      csv_without_azure: csv_file_names - azure_file_names,
+      azure_without_csv: azure_file_names - csv_file_names,
+      total_unmatched: (csv_file_names + azure_file_names - matched_file_names).uniq,
+    }
+  end
+
+  def self.build_summary_stats(import_stats, csv_data, azure_files)
+    {
+      total_csv_files: csv_data.size,
+      total_azure_files: azure_files.size,
+      successful_attachments: import_stats[:successful_attachments].size,
+      failed_attachments: import_stats[:failed_attachments].size,
+      topics_without_csv: import_stats[:topics_without_csv].size,
+      error_files: import_stats[:error_files].size,
+    }
+  end
+
   def self.log_import_summary(csv_rows, azure_files, importable_rows)
     puts "CSV rows with topics: #{csv_rows.size}"
     puts "Azure files found: #{azure_files.size}"