wip #221

Author: hernanvicente

app/models/import_error.rb

@@ -1,3 +1,27 @@
+# == Schema Information
+#
+# Table name: import_errors
+#
+#  id               :bigint           not null, primary key
+#  error_message    :text
+#  error_type       :string           not null
+#  file_name        :string
+#  metadata         :json
+#  created_at       :datetime         not null
+#  updated_at       :datetime         not null
+#  import_report_id :bigint           not null
+#  topic_id         :integer
+#
+# Indexes
+#
+#  index_import_errors_on_error_type        (error_type)
+#  index_import_errors_on_file_name         (file_name)
+#  index_import_errors_on_import_report_id  (import_report_id)
+#
+# Foreign Keys
+#
+#  fk_rails_...  (import_report_id => import_reports.id)
+#
 class ImportError < ApplicationRecord
   belongs_to :import_report
 

app/models/import_report.rb

@@ -1,9 +1,28 @@
+# == Schema Information
+#
+# Table name: import_reports
+#
+#  id              :bigint           not null, primary key
+#  completed_at    :datetime
+#  error_details   :json
+#  import_type     :string           not null
+#  started_at      :datetime
+#  status          :string           default("pending")
+#  summary_stats   :json
+#  unmatched_files :json
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#
+# Indexes
+#
+#  index_import_reports_on_import_type  (import_type)
+#
 class ImportReport < ApplicationRecord
   has_many :import_errors, dependent: :destroy
 
   validates :import_type, presence: true
 
-  enum :status, { pending: "pending", completed: "completed", failed: "failed" }
+  enum :status, { pending: "pending", planned: "planned", completed: "completed", failed: "failed" }
 
   scope :recent, -> { order(created_at: :desc) }
   scope :by_type, ->(type) { where(import_type: type) }

db/migrate/20250625101109_create_import_errors.rb

@@ -0,0 +1,17 @@
+class CreateImportErrors < ActiveRecord::Migration[7.0]
+  def change
+    create_table :import_errors do |t|
+      t.references :import_report, null: false, foreign_key: true
+      t.string :error_type, null: false
+      t.string :file_name
+      t.integer :topic_id
+      t.text :error_message
+      t.json :metadata
+
+      t.timestamps
+    end
+
+    add_index :import_errors, :error_type
+    add_index :import_errors, :file_name
+  end
+end

db/schema.rb

@@ -202,16 +202,19 @@ def self.import_training_documents
       importable_rows = match_csv_with_azure_files(valid_csv_rows, azure_files)
       unmatched_files = collect_unmatched_files(csv_data, azure_files, importable_rows)
 
-      log_import_summary(valid_csv_rows, azure_files, importable_rows)
+      report.update!(
+        summary_stats: build_summary_stats(import_stats, csv_data, azure_files),
+        unmatched_files: unmatched_files,
+        status: "planned"
+      )
 
-      process_document_attachments(importable_rows, import_stats)
+      process_document_attachments(importable_rows, import_stats, report)
 
       report.update!(
         completed_at: Time.current,
         status: "completed",
         summary_stats: build_summary_stats(import_stats, csv_data, azure_files),
-        unmatched_files: unmatched_files,
-        error_details: import_stats[:error_files]
+        unmatched_files: unmatched_files
       )
 
       log_final_results(import_stats)
@@ -254,16 +257,16 @@ def self.match_csv_with_azure_files(csv_rows, azure_files)
     end
   end
 
-  def self.process_document_attachments(rows, stats)
+  def self.process_document_attachments(rows, stats, report)
     rows.each do |row|
       topic = Topic.find_by(id: row["Topic_ID"])
       next unless topic
 
-      attach_document_to_topic(topic, row, stats)
+      attach_document_to_topic(topic, row, stats, report)
     end
   end
 
-  def self.attach_document_to_topic(topic, row, stats)
+  def self.attach_document_to_topic(topic, row, stats, report)
     file_path = get_file_path(topic.state, topic.language.name)
     filename = row["File_Name"]
 
@@ -285,7 +288,7 @@ def self.attach_document_to_topic(topic, row, stats)
       end
 
     rescue AzureFileShares::Errors::ApiError, URI::InvalidURIError => e
-      handle_attachment_error(topic, filename, e, stats)
+      handle_attachment_error(topic, filename, e, stats, report)
     end
   end
 
@@ -298,12 +301,20 @@ def self.download_azure_file(file_path, filename)
     )
   end
 
-  def self.handle_attachment_error(topic, filename, error, stats)
+  def self.handle_attachment_error(topic, filename, error, stats, report)
     error_info = {
       topic: topic,
       file: filename,
       error: error.message,
     }
+
+    ImportError.create(
+      import_report_id: report.id,
+      topic_id: topic.id,
+      file_name: filename,
+      error_message: error_info
+    )
+
     stats[:error_files] << error_info
     puts "Error with file: #{filename} for topic #{topic.title} - #{error.message}"
   end
@@ -331,12 +342,6 @@ def self.build_summary_stats(import_stats, csv_data, azure_files)
     }
   end
 
-  def self.log_import_summary(csv_rows, azure_files, importable_rows)
-    puts "CSV rows with topics: #{csv_rows.size}"
-    puts "Azure files found: #{azure_files.size}"
-    puts "Importable files: #{importable_rows.size}"
-  end
-
   def self.log_final_results(stats)
     puts "Topics not found: #{stats[:topics_without_csv].size}"
     puts "Successful attachments: #{stats[:successful_attachments].size}"