chore: Extract CSV parsing logic and optimize site import flow

- Introduce `CsvSiteParser`
- Simplify `SiteUpload` model
- Add `ProcessSiteUploadJob` and `ProcessSingleSiteJob` for parallelized site processing
- Fix audit callback to use `after_create` instead of `after_create_commit`

Author: Josyann

app/controllers/sites_controller.rb

@@ -42,7 +42,7 @@ def create
   def upload
     @upload = SiteUpload.new(site_upload_params)
     if @upload.save
-      redirect_to sites_path, notice: t(".uploaded", count: @upload.count)
+      redirect_to sites_path, notice: t(".started")
     else
       render :new, status: :unprocessable_content
     end

app/jobs/process_single_site_job.rb

@@ -0,0 +1,25 @@
+class ProcessSingleSiteJob < ApplicationJob
+  queue_as :default
+
+  def perform(site_data, team_id, tag_ids)
+    team = Team.find(team_id)
+
+    url = site_data["url"]
+    name = site_data["name"]
+    tag_names = site_data["tag_names"] || []
+
+    row_tag_ids = tag_names.map { |tag_name| team.tags.find_or_create_by(name: tag_name).id }
+    combined_tag_ids = (tag_ids + row_tag_ids).uniq
+
+    site = team.sites.find_by_url(url: url)
+
+    if site
+      site.tag_ids = combined_tag_ids.union(site.tag_ids)
+      site.name = name if name.present? && site.name.blank?
+      site.save!
+      site.audit!
+    else
+      Site.create!(url: url, team: team, name: name, tag_ids: combined_tag_ids)
+    end
+  end
+end

app/jobs/process_site_upload_job.rb

@@ -0,0 +1,13 @@
+class ProcessSiteUploadJob < ApplicationJob
+  queue_as :default
+
+  def perform(sites_data, team_id, tag_ids)
+    site_jobs = []
+
+    sites_data.each do |site_data|
+      site_jobs << ProcessSingleSiteJob.new(site_data, team_id, tag_ids)
+    end
+
+    ActiveJob.perform_all_later(site_jobs) if site_jobs.any?
+  end
+end

app/models/audit.rb

@@ -2,7 +2,7 @@ class Audit < ApplicationRecord
   belongs_to :site, touch: true, counter_cache: true
   has_many :checks, -> { prioritized }, dependent: :destroy
 
-  after_create_commit :create_checks
+  after_create :create_checks
   after_create_commit :schedule
 
   validates :url, presence: true, url: true

app/models/site_upload.rb

@@ -15,32 +15,23 @@ class SiteUpload
   ].freeze
   MAX_FILE_SIZE = 5.megabytes
   REQUIRED_HEADERS = ["url"].freeze
-  SUPPORTED_SEPARATORS = [",", ";"].freeze
-  BOM = /^\xEF\xBB\xBF/
 
-  attr_accessor :file, :team, :tag_ids, :tags, :new_sites, :existing_sites
+  attr_accessor :file, :team, :tag_ids
 
   validates :file, :team, presence: true
   validate :valid_file_size, :valid_file_format, :valid_headers, if: :file
 
-  delegate :create!, :transaction, :human, to: :Site
-
   def initialize(attributes = {})
     super
     @tag_ids ||= []
-    @new_sites = {}
-    @existing_sites = {}
   end
 
   def save
     return false unless valid?
+    sites_data = parser.parse
 
-    parse_sites
+    ProcessSiteUploadJob.perform_later(sites_data, team.id, tag_ids)
 
-    transaction do
-      create!(new_sites.values) if new_sites.any?
-      existing_sites.values.each { |site| site.save && site.audit! }
-    end
     true
   end
 
@@ -58,43 +49,13 @@ def assign_attributes(attributes)
   end
 
   def count
-    (new_sites&.length || 0) + (existing_sites&.length || 0)
-  end
-
-  def parse_sites
-    require "csv"
-
-    CSV.foreach(file.path, headers: true, encoding: "bom|utf-8", col_sep:) do |row|
-      row = row.to_h.transform_keys { |header| header.to_s.downcase } # Case-insensitive headers
-
-      url = Link.normalize(row["url"])
-      name = row["nom"] || row["name"]
-      tag_names = row["tags"].present? ? row["tags"].split(",").map(&:strip).compact_blank.uniq : []
-
-      row_tag_ids = tag_names.map { |n| team.tags.find_or_create_by(name: n).id }
-      combined_tag_ids = (tag_ids + row_tag_ids).uniq
-      existing_site = team.sites.find_by_url(url:)
-
-      if existing_site
-        existing_site.assign_attributes(tag_ids: combined_tag_ids.union(existing_site.tag_ids))
-        existing_site.assign_attributes(name:) unless existing_site.name
-        self.existing_sites[url] = existing_site
-      else
-        self.new_sites[url] = { url:, team:, name:, tag_ids: combined_tag_ids }
-      end
-    end
+    file ? parser.count : 0
   end
 
   private
 
-  def first_line
-    @first_line ||= File.open(file.path, &:gets)&.strip&.sub(BOM, "") || ""
-  end
-
-  def col_sep
-    SUPPORTED_SEPARATORS.max_by { |sep| first_line.count(sep) }
-  rescue StandardError
-    SUPPORTED_SEPARATORS.first
+  def parser
+    @parser ||= CsvSiteParser.new(file)
   end
 
   def valid_file_size
@@ -107,10 +68,7 @@ def valid_file_format
   end
 
   def valid_headers
-    headers = CSV.parse_line(first_line, col_sep:) || []
-    missing_headers = REQUIRED_HEADERS - headers.compact.collect(&:downcase)
+    missing_headers = REQUIRED_HEADERS - parser.headers
     errors.add(:file, :invalid_headers) unless missing_headers.empty?
-  rescue CSV::MalformedCSVError, StandardError
-    errors.add(:file, :invalid_headers)
   end
 end

app/services/csv_site_parser.rb

@@ -0,0 +1,50 @@
+class CsvSiteParser
+  require "csv"
+
+  BOM = /^\xEF\xBB\xBF/
+  SUPPORTED_SEPARATORS = [",", ";"].freeze
+
+  def initialize(file)
+    @file = file
+  end
+
+  def parse
+    sites_by_url = {}
+
+    CSV.foreach(@file.path, headers: true, encoding: "bom|utf-8", col_sep: detect_col_sep) do |row|
+      row = row.to_h.transform_keys { |header| header.to_s.downcase }
+
+      url = Link.normalize(row["url"]).to_s
+      name = row["nom"] || row["name"]
+      tag_names = row["tags"].present? ? row["tags"].split(",").map(&:strip).compact_blank.uniq : []
+
+      if sites_by_url[url]
+        sites_by_url[url]["tag_names"] = (sites_by_url[url]["tag_names"] + tag_names).uniq
+      else
+        sites_by_url[url] = {
+          "url" => url,
+          "name" => name,
+          "tag_names" => tag_names
+        }
+      end
+    end
+
+    sites_by_url.values
+  end
+
+  def headers
+    @headers ||= (CSV.parse_line(first_line, col_sep: detect_col_sep) || []).compact_blank.map(&:downcase)
+  end
+
+  private
+
+  def first_line
+    @first_line ||= File.open(@file.path, &:gets)&.strip&.sub(BOM, "") || ""
+  end
+
+  def detect_col_sep
+    SUPPORTED_SEPARATORS.max_by { |sep| first_line.count(sep) }
+  rescue StandardError
+    SUPPORTED_SEPARATORS.first
+  end
+end

config/locales/fr.yml

@@ -291,6 +291,7 @@ fr:
       new_audit: Site existant, nouvel audit programmé
     upload:
       title: Importer des sites
+      started: Import démarré
       uploaded:
         zero: Aucun site ajouté
         one: Un site ajouté