diff --git a/app/controllers/admin/bulk_affiliate_styles_upload_controller.rb b/app/controllers/admin/bulk_affiliate_styles_upload_controller.rb
index e33250225..d5b88ab9e 100644
--- a/app/controllers/admin/bulk_affiliate_styles_upload_controller.rb
+++ b/app/controllers/admin/bulk_affiliate_styles_upload_controller.rb
@@ -1,40 +1,34 @@
# frozen_string_literal: true
-module Admin
- class BulkAffiliateStylesUploadController < AdminController
- def index
- @page_title = 'Bulk Affiliate Styles Upload'
- end
-
- def upload
- begin
- @file = params[:bulk_upload_affiliate_styles]
- BulkAffiliateStyles::FileValidator.new(@file).validate!
- enqueue_job
- flash[:success] = success_message(@file.original_filename)
- rescue BulkAffiliateStylesUploader::Error => e
- Rails.logger.error e
- flash[:error] = e.message
- end
+class Admin::BulkAffiliateStylesUploadController < Admin::BulkUploadController
+ def upload
+ handle_bulk_upload(
+ params_key: :bulk_upload_affiliate_styles,
+ validator_class: BulkAffiliateStyles::FileValidator,
+ error_class: BulkAffiliateStylesUploader::Error,
+ success_path: admin_bulk_affiliate_styles_upload_index_path,
+ logger_message: 'Bulk Affiliate Styles upload failed'
+ )
+ end
- redirect_to admin_bulk_affiliate_styles_upload_index_path
- end
+ private
- private
+ def set_page_title
+ @page_title = 'Bulk Affiliate Styles Upload'
+ end
- def success_message(filename)
- <<~SUCCESS_MESSAGE
- Successfully uploaded #{filename} for processing.
- The results will be emailed to you.
- SUCCESS_MESSAGE
- end
+ def success_message(filename)
+ <<~SUCCESS_MESSAGE
+ Successfully uploaded #{filename} for processing.
+ The results will be emailed to you.
+ SUCCESS_MESSAGE
+ end
- def enqueue_job
- BulkAffiliateStylesUploaderJob.perform_later(
- current_user,
- @file.original_filename,
- @file.tempfile.path
- )
- end
+ def enqueue_job
+ BulkAffiliateStylesUploaderJob.perform_later(
+ current_user,
+ @file.original_filename,
+ @file.tempfile.path
+ )
end
end
diff --git a/app/controllers/admin/bulk_upload_controller.rb b/app/controllers/admin/bulk_upload_controller.rb
new file mode 100644
index 000000000..0f860aa2a
--- /dev/null
+++ b/app/controllers/admin/bulk_upload_controller.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Admin::BulkUploadController < Admin::AdminController
+ include BulkUploadHandler
+ before_action :set_page_title
+
+ def index; end
+end
diff --git a/app/controllers/admin/bulk_url_upload_controller.rb b/app/controllers/admin/bulk_url_upload_controller.rb
index b5ad0b749..09b15fb59 100644
--- a/app/controllers/admin/bulk_url_upload_controller.rb
+++ b/app/controllers/admin/bulk_url_upload_controller.rb
@@ -2,26 +2,27 @@
module Admin
class BulkUrlUploadController < AdminController
- def index
- @page_title = 'Bulk URL Upload'
- end
+ include BulkUploadHandler
+ before_action :set_page_title
- def upload
- begin
- @file = params[:bulk_upload_urls]
- BulkUrlUploader::UrlFileValidator.new(@file).validate!
- enqueue_job
- flash[:success] = success_message(@file.original_filename)
- rescue BulkUrlUploader::Error => e
- Rails.logger.error 'Url upload failed', e
- flash[:error] = e.message
- end
+ def index; end
- redirect_to admin_bulk_url_upload_index_path
+ def upload
+ handle_bulk_upload(
+ params_key: :bulk_upload_urls,
+ validator_class: BulkUrlUploader::UrlFileValidator,
+ error_class: BulkUrlUploader::Error,
+ success_path: admin_bulk_url_upload_index_path,
+ logger_message: 'Url upload failed'
+ )
end
private
+ def set_page_title
+ @page_title = 'Bulk URL Upload'
+ end
+
def success_message(filename)
<<~SUCCESS_MESSAGE
Successfully uploaded #{filename} for processing.
diff --git a/app/controllers/admin/bulk_zombie_url_upload_controller.rb b/app/controllers/admin/bulk_zombie_url_upload_controller.rb
new file mode 100644
index 000000000..e74551f44
--- /dev/null
+++ b/app/controllers/admin/bulk_zombie_url_upload_controller.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+class Admin::BulkZombieUrlUploadController < Admin::BulkUploadController
+ include BulkUploadHandler
+ before_action :set_page_title
+
+ def upload
+ handle_bulk_upload(
+ params_key: :bulk_upload_zombie_urls,
+ validator_class: BulkZombieUrls::FileValidator,
+ error_class: BulkZombieUrlUploader::Error,
+ success_path: admin_bulk_zombie_url_upload_index_path,
+ logger_message: 'Zombie Url upload failed'
+ )
+ end
+
+ private
+
+ def set_page_title
+ @page_title = 'Bulk Zombie Url Upload'
+ end
+
+ def success_message(filename)
+ <<~SUCCESS_MESSAGE
+ Successfully uploaded #{filename} for processing.
+ The results will be emailed to you.
+ SUCCESS_MESSAGE
+ end
+
+ def enqueue_job
+ s3_client = Aws::S3::Client.new(region: ENV['AWS_REGION'])
+ filepath = "#{Rails.env}/file_uploads/#{SecureRandom.uuid}/#{@file.original_filename}"
+
+ s3_client.put_object(
+ bucket: ENV['AWS_BUCKET'],
+ key: filepath,
+ body: @file.tempfile.set_encoding('UTF-8')
+ )
+
+ BulkZombieUrlUploaderJob.perform_now(
+ current_user,
+ @file.original_filename,
+ filepath,
+ )
+ end
+end
diff --git a/app/controllers/concerns/bulk_upload_handler.rb b/app/controllers/concerns/bulk_upload_handler.rb
new file mode 100644
index 000000000..7f9814b6d
--- /dev/null
+++ b/app/controllers/concerns/bulk_upload_handler.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module BulkUploadHandler
+ def handle_bulk_upload(params_key:, validator_class:, error_class:, success_path:, logger_message:)
+ @file = params[params_key]
+ validate_file(@file, validator_class)
+ perform_upload(@file)
+ rescue error_class => e
+ handle_upload_error(logger_message, e)
+ ensure
+ redirect_to success_path
+ end
+
+ private
+
+ def validate_file(file, validator_class)
+ validator_class.new(file).validate!
+ end
+
+ def perform_upload(file)
+ enqueue_job
+ flash[:success] = success_message(file.original_filename)
+ end
+
+ def handle_upload_error(logger_message, error)
+ Rails.logger.error logger_message, error
+ flash[:error] = error.message
+ end
+end
diff --git a/app/jobs/bulk_zombie_url_uploader_job.rb b/app/jobs/bulk_zombie_url_uploader_job.rb
new file mode 100644
index 000000000..8425176b1
--- /dev/null
+++ b/app/jobs/bulk_zombie_url_uploader_job.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class BulkZombieUrlUploaderJob < ApplicationJob
+ queue_as :searchgov
+
+ delegate :upload, to: :@uploader
+
+ def perform(user, filename, filepath)
+ @user = user
+ @filename = filename
+
+ s3_client = Aws::S3::Client.new(region: ENV['AWS_REGION'])
+ response = s3_client.get_object(bucket: ENV['AWS_BUCKET'], key: filepath)
+
+ local_filepath = Rails.root.join('tmp', filename)
+ File.open(local_filepath, 'wb') { |file| file.write(response.body.read) }
+
+ @uploader = BulkZombieUrlUploader.new(filename, local_filepath)
+ upload
+
+ File.delete(local_filepath) if File.exist?(local_filepath)
+ report_results
+ end
+
+ private
+
+ def report_results
+ log_results
+ send_results_email
+ end
+
+ def log_results
+ results = @uploader.results
+ Rails.logger.info(BulkZombieUrlUploaderJob: results.file_name, total_urls: results.total_count, errors: results.error_count)
+ end
+
+ def send_results_email
+ results = @uploader.results
+ email = BulkZombieUrlUploadResultsMailer.with(user: @user, results:).results_email
+ email.deliver_now!
+ end
+end
diff --git a/app/mailers/bulk_zombie_url_upload_results_mailer.rb b/app/mailers/bulk_zombie_url_upload_results_mailer.rb
new file mode 100644
index 000000000..7f07c3e0d
--- /dev/null
+++ b/app/mailers/bulk_zombie_url_upload_results_mailer.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class BulkZombieUrlUploadResultsMailer < ApplicationMailer
+ def results_email
+ @results = params[:results]
+ mail(to: params[:user].email, subject: "Bulk Zombie URL upload results for #{@results.file_name}")
+ end
+end
diff --git a/app/services/bulk_zombie_url_uploader.rb b/app/services/bulk_zombie_url_uploader.rb
new file mode 100644
index 000000000..fe3a99b65
--- /dev/null
+++ b/app/services/bulk_zombie_url_uploader.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class BulkZombieUrlUploader
+ attr_reader :results
+
+ class Error < StandardError; end
+
+ def initialize(filename, filepath)
+ @file_name = filename
+ @file_path = filepath
+ @results = nil
+ end
+
+ def upload
+ initialize_results
+ process_upload
+ rescue StandardError => e
+ Rails.logger.error("Failed to process bulk zombie URL document", e, filename: @file_name)
+ ensure
+ @results ||= BulkZombieUrls::Results.new(@file_name)
+ end
+
+ private
+
+ def initialize_results
+ @results = BulkZombieUrls::Results.new(@file_name)
+ raise Error, 'Results object not initialized' unless @results
+ end
+
+ def process_upload
+ CSV.parse(File.read(@file_path), headers: true).each { |row| process_row(row) }
+ end
+
+ def process_row(row)
+ raise Error, 'Results object not initialized' unless @results
+
+ url = row['URL']&.strip
+ document_id = row['DOC_ID']&.strip
+
+ return log_missing_document_id(row, url) if document_id.blank?
+
+ handle_url_processing(url, document_id, row)
+ end
+
+ def handle_url_processing(url, document_id, row)
+ process_url(url, document_id)
+ update_results
+ rescue StandardError => e
+ handle_processing_error(e, url, document_id, row)
+ end
+
+ def update_results
+ @results.delete_ok
+ @results.increment_updated
+ end
+
+ def log_missing_document_id(row, url)
+ @results.add_error('Document ID is missing', url || 'Unknown')
+ Rails.logger.error("Skipping row: #{row.inspect}. Document ID is mandatory.")
+ end
+
+ def handle_processing_error(error, url, document_id, row)
+ key = url.presence || document_id
+ @results&.add_error(error.message, key)
+ Rails.logger.error('Failure to process bulk upload zombie URL row:', error, url:, document_id:)
+ end
+
+ def process_url(url, document_id)
+ if url.present?
+ process_url_with_searchgov(url, document_id)
+ else
+ I14yDocument.delete(handle: 'searchgov', document_id:)
+ end
+ end
+
+ def process_url_with_searchgov(url, document_id)
+ searchgov_url = SearchgovUrl.find_by(url:)
+ searchgov_url.destroy if searchgov_url
+ end
+end
diff --git a/app/services/bulk_zombie_urls/file_validator.rb b/app/services/bulk_zombie_urls/file_validator.rb
new file mode 100644
index 000000000..4eefd2886
--- /dev/null
+++ b/app/services/bulk_zombie_urls/file_validator.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class BulkZombieUrls::FileValidator
+ MAXIMUM_FILE_SIZE = 4.megabytes
+ VALID_CONTENT_TYPES = %w[text/csv].freeze
+
+ def initialize(uploaded_file)
+ @uploaded_file = uploaded_file
+ end
+
+ def validate!
+ ensure_present
+ ensure_valid_content_type
+ ensure_not_too_big
+ end
+
+ private
+
+ def ensure_valid_content_type
+ return if VALID_CONTENT_TYPES.include?(@uploaded_file.content_type)
+
+ error_message = "Files of type #{@uploaded_file.content_type} are not supported."
+ raise(BulkZombieUrlUploader::Error, error_message)
+ end
+
+ def ensure_present
+ return if @uploaded_file.present?
+
+ error_message = 'Please choose a file to upload.'
+ raise(BulkZombieUrlUploader::Error, error_message)
+ end
+
+ def ensure_not_too_big
+ return if @uploaded_file.size <= MAXIMUM_FILE_SIZE
+
+ error_message = "#{@uploaded_file.original_filename} is too big; please split it."
+ raise(BulkZombieUrlUploader::Error, error_message)
+ end
+end
diff --git a/app/services/bulk_zombie_urls/results.rb b/app/services/bulk_zombie_urls/results.rb
new file mode 100644
index 000000000..b2b729ca5
--- /dev/null
+++ b/app/services/bulk_zombie_urls/results.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module BulkZombieUrls
+ class Results
+ attr_accessor :ok_count, :error_count, :file_name
+ attr_reader :errors, :updated
+
+ def initialize(filename)
+ @file_name = filename
+ @ok_count = 0
+ @error_count = 0
+ @updated = 0
+ @errors = Hash.new { |hash, key| hash[key] = [] }
+ end
+
+ def increment_updated
+ @updated += 1
+ end
+
+ def delete_ok
+ @ok_count += 1
+ end
+
+ def add_error(error_message, key)
+ self.error_count += 1
+ @errors[key] << error_message
+ end
+
+ def total_count
+ ok_count + error_count
+ end
+ end
+end
diff --git a/app/views/admin/bulk_zombie_url_upload/index.html.haml b/app/views/admin/bulk_zombie_url_upload/index.html.haml
new file mode 100644
index 000000000..daeee5505
--- /dev/null
+++ b/app/views/admin/bulk_zombie_url_upload/index.html.haml
@@ -0,0 +1,37 @@
+%h2
+ Bulk Zombie Url Upload
+%p
+ To upload a list of URLs to be deleted, follow these instructions:
+ %ul.bulk-upload-instructions
+ %li
+ Create a new CSV file containing one URL and Document id per line.
+ %li
+ Document ID is mandatorily required to delete URL from Kibana when URL is not present in Search DB. An example of this format
+ is shown below.
+ %li
+ Save the file on your computer; you can name the file whatever you like, as
+ long as it has a .txt extension.
+ %li
+ %b
+ Do not use Microsoft Word files, or any other file type except plain text.
+ %li
+ The maximum file size is #{number_to_human_size(BulkZombieUrls::FileValidator::MAXIMUM_FILE_SIZE)}.
+ %li
+ Browse for the file on your computer.
+ %li
+ Upload the file to Search.gov using the upload button below.
+ %li
+ You will receive an email when processing of your URLs is complete.
+ %strong
+ Sample file format:
+ %code
+ %pre
+ URL,DOC_ID
+ http://www.sample.gov/1.html,docid121
+ http://www.sample.gov/2.html,docid122
+ http://www.sample.gov/3.html,docid123
+ ,docid124
+ = form_tag upload_admin_bulk_zombie_url_upload_index_path, :multipart => true, :class => 'form' do
+ = file_field_tag 'bulk_upload_zombie_urls', :accept => 'text/csv'
+ %p
+ = submit_tag "Upload", :class => 'submit btn-upload'
diff --git a/app/views/admin/home/index.html.haml b/app/views/admin/home/index.html.haml
index fef634076..2989bf126 100644
--- a/app/views/admin/home/index.html.haml
+++ b/app/views/admin/home/index.html.haml
@@ -15,6 +15,7 @@
%li=link_to("Superfresh Urls", admin_superfresh_urls_path)
%li=link_to("Superfresh Bulk Upload", admin_superfresh_urls_bulk_upload_index_path)
%li=link_to("Bulk Search.gov URL Upload", admin_bulk_url_upload_index_path)
+ %li=link_to("Bulk Zombie URL Upload", admin_bulk_zombie_url_upload_index_path)
%li=link_to("ODIE URL Source Update", admin_odie_url_source_update_index_path)
%li=link_to("Bulk Affiliate Styles Upload", admin_bulk_affiliate_styles_upload_index_path)
diff --git a/app/views/bulk_zombie_url_upload_results_mailer/results_email.html.erb b/app/views/bulk_zombie_url_upload_results_mailer/results_email.html.erb
new file mode 100644
index 000000000..917a65ddd
--- /dev/null
+++ b/app/views/bulk_zombie_url_upload_results_mailer/results_email.html.erb
@@ -0,0 +1,27 @@
+
+ The bulk zombie url upload job for <%= @results.file_name %> has
+ finished.
+
+
+
+ There were <%= @results.total_count %> URLs.
+
+
+<% if @results.error_count == 0 %>
+
+ There were no errors.
+
+<% else %>
+
+ <%= @results.ok_count %> URLs were deleted successfully.
+
+
+
+ <%= @results.error_count %> URLs failed validation.
+
+ <% @results.errors.each do |k,error_message| %>
+ <%= k %>
+ <%= error_message %>
+
+ <% end %>
+<% end %>
diff --git a/app/views/bulk_zombie_url_upload_results_mailer/results_email.text.erb b/app/views/bulk_zombie_url_upload_results_mailer/results_email.text.erb
new file mode 100644
index 000000000..d1e4af080
--- /dev/null
+++ b/app/views/bulk_zombie_url_upload_results_mailer/results_email.text.erb
@@ -0,0 +1,17 @@
+The bulk zombie url upload job for <%= @results.file_name %> has
+finished.
+
+There were <%= @results.total_count %> URLs.
+
+<% if @results.error_count == 0 %>
+ There were no errors.
+<% else %>
+ <%= @results.ok_count %> URLs were deleted successfully.
+ <%= @results.error_count %> URLs failed validation.
+
+ <% @results.errors.each do |k,error_message| %>
+ <%= k %>
+ <%= error_message %>
+
+ <% end %>
+<