Remediation failure handling (#1843)

* Job to handle when remediation fails by creating a ticket in LibAnswers and storing the failed_at time on the file resource.

* Adds tests

* More tests

* Moved the timestamp storing step when failed out of the job and in the two fail outcomes in the controller.  I think it makes sense to store the timestamp outside of the jobs since the job is run async, so the time might not be at the same time the request come back from the API.

* Made the test more realistic

Author: ajkiessl

app/controllers/webhooks/pdf_accessibility_api_controller.rb

@@ -25,11 +25,14 @@ def handle_success(job_data)
       BuildAutoRemediatedWorkVersionJob.perform_later(job_data[:uuid], job_data[:output_url])
       render json: { message: 'Update successful' }, status: :ok
     rescue StandardError => e
+      store_failure(job_data[:uuid])
       render json: { error: e.message }, status: :internal_server_error
     end
 
     def handle_failure(job_data)
       Rails.logger.error("Auto-remediation job failed: #{job_data[:processing_error_message]}")
+      AutoRemediationFailedJob.perform_later(job_data[:uuid])
+      store_failure(job_data[:uuid])
       render json: { message: job_data[:processing_error_message] }, status: :ok
     end
 
@@ -45,4 +48,9 @@ def authenticate_request
 
       head(:unauthorized) unless request.headers['X-API-KEY'] == ENV['PDF_REMEDIATION_WEBHOOK_SECRET']
     end
+
+    def store_failure(job_uuid)
+      file_resource = FileResource.find_by(remediation_job_uuid: job_uuid)
+      file_resource.update(auto_remediation_failed_at: Time.current)
+    end
 end

app/jobs/auto_remediation_failed_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AutoRemediationFailedJob < ApplicationJob
+  queue_as :default
+
+  def perform(job_uuid)
+    work = FileResource.find_by!(remediation_job_uuid: job_uuid)
+      .work_versions
+      &.first
+      &.work
+
+    LibanswersApiService.new
+      .admin_create_ticket(work.id,
+                           'work_remediation_failed')
+  end
+end

app/services/libanswers_api_service.rb

@@ -43,6 +43,7 @@ def get_depositor(id, type)
         'work_curation' => Work,
         'work_accessibility_check' => Work,
         'work_remediation' => Work,
+        'work_remediation_failed' => Work,
         'collection' => Collection
       }
       deposit = deposit_types[type].find(id)
@@ -60,7 +61,7 @@ def get_admin_subject(id, type)
       when 'work_accessibility_check'
         work = Work.find(id)
         "ScholarSphere Deposit Accessibility Curation: #{work.latest_version.title}"
-      when 'work_remediation'
+      when 'work_remediation', 'work_remediation_failed'
         work = Work.find(id)
         "ScholarSphere PDF Auto-remediation Result: #{work.latest_version.title}"
       end
@@ -87,12 +88,17 @@ def get_ticket_details(id, type, admin_subject, base_url = '')
           (accessibility_check_results.empty? ? '' : "pdetails=#{accessibility_check_results}&") +
           "pname=#{work.display_name}&" +
           "pemail=#{work.email}"
-      when 'work_remediation'
+      when 'work_remediation', 'work_remediation_failed'
         @work = Work.find(id)
         "quid=#{ACCESSIBILITY_QUEUE_ID}&" +
           "pquestion=#{admin_subject}&" +
           "pname=#{work.display_name}&" +
-          "pemail=#{work.email}"
+          "pemail=#{work.email}" +
+          (if type == 'work_remediation_failed'
+             '&pdetails=A PDF associated with this work failed to auto-remediate and requires manual review.'
+           else
+             ''
+           end)
       end
     end
 

db/migrate/20251030184453_add_auto_remediation_failed_at_to_file_resource.rb

@@ -0,0 +1,5 @@
+class AddAutoRemediationFailedAtToFileResource < ActiveRecord::Migration[7.2]
+  def change
+    add_column :file_resources, :auto_remediation_failed_at, :datetime
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2025_10_07_200037) do
+ActiveRecord::Schema[7.2].define(version: 2025_10_30_184453) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
   enable_extension "plpgsql"
@@ -155,6 +155,7 @@
     t.uuid "uuid", default: -> { "uuid_generate_v4()" }
     t.string "remediation_job_uuid"
     t.boolean "auto_remediated_version", default: false, null: false
+    t.datetime "auto_remediation_failed_at"
   end
 
   create_table "file_version_memberships", force: :cascade do |t|

spec/controllers/webhooks/pdf_accessibility_api_controller_spec.rb

@@ -17,6 +17,7 @@
     @original_secret = ENV['PDF_REMEDIATION_WEBHOOK_SECRET']
     ENV['PDF_REMEDIATION_WEBHOOK_SECRET'] = secret
     allow(BuildAutoRemediatedWorkVersionJob).to receive(:perform_later).and_return(nil)
+    allow(AutoRemediationFailedJob).to receive(:perform_later).and_return(nil)
   end
 
   after do
@@ -70,6 +71,21 @@
           expect(BuildAutoRemediatedWorkVersionJob).to have_received(:perform_later)
             .with(file.remediation_job_uuid, 'https://example.com/out.pdf')
         end
+
+        context 'when an error occurs enqueuing the job' do
+          before do
+            allow(BuildAutoRemediatedWorkVersionJob).to receive(:perform_later).and_raise(StandardError.new('Redis error'))
+            allow(Rails.logger).to receive(:error)
+          end
+
+          it 'stores the failure timestamp and returns 500 with the error message' do
+            post :create, params: params, as: :json
+
+            expect(response).to have_http_status(:internal_server_error)
+            expect(response.parsed_body).to include('error' => 'Redis error')
+            expect(file.reload.auto_remediation_failed_at).not_to be_nil
+          end
+        end
       end
 
       describe 'job.failed handling' do
@@ -82,13 +98,15 @@
           allow(Rails.logger).to receive(:error)
         end
 
-        it 'logs the failure and returns 200 with the processing message' do
+        it 'logs the failure, enqueues the failed job, stores the failure timestamp and returns 200 with the processing message' do
           post :create, params: params, as: :json
 
           expect(response).to have_http_status(:ok)
           expect(response.parsed_body).to include('message' => error_message)
           expect(Rails.logger).to have_received(:error)
             .with("Auto-remediation job failed: #{error_message}")
+          expect(AutoRemediationFailedJob).to have_received(:perform_later).with(file.remediation_job_uuid)
+          expect(file.reload.auto_remediation_failed_at).not_to be_nil
         end
       end
     end

spec/jobs/auto_remediation_failed_job_spec.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe AutoRemediationFailedJob do
+  describe '#perform' do
+    let(:existing_job_uuid) { SecureRandom.uuid }
+    let!(:file_resource) do
+      create(:file_resource,
+             remediation_job_uuid: existing_job_uuid,
+             work_versions: [create(:work_version)])
+    end
+    let(:work) { file_resource.work_versions.first.work }
+    let(:service) do
+      instance_double(LibanswersApiService,
+                      admin_create_ticket: 'https://psu.libanswers.com/fake-ticket')
+    end
+
+    before do
+      allow(LibanswersApiService).to receive(:new).and_return(service)
+    end
+
+    context 'when the file resource with the given remediation_job_uuid exists' do
+      it 'creates a LibAnswers ticket' do
+        described_class.perform_now(existing_job_uuid)
+
+        expect(LibanswersApiService).to have_received(:new)
+        expect(service).to have_received(:admin_create_ticket).with(work.id, 'work_remediation_failed')
+      end
+    end
+
+    context 'when the file resource with the given remediation_job_uuid does not exist' do
+      it 'raises ActiveRecord::RecordNotFound' do
+        expect {
+          described_class.perform_now('nonexistent-uuid')
+        }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+  end
+end

spec/models/file_resource_spec.rb

@@ -17,6 +17,7 @@
     it { is_expected.to have_db_column(:file_data).of_type(:jsonb) }
     it { is_expected.to have_db_column(:remediation_job_uuid).of_type(:string) }
     it { is_expected.to have_db_column(:auto_remediated_version).of_type(:boolean) }
+    it { is_expected.to have_db_column(:auto_remediation_failed_at).of_type(:datetime) }
   end
 
   describe 'factories' do
@@ -269,6 +270,7 @@
         thumbnail_url_ssi
         remediation_job_uuid_tesim
         auto_remediated_version_tesim
+        auto_remediation_failed_at_dtsi
       )
     end
 

spec/services/libanswers_api_service_spec.rb

@@ -184,6 +184,36 @@
           end
         end
       end
+
+      context 'when the ticket is a Work Remediation failed ticket' do
+        it 'uses id 2590 for quid and ScholarSphere PDF Auto-remediation Result for question but includes details' do
+          described_class.new.admin_create_ticket(work.id, 'work_remediation_failed')
+          expect(mock_faraday_connection).to have_received(:post).with(
+            '/api/1.1/ticket/create',
+            "quid=#{accessibility_quid}&pquestion=ScholarSphere PDF Auto-remediation Result: #{
+              work.latest_version.title}&pname=#{work.display_name}&pemail=#{work.email}" \
+            '&pdetails=A PDF associated with this work failed to auto-remediate and requires manual review.'
+          )
+        end
+
+        context 'when the user is not an active member' do
+          let!(:inactive_member) { object_double(PsuIdentity::SearchService::Person.new, affiliation: ['MEMBER']) }
+
+          before do
+            allow(mock_identity_search).to receive(:userid).and_return(inactive_member)
+          end
+
+          it 'still creates a ticket' do
+            described_class.new.admin_create_ticket(work.id, 'work_remediation_failed')
+            expect(mock_faraday_connection).to have_received(:post).with(
+              '/api/1.1/ticket/create',
+              "quid=#{accessibility_quid}&pquestion=ScholarSphere PDF Auto-remediation Result: #{
+                work.latest_version.title}&pname=#{work.display_name}&pemail=#{work.email}" \
+                '&pdetails=A PDF associated with this work failed to auto-remediate and requires manual review.'
+            )
+          end
+        end
+      end
     end
   end
 

spec/support/vcr.rb

@@ -9,7 +9,7 @@
 WebMock.allow_net_connect!(net_http_connect_on_start: true)
 
 # Allow connections to Docker images and webriver update urls
-allowed_hosts = %w(selenium minio solr)
+allowed_hosts = %w(selenium minio solr docker.for.mac.localhost)
 
 VCR.configure do |c|
   c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'