Merge pull request #244 from psu-libraries/242-243-api-pages

ajkiessl · web-flow · commit 427efb70bee3 · 2026-02-23T16:42:16.000-05:00
Count pages on API upload
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -249,20 +249,9 @@ jobs:
           command: |
             gem install bundler -v "$(grep -A 1 'BUNDLED WITH' Gemfile.lock | tail -n 1 | xargs)"
             bundle _$(grep -A 1 'BUNDLED WITH' Gemfile.lock | tail -n 1 | xargs)_ install
-      - attach_workspace:
-          at: /tmp
       - run:
-          name: Load Docker image from build-image
-          command: |
-            if [ -f /tmp/docker-image.tar ]; then
-              docker load -i /tmp/docker-image.tar
-            else
-              echo "No persisted image found; falling back to local build"
-              docker-compose -f docker-compose.yml build
-            fi
-      - run:
-          name: Start services (no-build when image available)
-          command: docker-compose -f docker-compose.yml up --force-recreate -d --no-build
+          name: Build and run containers
+          command: docker-compose -f docker-compose.yml build && docker-compose -f docker-compose.yml up --force-recreate -d
       - run:
           name: Wait for mysql to be ready
           command: docker-compose exec web bash ./wait_for_db.sh
@@ -336,6 +325,3 @@ workflows:
           name: test-application
           context:
             - org-global
-          requires:
-            - build-image
-
diff --git a/Gemfile b/Gemfile
@@ -45,6 +45,9 @@ gem 'image_processing'
 # Alt-text generation
 gem 'alt_text'
 
+# PDF page counting
+gem 'pdf-reader'
+
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
   gem 'debug', platforms: %i[mri mswin mswin64 mingw x64_mingw], require: 'debug/prelude'
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,6 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    Ascii85 (2.0.1)
     actioncable (7.2.2.2)
       actionpack (= 7.2.2.2)
       activesupport (= 7.2.2.2)
@@ -78,6 +79,7 @@ GEM
       tzinfo (~> 2.0, >= 2.0.5)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
+    afm (1.0.0)
     alt_text (0.1.0)
       aws-sdk-bedrockruntime (~> 1.55.0)
       dotenv (~> 3.1.8)
@@ -172,6 +174,7 @@ GEM
     ffi (1.17.2-x86_64-linux-gnu)
     globalid (1.2.1)
       activesupport (>= 6.1)
+    hashery (2.1.2)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     image_processing (1.14.0)
@@ -259,6 +262,12 @@ GEM
     parser (3.3.9.0)
       ast (~> 2.4.1)
       racc
+    pdf-reader (2.15.1)
+      Ascii85 (>= 1.0, < 3.0, != 2.0.0)
+      afm (>= 0.2.1, < 2)
+      hashery (~> 2.0)
+      ruby-rc4
+      ttfunk
     pp (0.6.2)
       prettyprint
     prettyprint (0.2.0)
@@ -404,6 +413,7 @@ GEM
       rubocop (~> 1.72, >= 1.72.1)
       rubocop-rspec (~> 3.5)
     ruby-progressbar (1.13.0)
+    ruby-rc4 (0.1.5)
     ruby-vips (2.2.5)
       ffi (~> 1.12)
       logger
@@ -460,6 +470,8 @@ GEM
     thor (1.4.0)
     tilt (2.7.0)
     timeout (0.4.3)
+    ttfunk (1.8.0)
+      bigdecimal (~> 3.1)
     turbo-rails (2.0.23)
       actionpack (>= 7.1.0)
       railties (>= 7.1.0)
@@ -509,6 +521,7 @@ DEPENDENCIES
   jbuilder
   mysql2 (~> 0.5.7)
   niftany
+  pdf-reader
   puma (>= 5.0)
   rails (~> 7.2.2, >= 7.2.2.1)
   rails_admin
diff --git a/app/jobs/api_remediation_job.rb b/app/jobs/api_remediation_job.rb
@@ -5,20 +5,27 @@ class APIRemediationJob < ApplicationJob
 
   def perform(job_uuid, output_polling_timeout: OUTPUT_POLLING_TIMEOUT)
     job = PdfJob.find_by!(uuid: job_uuid)
+    tempfile = nil
     tempfile = Down.download(job.source_url)
     original_filename = tempfile&.original_filename
     job.update!(output_object_key: original_filename)
+
+    page_count = PDF::Reader.new(tempfile.path).page_count
+    PageCountQuotaValidator.validate!(owner: job.owner, page_count: page_count)
+    job.update!(page_count: page_count)
+
     safe_original_filename = original_filename.gsub(/[^A-Za-z0-9.\-_ ]/, '')
     object_key = "#{SecureRandom.hex(8)}_#{safe_original_filename}"
     file_path = tempfile&.path
     s3_handler = S3Handler.new(object_key)
     s3_handler.upload_to_input(file_path)
     poll_and_update(job_uuid, s3_handler, output_polling_timeout)
-  rescue S3Handler::Error => e
-    update_with_failure(job, "Failed to upload file to remediation input location:  #{e.message}")
-  rescue Down::Error => e
+  rescue Down::Error,
+         S3Handler::Error,
+         PDF::Reader::MalformedPDFError,
+         PageCountQuotaValidator::Error => e
     # We may want to retry the download depending on the more specific nature of the failure.
-    update_with_failure(job, "Failed to download file from source URL:  #{e.message}")
+    update_with_failure(job, "Failed to process job: #{e.class}: #{e.message}")
   ensure
     RemediationStatusNotificationJob.perform_later(job_uuid)
     tempfile&.close!
diff --git a/app/models/api_user.rb b/app/models/api_user.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class APIUser < ApplicationRecord
+  include OwnerHelpers
+
   belongs_to :unit, optional: true
 
   has_many :jobs, as: :owner, dependent: :restrict_with_exception
diff --git a/app/models/concerns/owner_helpers.rb b/app/models/concerns/owner_helpers.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module OwnerHelpers
+  def total_pages_processed_last_24_hours
+    jobs
+      .where(created_at: 24.hours.ago..)
+      .where.not(page_count: nil)
+      .sum(:page_count)
+  end
+end
diff --git a/app/models/unit.rb b/app/models/unit.rb
@@ -8,6 +8,10 @@ class Unit < ApplicationRecord
   validates :daily_page_limit, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
   validates :overall_page_limit, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
+  def total_pages_processed
+    api_users.joins(:jobs).sum(:page_count) + gui_users.joins(:jobs).sum(:page_count)
+  end
+
   RailsAdmin.config do |config|
     config.model 'Unit' do
       list do
diff --git a/app/validators/page_count_quota_validator.rb b/app/validators/page_count_quota_validator.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class PageCountQuotaValidator
+  class Error < StandardError; end
+
+  class QuotaExceededError < Error; end
+  class MissingUnitError < Error; end
+  class InvalidPageCountError < Error; end
+
+  def self.validate!(owner:, page_count:)
+    unit = owner&.unit
+    raise MissingUnitError, 'Owner must belong to a unit to validate page quota' if unit.nil?
+
+    page_count_int = Integer(page_count)
+    raise InvalidPageCountError, 'page_count must be greater than 0' if page_count_int <= 0
+
+    total_quota = unit.overall_page_limit
+    total_processed = unit.total_pages_processed
+
+    if page_count_int + total_processed > total_quota
+      raise QuotaExceededError, "page_count exceeds the unit's overall page limit of #{total_quota}"
+    end
+
+    todays_quota = unit.daily_page_limit
+    todays_processed = owner.total_pages_processed_last_24_hours
+
+    if page_count_int + todays_processed > todays_quota
+      raise QuotaExceededError, "page_count exceeds the user's daily page limit of #{todays_quota}"
+    end
+
+    true
+  rescue ArgumentError, TypeError
+    raise InvalidPageCountError, 'page_count must be an integer'
+  end
+end
diff --git a/db/migrate/20260203200259_add_page_count_to_job.rb b/db/migrate/20260203200259_add_page_count_to_job.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddPageCountToJob < ActiveRecord::Migration[7.2]
   def change
     add_column :jobs, :page_count, :integer, null: true
diff --git a/db/schema.rb b/db/schema.rb
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -70,6 +70,9 @@ services:
       MYSQL_DATABASE: pdf_accessibility_api
       MYSQL_USER: pdf_accessibility_api
       MYSQL_PASSWORD: pdf_accessibility_api
+    command:
+      - mysqld
+      - --character-set-server=utf8mb4
     image: mariadb:10.4.6
     ports:
     - "3306:3306"
diff --git a/spec/factories/api_users.rb b/spec/factories/api_users.rb
@@ -3,5 +3,6 @@
 FactoryBot.define do
   factory :api_user do
     webhook_endpoint { 'https://test.com/endpoint' }
+    unit
   end
 end
diff --git a/spec/jobs/api_remediation_job_spec.rb b/spec/jobs/api_remediation_job_spec.rb
@@ -8,6 +8,7 @@ class Tempfile
 
 RSpec.describe APIRemediationJob do
   let!(:job) { create(:pdf_job, source_url: 'https://test.com/file.pdf') }
+  let(:reader) { instance_double(PDF::Reader, page_count: 2) }
   let(:file) {
     instance_double(
       Tempfile,
@@ -39,6 +40,7 @@ class Tempfile
   before do
     allow(Down).to receive(:download).with('https://test.com/file.pdf').and_return file
     allow(Down).to receive(:download).with('https://test.com/special%characters!.pdf').and_return special_chars_file
+    allow(PDF::Reader).to receive(:new).and_return(reader)
     allow(S3Handler).to receive(:new).and_return s3
     allow(RemediationStatusNotificationJob).to receive(:perform_later)
     allow(File).to receive(:delete).with(file_path)
@@ -58,6 +60,7 @@ class Tempfile
         expect(reloaded_job.output_url).to eq 'https://example.com/presigned-file-url'
         expect(reloaded_job.finished_at).to be_within(1.minute).of(Time.zone.now)
         expect(reloaded_job.output_object_key).to eq('file.pdf')
+        expect(reloaded_job.page_count).to eq(2)
         expect(reloaded_job.output_url_expires_at).to be_within(1.minute)
           .of(AppJobModule::PRESIGNED_URL_EXPIRES_IN.seconds.from_now)
       end
@@ -80,7 +83,27 @@ class Tempfile
 
         it 'strips the special character for the s3 bucket' do
           described_class.perform_now(special_chars_job.uuid)
-          expect(S3Handler).to have_received(:new).with(match /[a-f0-9]{16}_specialcharacters\.pdf/)
+          expect(S3Handler).to have_received(:new).with(match(/[a-f0-9]{16}_specialcharacters\.pdf/))
+        end
+      end
+
+      context 'when the unit quota would be exceeded' do
+        before do
+          allow(PageCountQuotaValidator).to receive(:validate!).and_raise(
+            PageCountQuotaValidator::QuotaExceededError,
+            "page_count exceeds the unit's overall page limit of 1"
+          )
+        end
+
+        it 'fails the job and does not upload to S3' do
+          described_class.perform_now(job.uuid)
+          reloaded_job = job.reload
+          expect(reloaded_job.status).to eq 'failed'
+          expect(reloaded_job.processing_error_message).to eq(
+            'Failed to process job: PageCountQuotaValidator::QuotaExceededError: page_count ' \
+            "exceeds the unit's overall page limit of 1"
+          )
+          expect(s3).not_to have_received(:upload_to_input)
         end
       end
     end
@@ -121,7 +144,7 @@ class Tempfile
         expect(reloaded_job.output_url).to be_nil
         expect(reloaded_job.finished_at).to be_within(1.minute).of(Time.zone.now)
         expect(reloaded_job.output_object_key).to be_nil
-        expect(reloaded_job.processing_error_message).to eq 'Failed to download file from source URL:  download error'
+        expect(reloaded_job.processing_error_message).to eq 'Failed to process job: Down::Error: download error'
         expect(reloaded_job.output_url_expires_at).to be_nil
       end
 
@@ -143,7 +166,7 @@ class Tempfile
         expect(reloaded_job.output_url).to be_nil
         expect(reloaded_job.finished_at).to be_within(1.minute).of(Time.zone.now)
         expect(reloaded_job.processing_error_message).to eq(
-          'Failed to upload file to remediation input location:  upload error'
+          'Failed to process job: S3Handler::Error: upload error'
         )
         expect(reloaded_job.output_url_expires_at).to be_nil
       end
@@ -158,5 +181,20 @@ class Tempfile
         expect(file).to have_received(:close!)
       end
     end
+
+    context 'when an error occurs while reading the pdf for page counting' do
+      before do
+        allow(PDF::Reader).to receive(:new).and_raise(PDF::Reader::MalformedPDFError, 'bad pdf')
+      end
+
+      it 'fails the job and does not upload to S3' do
+        described_class.perform_now(job.uuid)
+        reloaded_job = job.reload
+        expect(reloaded_job.status).to eq 'failed'
+        expect(reloaded_job.processing_error_message).to eq 'Failed to process job: PDF::Reader::MalformedPDFError: ' \
+                                                            'bad pdf'
+        expect(s3).not_to have_received(:upload_to_input)
+      end
+    end
   end
 end
diff --git a/spec/models/concerns/owner_helpers_spec.rb b/spec/models/concerns/owner_helpers_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe OwnerHelpers do
+  describe '#total_pages_processed_last_24_hours' do
+    it 'sums page_count for jobs created in the last 24 hours (APIUser)' do
+      api_user = create(:api_user)
+      create(:pdf_job, owner: api_user, page_count: 3, created_at: 2.hours.ago)
+      create(:pdf_job, owner: api_user, page_count: 1, created_at: 3.hours.ago)
+      create(:pdf_job, owner: api_user, page_count: 5, created_at: 26.hours.ago)
+      create(:pdf_job, owner: api_user, page_count: nil, created_at: 1.hour.ago)
+
+      expect(api_user.total_pages_processed_last_24_hours).to eq(4)
+    end
+  end
+end
diff --git a/spec/models/unit_spec.rb b/spec/models/unit_spec.rb
diff --git a/spec/validators/page_count_quota_validator_spec.rb b/spec/validators/page_count_quota_validator_spec.rb

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+# frozen_string_literal: true`
	`2`	`+`
`1`	`3`	`class AddPageCountToJob < ActiveRecord::Migration[7.2]`
`2`	`4`	`def change`
`3`	`5`	`add_column :jobs, :page_count, :integer, null: true`