Refactor DVZ/P models to multiple service classes

Fixes #2519

Author: mjgiarlo

app/models/replication/druid_version_zip.rb

@@ -5,18 +5,18 @@
 # This model's data is populated by Replication::DeliveryDispatcherJob.
 class ZipPart < ApplicationRecord
   belongs_to :zipped_moab_version, inverse_of: :zip_parts
-  delegate :zip_endpoint, :preserved_object, :druid_version_zip, to: :zipped_moab_version
+  delegate :zip_endpoint, :preserved_object, :zip_part_pathfinder, to: :zipped_moab_version
 
   validates :md5, presence: true, format: { with: /\A[0-9a-f]{32}\z/ }
   validates :size, presence: true, numericality: { only_integer: true, greater_than: 0 }
   validates :suffix, presence: true, format: { with: /\A\.z(ip|[0-9]+)\z/ }
 
-  def druid_version_zip_part
-    @druid_version_zip_part ||= Replication::DruidVersionZipPart.new(druid_version_zip, s3_key)
+  def zip_part_file
+    @zip_part_file ||= Replication::ZipPartFile.new(filename: s3_key)
   end
 
   def s3_key
-    druid_version_zip.s3_key(suffix)
+    zip_part_pathfinder.s3_key(suffix:)
   end
 
   def s3_part

app/models/zip_part.rb

@@ -44,10 +44,16 @@ def update_status_details
     self.status_details = nil if status_changed? && !status_details_changed?
   end
 
-  def druid_version_zip
-    @druid_version_zip ||= Replication::DruidVersionZip.new(
+  def zip_part_pathfinder
+    @zip_part_pathfinder ||= Replication::ZipPartPathfinder.new(
+      druid: preserved_object.druid,
+      version:,
       # In tests, a PreservedObject may not have a MoabRecord, hence the safe navigation.
-      preserved_object.druid, version, preserved_object&.moab_record&.moab_storage_root&.storage_location # rubocop:disable Style/SafeNavigationChainLength
+      storage_location: preserved_object&.moab_record&.moab_storage_root&.storage_location # rubocop:disable Style/SafeNavigationChainLength
     )
   end
+
+  def filesystem_size
+    @filesystem_size ||= Replication::MoabVersionFiles.new(root: zip_part_pathfinder.moab_version_root).size
+  end
 end

app/models/zipped_moab_version.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Replication
+  # Replication-related exceptions
+  module Errors
+    # Raised when Moab version root is not found
+    class MoabVersionNotFound < RuntimeError; end
+
+    # Raised when a file is unreadable due to access restrictions, stale
+    # handles, I/O snafus, and so on
+    class UnreadableFile < RuntimeError; end
+
+    # Raised when the zip command fails
+    class ZipmakerFailure < RuntimeError; end
+  end
+end

app/services/replication/errors.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Replication
+  # Calculate the total size of all files in a Moab and ensure they are readable
+  class MoabVersionFiles
+    # @param [String] root The root path for a particular Moab
+    def initialize(root:)
+      @root = root
+    end
+
+    # @raise [Replication::Errors::MoabVersionNotFound] when Moab version root is not found
+    # @return [Integer] the sum of all file sizes in the Moab
+    def size
+      @size ||= files.sum { |f| File.size(f) }
+    end
+
+    # Raises an error if any of the files in the moab are not readable, for example, due to
+    # Ceph MDS instance for a prescat worker VM thinking that a file is a stray as a result of our
+    # particular use of hard-linking in preservation ingest. Allows for a quick directory walk before
+    # attempting to create the zip file(s).
+    #
+    # @see https://github.com/sul-dlss/preservation_catalog/issues/1633
+    # @raise [Replication::Errors::UnreadableFile] wraps an underlying file access / readability exception
+    # @raise [Replication::Errors::MoabVersionNotFound] when Moab version root is not found
+    # @return [NilClass] indicates success
+    def ensure_readable!
+      files.each { |f| File.stat(f) }
+
+      nil
+    rescue StandardError => e # e.g., Errno::EACCES, Errno::EIO, Errno::ENOENT, Errno::ESTALE
+      raise Errors::UnreadableFile, "Error reading files (#{e.class}): #{e.message}", e.backtrace
+    end
+
+    private
+
+    def files
+      raise Errors::MoabVersionNotFound, "Moab version does not exist: #{root}" unless File.exist?(root)
+
+      @files ||= Dir.glob("#{root}/**/*").select { |path| File.file?(path) }
+    end
+
+    attr_reader :root
+  end
+end

app/services/replication/moab_version_files.rb

@@ -37,7 +37,7 @@ def call # rubocop:disable Metrics/AbcSize
       replicate_incomplete_zipped_moab_versions
 
       # Delete the local zip part files
-      druid_version_zip.cleanup_zip_parts!
+      ZipPartCleaner.clean!(pathfinder: zip_part_pathfinder)
     end
 
     private
@@ -50,15 +50,19 @@ def zipped_moab_versions
       preserved_object.zipped_moab_versions.where(version:)
     end
 
-    def druid_version_zip
-      @druid_version_zip ||= zipped_moab_versions.first.druid_version_zip
+    def zip_part_pathfinder
+      @zip_part_pathfinder ||= zipped_moab_versions.first.zip_part_pathfinder
+    end
+
+    def zip_part_files
+      @zip_part_files ||= zip_part_pathfinder.zip_keys.map { |zip_key| ZipPartFile.new(filename: zip_key) }
     end
 
     def create_zip_if_necessary
-      return if druid_version_zip.complete?
+      return if ZipPartCompletenessChecker.complete?(pathfinder: zip_part_pathfinder)
 
-      druid_version_zip.cleanup_zip_parts!
-      druid_version_zip.create_zip!
+      ZipPartCleaner.clean!(pathfinder: zip_part_pathfinder)
+      ZipPartCreator.create!(pathfinder: zip_part_pathfinder)
     end
 
     def reset_to_created!(zipped_moab_version)
@@ -83,11 +87,11 @@ def check_zip_parts_to_zip_file(zipped_moab_version)
 
     def populate_zip_parts!(zipped_moab_version)
       ZippedMoabVersion.transaction do
-        druid_version_zip.druid_version_zip_parts.each do |druid_version_zip_part|
+        zip_part_files.each do |zip_part_file|
           zipped_moab_version.zip_parts.create!(
-            suffix: druid_version_zip_part.extname,
-            size: druid_version_zip_part.size,
-            md5: druid_version_zip_part.read_md5
+            suffix: zip_part_file.extname,
+            size: zip_part_file.size,
+            md5: zip_part_file.read_md5
           )
         end
         zipped_moab_version.update!(zip_parts_count: zipped_moab_version.zip_parts.count)

app/services/replication/replicate_version_service.rb

@@ -17,7 +17,7 @@ def call
 
       return results if already_replicated? || !check_existing_part_file_on_endpoint
 
-      transfer_manager.upload_file(druid_version_zip_part.file_path,
+      transfer_manager.upload_file(zip_part_file.file_path,
                                    bucket: zip_part.zip_endpoint.bucket_name,
                                    key: zip_part.s3_key, metadata:)
       results
@@ -27,7 +27,7 @@ def call
 
     attr_reader :zip_part
 
-    delegate :s3_part, :druid_version_zip_part, :preserved_object, :zip_endpoint, to: :zip_part
+    delegate :s3_part, :zip_part_file, :preserved_object, :zip_endpoint, to: :zip_part
     delegate :druid, to: :preserved_object
 
     def zip_part_file_exists_on_endpoint?
@@ -53,8 +53,8 @@ def set_hb_context
 
     def metadata
       {
-        checksum_md5: druid_version_zip_part.read_md5,
-        size: druid_version_zip_part.size.to_s # S3 metadata values must be strings
+        checksum_md5: zip_part_file.read_md5,
+        size: zip_part_file.size.to_s # S3 metadata values must be strings
       }
     end
 

app/services/replication/replicate_zip_part_service.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Replication
+  # Deletes all zip part files and their md5 sidecar files from local zip storage
+  class ZipPartCleaner
+    # @param [ZipPartPathfinder] pathfinder The pathfinder instance for a zip part
+    def self.clean!(pathfinder:)
+      FileUtils.rm_f(pathfinder.all_file_paths)
+    end
+  end
+end