pull source dir creation into its own stage post-validation

Author: niquerio

config/config.dlxs.yml

@@ -9,6 +9,9 @@ stages:
   - name: Pagination Check
     class: PaginationCheck
     file: pagination_check
+  - name: Create Source Directory
+    class: CreateSourceDirectory
+    file: create_source_directory
   - name: Tagger
     class: Tagger
     file: tagger

config/config.yml

@@ -9,6 +9,9 @@ stages:
   - name: Pagination Check
     class: PaginationCheck
     file: pagination_check
+  - name: Create Source Directory
+    class: CreateSourceDirectory
+    file: create_source_directory
   - name: Tagger
     class: Tagger
     file: tagger

lib/stage/create_source_directory.rb

@@ -0,0 +1,18 @@
+require "stage"
+
+class CreateSourceDirectory < Stage
+  def run(agenda)
+    setup_source_directory
+  end
+
+  private
+
+  def setup_source_directory
+    shipment.setup_source_directory do |objid|
+      @bar.next! "setup source/#{objid}"
+    end
+    shipment.checksum_source_directory do |objid|
+      @bar.next! "checksum source/#{objid}"
+    end
+  end
+end

lib/stage/pagination_check.rb

@@ -66,4 +66,13 @@ def duplicate_pages(pages)
     dups.delete_if { |_k, v| v.size == 1 }
     dups.keys.map(&:to_s)
   end
+
+  def setup_source_directory
+    shipment.setup_source_directory do |objid|
+      @bar.next! "setup source/#{objid}"
+    end
+    shipment.checksum_source_directory do |objid|
+      @bar.next! "checksum source/#{objid}"
+    end
+  end
 end

lib/stage/preflight.rb

@@ -40,9 +40,9 @@ def run(agenda)
     @bar.next! "validate #{File.split(shipment_directory)[-1]}"
     validate_shipment_directory
     validate_objects agenda
-    return if fatal_error?
+    nil if fatal_error?
 
-    setup_source_directory
+    #    setup_source_directory
   end
 
   private

test/config/config.dlxs.yml

@@ -9,6 +9,9 @@ stages:
   - name: Pagination Check
     class: PaginationCheck
     file: pagination_check
+  - name: Create Source Directory
+    class: CreateSourceDirectory
+    file: create_source_directory
   - name: Tagger
     class: Tagger
     file: tagger

test/config/config.dlxsfails.yml

@@ -0,0 +1,30 @@
+shipment_class: DLXSShipment
+stages:
+  - name: Preflight
+    class: Preflight
+    file: preflight
+  - name: Image Validator
+    class: ImageValidator
+    file: image_validator
+  - name: Pagination Check
+    class: PaginationCheck
+    file: pagination_check
+  - name: Create Source Directory
+    class: CreateSourceDirectory
+    file: create_source_directory
+  - name: Fails
+    class: Fails
+    file: fails
+  - name: Tagger
+    class: Tagger
+    file: tagger
+  - name: Compression
+    class: Compression
+    file: compression
+  - name: DLXSCompressor
+    class: DLXSCompressor
+    file: dlxs_compressor
+  - name: Postflight
+    class: Postflight
+    file: postflight
+

test/config/config.fails.yml

@@ -0,0 +1,26 @@
+stages:
+  - name: Preflight
+    class: Preflight
+    file: preflight
+  - name: Image Validator
+    class: ImageValidator
+    file: image_validator
+  - name: Pagination Check
+    class: PaginationCheck
+    file: pagination_check
+  - name: Create Source Directory
+    class: CreateSourceDirectory
+    file: create_source_directory
+  - name: Fails
+    class: Fails
+    file: fails
+  - name: Tagger
+    class: Tagger
+    file: tagger
+  - name: Compression
+    class: Compression
+    file: compression
+  - name: Postflight
+    class: Postflight
+    file: postflight
+

test/config/config.yml

@@ -9,6 +9,9 @@ stages:
   - name: Pagination Check
     class: PaginationCheck
     file: pagination_check
+  - name: Create Source Directory
+    class: CreateSourceDirectory
+    file: create_source_directory
   - name: Tagger
     class: Tagger
     file: tagger

test/postflight_test.rb

@@ -2,6 +2,9 @@
 # frozen_string_literal: true
 
 require "minitest/autorun"
+require_relative "test_helper"
+require "create_source_directory"
+require "preflight"
 require "postflight"
 
 class PostflightTest < Minitest::Test
@@ -32,6 +35,8 @@ def self.gen_run
       shipment = shipment_class.new(test_shipment.directory)
       stage = Preflight.new(shipment, config: opts.merge(@config))
       stage.run!
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
+      stage.run!
       stage = Postflight.new(shipment, config: opts.merge(@config))
       stage.run!
       assert_equal 0, stage.errors.count, "stage runs without errors"
@@ -46,6 +51,8 @@ def self.gen_metadata_mismatch_removed
       shipment = shipment_class.new(test_shipment.directory)
       stage = Preflight.new(shipment, config: opts.merge(@config))
       stage.run!
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
+      stage.run!
       FileUtils.rm_r(File.join(shipment.directory,
         shipment.objid_to_path(shipment.objids[0])),
         force: true)
@@ -62,7 +69,7 @@ def self.gen_metadata_mismatch_added
       spec = "BC T bitonal 1 BC T bitonal 1"
       test_shipment = test_shipment_class.new(dir, spec)
       shipment = shipment_class.new(test_shipment.directory)
-      stage = Preflight.new(shipment, config: opts.merge(@config))
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
       stage.run!
       new_objid = test_shipment_class.generate_objid
       FileUtils.mkdir_p File.join(shipment.directory,
@@ -79,7 +86,7 @@ def self.gen_feed_validate_error
     test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
       test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
       shipment = shipment_class.new(test_shipment.directory)
-      stage = Preflight.new(shipment, config: opts.merge(@config))
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
       stage.run!
       tiff = File.join(shipment.objid_to_path(shipment.objids[0]),
         "00000001.tif")
@@ -105,7 +112,7 @@ def self.gen_feed_validate_crash
       ENV["FAKE_FEED_VALIDATE_CRASH"] = "1"
       test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
       shipment = shipment_class.new(test_shipment.directory)
-      stage = Preflight.new(shipment, config: opts.merge(@config))
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
       stage.run!
       stage = Postflight.new(shipment, config: opts.merge(@config))
       stage.run!
@@ -120,7 +127,7 @@ def self.gen_checksum_mismatch
     test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
       test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
       shipment = shipment_class.new(test_shipment.directory)
-      stage = Preflight.new(shipment, config: opts.merge(@config))
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
       stage.run!
       tiff = File.join(shipment.source_directory,
         shipment.objid_to_path(shipment.objids[0]),
@@ -138,7 +145,7 @@ def self.gen_file_missing
     test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
       test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
       shipment = shipment_class.new(test_shipment.directory)
-      stage = Preflight.new(shipment, config: opts.merge(@config))
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
       stage.run!
       tiff = File.join(shipment.source_directory,
         shipment.objid_to_path(shipment.objids[0]),
@@ -156,7 +163,7 @@ def self.gen_file_added
     test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
       test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
       shipment = shipment_class.new(test_shipment.directory)
-      stage = Preflight.new(shipment, config: opts.merge(@config))
+      stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
       stage.run!
       tiff = File.join(shipment.source_directory,
         shipment.objid_to_path(shipment.objids[0]),