Skip to content

AIM-47 Move source directory creation to it's own, later stage #145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
niquerio merged 1 commit into from
Jan 26, 2026
Merged

AIM-47 Move source directory creation to it's own, later stage #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions config/config.dlxs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ stages:
- name: Pagination Check
class: PaginationCheck
file: pagination_check
- name: Create Source Directory
class: CreateSourceDirectory
file: create_source_directory
- name: Tagger
class: Tagger
file: tagger
Expand Down
3 changes: 3 additions & 0 deletions config/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ stages:
- name: Pagination Check
class: PaginationCheck
file: pagination_check
- name: Create Source Directory
class: CreateSourceDirectory
file: create_source_directory
- name: Tagger
class: Tagger
file: tagger
Expand Down
18 changes: 18 additions & 0 deletions lib/stage/create_source_directory.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
require "stage"

class CreateSourceDirectory < Stage
def run(agenda)
setup_source_directory
end

private

def setup_source_directory
shipment.setup_source_directory do |objid|
@bar.next! "setup source/#{objid}"
end
shipment.checksum_source_directory do |objid|
@bar.next! "checksum source/#{objid}"
end
end
end
9 changes: 9 additions & 0 deletions lib/stage/pagination_check.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,13 @@ def duplicate_pages(pages)
dups.delete_if { |_k, v| v.size == 1 }
dups.keys.map(&:to_s)
end

def setup_source_directory
shipment.setup_source_directory do |objid|
@bar.next! "setup source/#{objid}"
end
shipment.checksum_source_directory do |objid|
@bar.next! "checksum source/#{objid}"
end
end
end
4 changes: 2 additions & 2 deletions lib/stage/preflight.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ def run(agenda)
@bar.next! "validate #{File.split(shipment_directory)[-1]}"
validate_shipment_directory
validate_objects agenda
return if fatal_error?
nil if fatal_error?

setup_source_directory
# setup_source_directory
end

private
Expand Down
3 changes: 3 additions & 0 deletions test/config/config.dlxs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ stages:
- name: Pagination Check
class: PaginationCheck
file: pagination_check
- name: Create Source Directory
class: CreateSourceDirectory
file: create_source_directory
- name: Tagger
class: Tagger
file: tagger
Expand Down
30 changes: 30 additions & 0 deletions test/config/config.dlxsfails.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
shipment_class: DLXSShipment
stages:
- name: Preflight
class: Preflight
file: preflight
- name: Image Validator
class: ImageValidator
file: image_validator
- name: Pagination Check
class: PaginationCheck
file: pagination_check
- name: Create Source Directory
class: CreateSourceDirectory
file: create_source_directory
- name: Fails
class: Fails
file: fails
- name: Tagger
class: Tagger
file: tagger
- name: Compression
class: Compression
file: compression
- name: DLXSCompressor
class: DLXSCompressor
file: dlxs_compressor
- name: Postflight
class: Postflight
file: postflight

26 changes: 26 additions & 0 deletions test/config/config.fails.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
stages:
- name: Preflight
class: Preflight
file: preflight
- name: Image Validator
class: ImageValidator
file: image_validator
- name: Pagination Check
class: PaginationCheck
file: pagination_check
- name: Create Source Directory
class: CreateSourceDirectory
file: create_source_directory
- name: Fails
class: Fails
file: fails
- name: Tagger
class: Tagger
file: tagger
- name: Compression
class: Compression
file: compression
- name: Postflight
class: Postflight
file: postflight

3 changes: 3 additions & 0 deletions test/config/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ stages:
- name: Pagination Check
class: PaginationCheck
file: pagination_check
- name: Create Source Directory
class: CreateSourceDirectory
file: create_source_directory
- name: Tagger
class: Tagger
file: tagger
Expand Down
19 changes: 13 additions & 6 deletions test/postflight_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
# frozen_string_literal: true

require "minitest/autorun"
require_relative "test_helper"
require "create_source_directory"
require "preflight"
require "postflight"

class PostflightTest < Minitest::Test
Expand Down Expand Up @@ -32,6 +35,8 @@ def self.gen_run
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage.run!
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
stage = Postflight.new(shipment, config: opts.merge(@config))
stage.run!
assert_equal 0, stage.errors.count, "stage runs without errors"
Expand All @@ -46,6 +51,8 @@ def self.gen_metadata_mismatch_removed
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage.run!
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
FileUtils.rm_r(File.join(shipment.directory,
shipment.objid_to_path(shipment.objids[0])),
force: true)
Expand All @@ -62,7 +69,7 @@ def self.gen_metadata_mismatch_added
spec = "BC T bitonal 1 BC T bitonal 1"
test_shipment = test_shipment_class.new(dir, spec)
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
new_objid = test_shipment_class.generate_objid
FileUtils.mkdir_p File.join(shipment.directory,
Expand All @@ -79,7 +86,7 @@ def self.gen_feed_validate_error
test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
tiff = File.join(shipment.objid_to_path(shipment.objids[0]),
"00000001.tif")
Expand All @@ -105,7 +112,7 @@ def self.gen_feed_validate_crash
ENV["FAKE_FEED_VALIDATE_CRASH"] = "1"
test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
stage = Postflight.new(shipment, config: opts.merge(@config))
stage.run!
Expand All @@ -120,7 +127,7 @@ def self.gen_checksum_mismatch
test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
tiff = File.join(shipment.source_directory,
shipment.objid_to_path(shipment.objids[0]),
Expand All @@ -138,7 +145,7 @@ def self.gen_file_missing
test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
tiff = File.join(shipment.source_directory,
shipment.objid_to_path(shipment.objids[0]),
Expand All @@ -156,7 +163,7 @@ def self.gen_file_added
test_proc = proc { |shipment_class, test_shipment_class, dir, opts|
test_shipment = test_shipment_class.new(dir, "BC T bitonal 1 T contone 2")
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
tiff = File.join(shipment.source_directory,
shipment.objid_to_path(shipment.objids[0]),
Expand Down
2 changes: 2 additions & 0 deletions test/preflight_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ def self.gen_run
shipment = shipment_class.new(test_shipment.directory)
stage = Preflight.new(shipment, config: opts.merge(@config))
stage.run!
stage = CreateSourceDirectory.new(shipment, config: opts.merge(@config))
stage.run!
assert_equal 0, stage.errors.count, "stage runs without errors"
assert_equal 2, shipment.metadata[:initial_barcodes].count,
"correct number of initial objids in metadata"
Expand Down
14 changes: 13 additions & 1 deletion test/processor_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,16 @@
require "digest"

require "minitest/autorun"
require_relative "test_helper"
require "processor"
require "fixtures"

class Fails < Stage
def run(agenda)
add_error Error.new("fails!")
end
end

class ProcessorTest < Minitest::Test
def setup
@options = {config_dir: File.join(TEST_ROOT, "config")}
Expand Down Expand Up @@ -161,7 +168,12 @@ def self.gen_finalize

def self.gen_finalize_does_nothing
test_proc = proc { |_shipment_class, test_shipment_class, dir, opts|
test_shipment = test_shipment_class.new(dir, "BC T bad_16bps 1")
test_shipment = test_shipment_class.new(dir, "BC T contone 1")
@options[:config_profile] = if test_shipment_class == DLXSTestShipment
"dlxsfails"
else
"fails"
end
processor = Processor.new(test_shipment.directory, opts.merge(@options))
capture_io do
processor.run
Expand Down