Skip to content

Commit 224a5bc

Browse files
jerowejerowe
committed
wheee fixed hpc and file_rule
1 parent 684fb98 commit 224a5bc

File tree

7 files changed

+194
-28
lines changed

7 files changed

+194
-28
lines changed

‎lib/BioX/Workflow/Command/run/Utils/Rules.pm‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ has 'global_keys' => (
191191
handles => {
192192
all_global_keys => 'elements',
193193
has_global_keys => 'count',
194+
first_index_global_keys => 'first_index',
194195
},
195196
);
196197

@@ -923,7 +924,7 @@ sub sanity_check_fail {
923924
global:
924925
- indir: data/raw
925926
- outdir: data/processed
926-
- file_rule: (sample.*)$
927+
- sample_rule: (sample.*)$
927928
- by_sample_outdir: 1
928929
- find_sample_bydir: 1
929930
- copy1:

‎lib/BioX/Workflow/Command/run/Utils/Samples.pm‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,13 @@ sub get_samples {
165165
$attr->walk_process_data( $self->global_keys );
166166
}
167167

168-
if ( $self->global_attr->can('sample_rule') ) {
168+
#Backwards compatibility
169+
#For both file_rule and sample_rule
170+
if ( $self->first_index_global_keys( sub { $_ eq 'sample_rule' } ) != -1 ) {
171+
$text = $self->global_attr->sample_rule;
172+
}
173+
elsif ( $self->first_index_global_keys( sub { $_ eq 'file_rule' } ) != -1 )
174+
{
169175
$text = $self->global_attr->sample_rule;
170176
}
171177
else {
@@ -186,10 +192,7 @@ sub get_samples {
186192
if ( $whole[0] eq $attr->indir ) {
187193
shift(@whole);
188194
}
189-
190-
#File find puts directory we are looking in, not just subdirs
191195
@basename = map { basename($_) } @whole;
192-
@basename = sort(@basename);
193196
}
194197
}
195198
else {
@@ -199,17 +202,18 @@ sub get_samples {
199202
extras => { follow => 1 },
200203
in => $attr->indir
201204
);
202-
205+
@basename = @whole;
203206
@basename = map { $self->match_samples( $_, $text ) } @whole;
204-
@basename = uniq(@basename);
205-
@basename = sort(@basename);
206207
}
207208

208209
my @sample_files = map { path($_)->absolute } @whole;
209210
@sample_files = sort(@sample_files);
210211

211-
#Throw error if sample don't exist
212-
$self->samples( \@basename ) if @basename;
212+
if (@basename) {
213+
@basename = uniq(@basename);
214+
@basename = sort(@basename);
215+
$self->samples( \@basename );
216+
}
213217
$self->sample_files( \@sample_files ) if @sample_files;
214218

215219
$self->global_attr->samples( dclone( $self->samples ) );
@@ -226,15 +230,15 @@ sub get_samples {
226230
sub check_sample_exist {
227231
my $self = shift;
228232

229-
my $exists = 0;
233+
my $exists = 0;
230234
if ( $self->has_samples && !$self->resample ) {
231235
my (@samples) = $self->sorted_samples;
232236
$self->samples( \@samples );
233237
$self->app_log->info('Samples passed in on command line.');
234238
$exists = 1;
235239
}
236240
elsif ( $self->global_attr->has_samples ) {
237-
my (@samples) = @{$self->global_attr->samples};
241+
my (@samples) = @{ $self->global_attr->samples };
238242
@samples = sort(@samples);
239243
$self->samples( \@samples );
240244
$self->app_log->info('Samples were defined in the global key.');
@@ -256,10 +260,16 @@ sub match_samples {
256260
my $file = shift;
257261
my $text = shift;
258262

259-
my @tmp = fileparse($_);
260-
my ($m) = $tmp[0] =~ qr/$text/;
263+
if ( $text =~ m/\(/ ) {
264+
my @tmp = fileparse($file);
265+
my ($m) = $tmp[0] =~ qr/$text/;
261266

262-
return $m;
267+
return $m;
268+
}
269+
else {
270+
my @tmp = fileparse($file);
271+
return $tmp[0];
272+
}
263273
}
264274

265275
=head3 process_by_sample_outdir

‎lib/BioX/Workflow/Command/run/Utils/WriteMeta.pm‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ sub write_workflow_meta_start {
9090
$self->fh->say("$self->{comment_char}");
9191
$self->fh->say("$self->{comment_char}");
9292

93-
9493
$self->fh->say("$self->{comment_char} Samples:");
95-
$self->fh->say("$self->{comment_char} \t".join(', ', @{$self->samples}));
94+
$self->fh->say(
95+
"$self->{comment_char} \t" . join( ', ', @{ $self->samples } ) );
9696
$self->fh->say("$self->{comment_char}");
9797
$self->fh->say("$self->{comment_char}");
9898

@@ -215,7 +215,7 @@ sub trim {
215215
sub write_hpc_meta {
216216
my $self = shift;
217217

218-
##TODO Fetch Global HPC
218+
##TODO Fix this for mixed data types
219219

220220
$self->local_attr->add_before_meta( ' ### HPC Directives' . "\n" );
221221
if ( ref( $self->local_attr->HPC ) eq 'HASH' ) {
@@ -224,6 +224,7 @@ sub write_hpc_meta {
224224
elsif ( ref( $self->local_attr->HPC ) eq 'ARRAY' ) {
225225
$self->write_hpc_array_meta;
226226
}
227+
227228
}
228229

229230
=head3 write_hpc_hash_meta
@@ -267,8 +268,15 @@ sub write_hpc_array_meta {
267268

268269
my %lookup = ();
269270

270-
%lookup = %{$self->iter_hpc_array($self->global_attr->HPC, \%lookup)};
271-
%lookup = %{$self->iter_hpc_array($self->local_attr->HPC, \%lookup)};
271+
if ( ref( $self->global_attr->HPC ) eq 'ARRAY' ) {
272+
%lookup =
273+
%{ $self->iter_hpc_array( $self->global_attr->HPC, \%lookup ) };
274+
}
275+
elsif(ref($self->global_attr->HPC) eq 'HASH'){
276+
%lookup = %{$self->global_attr->HPC};
277+
}
278+
279+
%lookup = %{ $self->iter_hpc_array( $self->local_attr->HPC, \%lookup ) };
272280

273281
if ( !exists $lookup{jobname} ) {
274282
$self->local_attr->add_before_meta(
@@ -296,7 +304,7 @@ sub iter_hpc_array {
296304
my $aref = shift;
297305
my $lookup = shift;
298306

299-
foreach my $href ( @{ $aref } ) {
307+
foreach my $href ( @{$aref} ) {
300308
if ( ref($href) eq 'HASH' ) {
301309
my @keys = keys %{$href};
302310
map { $lookup->{$_} = $href->{$_} } @keys;

‎t/lib/TestsFor/BioX/Workflow/Command/Test001.pm‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package TestsFor::BioX::Workflow::Command::Test001;
2+
3+
use Test::Class::Moose;
4+
use Cwd;
5+
use FindBin qw($Bin);
6+
use File::Path qw(make_path remove_tree);
7+
use Data::Dumper;
8+
use Capture::Tiny ':all';
9+
use BioX::Workflow::Command;
10+
use YAML::XS;
11+
12+
sub test_001 : Tags(req) {
13+
require_ok('BioX::Workflow::Command');
14+
15+
require_ok('BioX::Workflow::Command::run');
16+
require_ok('BioX::Workflow::Command::new');
17+
require_ok('BioX::Workflow::Command::add');
18+
19+
require_ok('BioX::Workflow::Command::run::Utils::Attributes');
20+
require_ok('BioX::Workflow::Command::run::Utils::Directives');
21+
require_ok('BioX::Workflow::Command::run::Utils::Rules');
22+
require_ok('BioX::Workflow::Command::run::Utils::Samples');
23+
require_ok('BioX::Workflow::Command::run::Utils::WriteMeta');
24+
require_ok('BioX::Workflow::Command::run::Utils::Files::ResolveDeps');
25+
require_ok('BioX::Workflow::Command::run::Utils::Files::TrackChanges');
26+
27+
require_ok('BioX::Workflow::Command::Utils::Create');
28+
require_ok('BioX::Workflow::Command::Utils::Files');
29+
require_ok('BioX::Workflow::Command::Utils::Log');
30+
require_ok('BioX::Workflow::Command::Utils::Plugin');
31+
require_ok('BioX::Workflow::Command::Utils::Traits');
32+
33+
##DEPRACATED
34+
require_ok('BioX::Workflow::Command::Utils::Files::TrackChanges');
35+
}
36+
37+
1;

‎t/lib/TestsFor/BioX/Workflow/Command/run/Test001.pm‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,6 @@ extends 'TestMethod::Base';
1313

1414
#This tests the functionality with sample_by_dir: 0
1515

16-
sub test_000 : Tags(require) {
17-
18-
require_ok('BioX::Workflow::Command');
19-
ok(1);
20-
}
21-
2216
sub write_test_file {
2317
my $test_dir = shift;
2418

@@ -27,7 +21,7 @@ sub write_test_file {
2721
global => [
2822
{ indir => 'data/raw' },
2923
{ outdir => 'data/processed' },
30-
{ sample_rule => "Sample_(\\w+)" },
24+
{ sample_rule => "Sample_(\\d.*)" },
3125
{ gatk => '{$self->outdir}/{$sample}/gatk' },
3226
],
3327
rules => [
@@ -132,10 +126,13 @@ sub test_004 : Tags(global_attr) {
132126
# $test->execute();
133127
$test->load_yaml_workflow;
134128
$test->apply_global_attributes;
129+
$test->get_samples;
135130

136131
is($test->global_attr->indir, 'data/raw', 'Indir matches');
137132
is($test->global_attr->indir->absolute, $test_dir.'/data/raw', 'Absolute Indir matches');
138133
is($test->global_attr->gatk, '{$self->outdir}/{$sample}/gatk', 'GATK matches');
134+
##We have a regular expression for samples here
135+
is_deeply($test->samples, ['01', '02']);
139136
}
140137

141138
1;

‎t/lib/TestsFor/BioX/Workflow/Command/run/Test008.pm‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package TestsFor::BioX::Workflow::Command::run::Test008;
2+
3+
use Test::Class::Moose;
4+
use Cwd;
5+
use FindBin qw($Bin);
6+
use File::Path qw(make_path remove_tree);
7+
use Data::Dumper;
8+
use Capture::Tiny ':all';
9+
use BioX::Workflow::Command;
10+
use YAML::XS;
11+
use Data::Walk;
12+
use File::Slurp;
13+
use File::Spec;
14+
use DateTime;
15+
use DateTime::Format::Strptime;
16+
use Storable qw(dclone);
17+
18+
extends 'TestMethod::Base';
19+
20+
=head1 Purpose
21+
22+
This tests references -
23+
https://github.com/biosails/BioX-Workflow-Command/issues/12 The special HPC var
24+
supports arrays and hashes, but it wasn't properly dealt with when they were
25+
mixed and matched
26+
27+
=cut
28+
29+
sub write_test_file {
30+
my $test_dir = shift;
31+
32+
my $fh;
33+
my $href = {
34+
global => [
35+
{ sample_rule => "Sample_.*" },
36+
{ root_dir => 'data/raw' },
37+
{ indir => '{$self->root_dir}' },
38+
{ outdir => 'data/processed' },
39+
{ find_sample_bydir => 1 },
40+
{ by_sample_outdir => 1 },
41+
],
42+
rules => [
43+
{
44+
t3_rule1 => {
45+
'local' => [
46+
{ root_dir => 'data/raw' },
47+
{
48+
INPUT =>
49+
'{$self->root_dir}/{$sample}/some_input_rule1.{$self->chunk}'
50+
},
51+
{ OUTPUT => ['some_output_rule1'] },
52+
{ HPC => [{'deps' => 'some_dep'}]}
53+
],
54+
process =>
55+
'R1: INDIR: {$self->indir} INPUT: {$self->INPUT} outdir: {$self->outdir} OUTPUT: {$self->OUTPUT->[0]}',
56+
},
57+
},
58+
]
59+
};
60+
61+
#Write out the config
62+
open( $fh, ">$test_dir/conf/test1.1.yml" )
63+
or die print "Couldn't open file! $!";
64+
my $yaml = Dump $href;
65+
print $fh $yaml;
66+
close($fh);
67+
68+
make_path( $test_dir . "/data/raw/Sample_01" );
69+
write_file( $test_dir . "/data/raw/Sample_01/" . "some_input_rule1" );
70+
}
71+
72+
sub construct_tests {
73+
my $test_methods = TestMethod::Base->new();
74+
my $test_dir = $test_methods->make_test_dir();
75+
write_test_file($test_dir);
76+
77+
my $t = "$test_dir/conf/test1.1.yml";
78+
my $test = $test_methods->make_test_env($t);
79+
my $rules = $test->workflow_data->{rules};
80+
81+
return ( $test, $test_dir, $rules );
82+
}
83+
84+
sub test_001 {
85+
my ( $test, $test_dir, $rules ) = construct_tests;
86+
87+
$test->set_rule_names;
88+
$test->filter_rule_keys;
89+
90+
foreach my $rule ( @{$rules} ) {
91+
_init_rule( $test, $rule );
92+
}
93+
94+
$test->post_process_rules;
95+
is_deeply($test->samples, ['Sample_01']);
96+
97+
is_deeply($test->process_obj->{t3_rule1}->{meta}->[18], '#HPC jobname=t3_rule1');
98+
is_deeply($test->process_obj->{t3_rule1}->{meta}->[19], '#HPC deps=some_dep');
99+
ok(1);
100+
}
101+
102+
103+
sub _init_rule {
104+
my $test = shift;
105+
my $rule = shift;
106+
107+
$test->local_rule($rule);
108+
$test->process_rule;
109+
$test->p_rule_name( $test->rule_name );
110+
$test->p_local_attr( dclone( $test->local_attr ) );
111+
}

‎t/test_class_tests.t‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@ use Test::Class::Moose::Runner;
77

88
Test::Class::Moose::Runner->new(
99
test_classes => [
10+
'TestsFor::BioX::Workflow::Command::Test001',
1011
'TestsFor::BioX::Workflow::Command::run::Test001',
1112
'TestsFor::BioX::Workflow::Command::run::Test002',
1213
'TestsFor::BioX::Workflow::Command::run::Test003',
1314
'TestsFor::BioX::Workflow::Command::run::Test004',
1415
'TestsFor::BioX::Workflow::Command::run::Test005',
1516
'TestsFor::BioX::Workflow::Command::run::Test006',
1617
'TestsFor::BioX::Workflow::Command::run::Test007',
18+
'TestsFor::BioX::Workflow::Command::run::Test008',
1719
],
1820
)->runtests;

0 commit comments

Comments
 (0)