Skip to content

Commit 24a3493

Browse files
tetrontetron
authored
Simplify interface of main() (#794)
* Simplify interface of main(), can provide custom loadingContext/runtimeContext. * Fix make_path_mapper hook. * Make parameters to Builder optional
1 parent a500e8f commit 24a3493

File tree

10 files changed

+136
-101
lines changed

10 files changed

+136
-101
lines changed

‎cwltool/argparser.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ def get_default_args():
267267
Get default values of cwltool's command line options
268268
"""
269269
ap = arg_parser()
270-
args = ap.parse_args()
270+
args = ap.parse_args([])
271271
return vars(args)
272272

273273

‎cwltool/builder.py

Lines changed: 60 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -103,43 +103,73 @@ def get_requirement(self,
103103

104104
class Builder(HasReqsHints):
105105
def __init__(self,
106-
job, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
107-
files, # type: List[Dict[Text, Text]]
108-
bindings, # type: List[Dict[Text, Any]]
109-
schemaDefs, # type: Dict[Text, Dict[Text, Any]]
110-
names, # type: schema_salad.schema.Names
111-
requirements, # type: List[Dict[Text, Any]]
112-
hints, # type: List[Dict[Text, Any]]
113-
timeout, # type: float
114-
debug, # type: bool
115-
resources, # type: Dict[Text, int]
116-
js_console, # type: bool
117-
mutation_manager, # type: Optional[MutationManager]
118-
formatgraph, # type: Optional[Graph]
119-
make_fs_access, # type: Type[StdFsAccess]
120-
fs_access, # type: StdFsAccess
121-
force_docker_pull, # type: bool
122-
loadListing, # type: Text
123-
outdir, # type: Text
124-
tmpdir, # type: Text
125-
stagedir, # type: Text
126-
job_script_provider # type: Optional[Any]
106+
job, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
107+
files=None, # type: List[Dict[Text, Text]]
108+
bindings=None, # type: List[Dict[Text, Any]]
109+
schemaDefs=None, # type: Dict[Text, Dict[Text, Any]]
110+
names=None, # type: schema_salad.schema.Names
111+
requirements=None, # type: List[Dict[Text, Any]]
112+
hints=None, # type: List[Dict[Text, Any]]
113+
timeout=None, # type: float
114+
debug=False, # type: bool
115+
resources=None, # type: Dict[Text, int]
116+
js_console=False, # type: bool
117+
mutation_manager=None, # type: Optional[MutationManager]
118+
formatgraph=None, # type: Optional[Graph]
119+
make_fs_access=None, # type: Type[StdFsAccess]
120+
fs_access=None, # type: StdFsAccess
121+
force_docker_pull=False, # type: bool
122+
loadListing=u"", # type: Text
123+
outdir=u"", # type: Text
124+
tmpdir=u"", # type: Text
125+
stagedir=u"", # type: Text
126+
job_script_provider=None # type: Optional[Any]
127127
): # type: (...) -> None
128-
self.names = names
129-
self.schemaDefs = schemaDefs
130-
self.files = files
131-
self.fs_access = fs_access
128+
129+
if names is None:
130+
self.names = schema_salad.schema.Names()
131+
else:
132+
self.names = names
133+
134+
if schemaDefs is None:
135+
self.schemaDefs = {} # type: Dict[Text, Dict[Text, Any]]
136+
else:
137+
self.schemaDefs = schemaDefs
138+
139+
if files is None:
140+
self.files = [] # type: List[Dict[Text, Text]]
141+
else:
142+
self.files = files
143+
144+
if fs_access is None:
145+
self.fs_access = StdFsAccess("")
146+
else:
147+
self.fs_access = fs_access
148+
132149
self.job = job
133150
self.requirements = requirements
134151
self.hints = hints
135152
self.outdir = outdir
136153
self.tmpdir = tmpdir
137-
self.resources = resources
138-
self.bindings = bindings
154+
155+
if resources is None:
156+
self.resources = {} # type: Dict[Text, int]
157+
else:
158+
self.resources = resources
159+
160+
if bindings is None:
161+
self.bindings = [] # type: List[Dict[Text, Any]]
162+
else:
163+
self.bindings = bindings
139164
self.timeout = timeout
140165
self.pathmapper = None # type: Optional[PathMapper]
141166
self.stagedir = stagedir
142-
self.make_fs_access = make_fs_access
167+
168+
if make_fs_access is None:
169+
self.make_fs_access = StdFsAccess
170+
else:
171+
self.make_fs_access = make_fs_access
172+
143173
self.debug = debug
144174
self.js_console = js_console
145175
self.mutation_manager = mutation_manager
@@ -167,6 +197,7 @@ def bind_input(self,
167197
lead_pos=None, # type: Optional[Union[int, List[int]]]
168198
tail_pos=None, # type: Optional[List[int]]
169199
): # type: (...) -> List[Dict[Text, Any]]
200+
170201
if tail_pos is None:
171202
tail_pos = []
172203
if lead_pos is None:
@@ -379,8 +410,7 @@ def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
379410
if isinstance(ex, list):
380411
return [self.do_eval(v, context, recursive)
381412
for v in ex]
382-
if context is None and isinstance(ex, string_types) and "self" in ex:
383-
return None
413+
384414
return expression.do_eval(ex, self.job, self.requirements,
385415
self.outdir, self.tmpdir,
386416
self.resources,

‎cwltool/command_line_tool.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,6 @@ def check_valid_locations(fs_access, ob):
207207
if ob["class"] == "Directory" and not fs_access.isdir(ob["location"]):
208208
raise validate.ValidationException("Does not exist or is not a Directory: '%s'" % ob["location"])
209209

210-
def make_path_mapper(reffiles, stagedir, basedir, separateDirs=True):
211-
# type: (List[Any], Text, Text, bool) -> PathMapper
212-
return PathMapper(reffiles, basedir, stagedir, separateDirs)
213-
214210

215211
OutputPorts = Dict[Text, Union[None, Text, List[Union[Dict[Text, Any], Text]], Dict[Text, Any]]]
216212

@@ -249,6 +245,10 @@ def make_job_runner(self,
249245
"DockerRequirement under 'requirements'.")
250246
return CommandLineJob
251247

248+
def make_path_mapper(self, reffiles, stagedir, runtimeContext, separateDirs):
249+
# type: (List[Any], Text, RuntimeContext, bool) -> PathMapper
250+
return PathMapper(reffiles, runtimeContext.basedir, stagedir, separateDirs)
251+
252252
def updatePathmap(self, outdir, pathmap, fn):
253253
# type: (Text, PathMapper, Dict) -> None
254254
if "location" in fn and fn["location"] in pathmap:
@@ -367,7 +367,7 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
367367
reffiles = copy.deepcopy(builder.files)
368368

369369
j = self.make_job_runner(runtimeContext)(
370-
builder, builder.job, make_path_mapper, self.requirements,
370+
builder, builder.job, self.make_path_mapper, self.requirements,
371371
self.hints, jobname)
372372
j.successCodes = self.tool.get("successCodes")
373373
j.temporaryFailCodes = self.tool.get("temporaryFailCodes")
@@ -384,8 +384,8 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
384384
_logger.debug(u"[job %s] %s", j.name, json_dumps(job_order,
385385
indent=4))
386386

387-
builder.pathmapper = make_path_mapper(
388-
reffiles, builder.stagedir, runtimeContext.basedir, True)
387+
builder.pathmapper = self.make_path_mapper(
388+
reffiles, builder.stagedir, runtimeContext, True)
389389
builder.requirements = j.requirements
390390

391391
_check_adjust = partial(check_adjust, builder)

‎cwltool/context.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def __init__(self, kwargs=None):
3939
self.metadata = {} # type: Dict[Text, Any]
4040
self.requirements = None
4141
self.hints = None
42-
self.overrides = [] # type: List[Dict[Text, Any]]
42+
self.overrides_list = [] # type: List[Dict[Text, Any]]
4343
self.loader = None # type: Optional[Loader]
4444
self.avsc_names = None # type: Optional[schema.Names]
4545
self.disable_js_validation = False # type: bool
@@ -83,7 +83,7 @@ def __init__(self, kwargs=None):
8383
self.debug = False # type: bool
8484
self.compute_checksum = True # type: bool
8585
self.name = "" # type: Text
86-
self.default_container = None
86+
self.default_container = "" # type: Text
8787
self.find_default_container = None # type: Optional[Callable[[HasReqsHints], Optional[Text]]]
8888
self.cachedir = None # type: Optional[Text]
8989
self.outdir = None # type: Optional[Text]

‎cwltool/job.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,9 +186,11 @@ def _setup(self, runtimeContext): # type: (RuntimeContext) -> None
186186
"file." % (knownfile, self.pathmapper.mapper(knownfile)[0]))
187187

188188
if self.generatefiles["listing"]:
189+
runtimeContext = runtimeContext.copy()
190+
runtimeContext.outdir = self.outdir
189191
self.generatemapper = self.make_path_mapper(
190192
cast(List[Any], self.generatefiles["listing"]),
191-
self.builder.outdir, self.outdir, separateDirs=False)
193+
self.builder.outdir, runtimeContext, False)
192194
_logger.debug(u"[job %s] initial work dir %s", self.name,
193195
json_dumps({p: self.generatemapper.mapper(p)
194196
for p in self.generatemapper.files()},

‎cwltool/load_tool.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ def load_tool(argsworkflow, # type: Union[Text, Dict[Text, Any]]
364364
enable_dev=loadingContext.enable_dev,
365365
strict=loadingContext.strict,
366366
fetcher_constructor=loadingContext.fetcher_constructor,
367-
overrides=loadingContext.overrides,
367+
overrides=loadingContext.overrides_list,
368368
metadata=loadingContext.metadata)
369369

370370
return make_tool(document_loader,

0 commit comments

Comments
 (0)