Skip to content

Commit aaaece1

Browse files
mr-cmr-c
authored
Merge pull request #994 from common-workflow-language/nested_anonymous_types
avroize all the things
2 parents 047e69b + 8a93d33 commit aaaece1

File tree

4 files changed

+114
-168
lines changed

4 files changed

+114
-168
lines changed

‎cwltool/process.py

Lines changed: 7 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -208,24 +208,6 @@ def checkRequirements(rec, supported_process_requirements):
208208
checkRequirements(entry, supported_process_requirements)
209209

210210

211-
def adjustFilesWithSecondary(rec, op, primary=None):
212-
"""Apply a mapping function to each File path in the object `rec`, propagating
213-
the primary file associated with a group of secondary files.
214-
"""
215-
216-
if isinstance(rec, MutableMapping):
217-
if rec.get("class") == "File":
218-
rec["path"] = op(rec["path"], primary=primary)
219-
adjustFilesWithSecondary(rec.get("secondaryFiles", []), op,
220-
primary if primary else rec["path"])
221-
else:
222-
for d in rec:
223-
adjustFilesWithSecondary(rec[d], op)
224-
if isinstance(rec, MutableSequence):
225-
for d in rec:
226-
adjustFilesWithSecondary(d, op, primary)
227-
228-
229211
def stage_files(pathmapper, # type: PathMapper
230212
stage_func=None, # type: Callable[..., Any]
231213
ignore_writable=False, # type: bool
@@ -396,8 +378,8 @@ def avroize_type(field_type, name_prefix=""):
396378
adds missing information to a type so that CWL types are valid in schema_salad.
397379
"""
398380
if isinstance(field_type, MutableSequence):
399-
for f in field_type:
400-
avroize_type(f, name_prefix)
381+
for field in field_type:
382+
avroize_type(field, name_prefix)
401383
elif isinstance(field_type, MutableMapping):
402384
if field_type["type"] in ("enum", "record"):
403385
if "name" not in field_type:
@@ -406,6 +388,9 @@ def avroize_type(field_type, name_prefix=""):
406388
avroize_type(field_type["fields"], name_prefix)
407389
if field_type["type"] == "array":
408390
avroize_type(field_type["items"], name_prefix)
391+
if isinstance(field_type["type"], MutableSequence):
392+
for ctype in field_type["type"]:
393+
avroize_type(ctype, name_prefix)
409394
return field_type
410395

411396
def get_overrides(overrides, toolid): # type: (List[Dict[Text, Any]], Text) -> Dict[Text, Any]
@@ -499,8 +484,8 @@ def __init__(self,
499484
sd, _ = self.get_requirement("SchemaDefRequirement")
500485

501486
if sd is not None:
502-
sdtypes = sd["types"]
503-
av = schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set())
487+
sdtypes = avroize_type(sd["types"])
488+
av = schema.make_valid_avro(sdtypes, {t["name"]: t for t in sdtypes}, set())
504489
for i in av:
505490
self.schemaDefs[i["name"]] = i # type: ignore
506491
schema.make_avsc_object(schema.convert_to_dict(av), self.names)
@@ -800,25 +785,6 @@ def job(self,
800785
pass
801786

802787

803-
def empty_subtree(dirpath): # type: (Text) -> bool
804-
# Test if a directory tree contains any files (does not count empty
805-
# subdirectories)
806-
for d in os.listdir(dirpath):
807-
d = os.path.join(dirpath, d)
808-
try:
809-
if stat.S_ISDIR(os.stat(d).st_mode):
810-
if empty_subtree(d) is False:
811-
return False
812-
else:
813-
return False
814-
except OSError as e:
815-
if e.errno == errno.ENOENT:
816-
pass
817-
else:
818-
raise
819-
return True
820-
821-
822788
_names = set() # type: Set[Text]
823789

824790

‎tests/test_anon_types.py

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import pytest
2+
3+
from cwltool.command_line_tool import CommandLineTool
4+
from cwltool.context import LoadingContext
5+
6+
7+
snippet = [{
8+
"cwlVersion": "v1.0",
9+
"class": "CommandLineTool",
10+
"inputs": [
11+
{
12+
"type": {
13+
"type": "record",
14+
"fields": [
15+
{
16+
"type": [
17+
{
18+
"type": "enum",
19+
"symbols": [
20+
"anon_enum_inside_array.cwl#first/species/homo_sapiens",
21+
"anon_enum_inside_array.cwl#first/species/mus_musculus"
22+
]
23+
},
24+
"null"
25+
],
26+
"name": "anon_enum_inside_array.cwl#first/species"
27+
}
28+
]
29+
},
30+
"id": "anon_enum_inside_array.cwl#first"
31+
},
32+
{
33+
"type": [
34+
"null",
35+
{
36+
"type": "enum",
37+
"symbols": [
38+
"anon_enum_inside_array.cwl#second/homo_sapiens",
39+
"anon_enum_inside_array.cwl#second/mus_musculus"
40+
]
41+
}
42+
],
43+
"id": "anon_enum_inside_array.cwl#second"
44+
}
45+
],
46+
"baseCommand": "echo",
47+
"outputs": [],
48+
"id": "anon_enum_inside_array.cwl",
49+
},{
50+
"cwlVersion": "v1.0",
51+
"class": "CommandLineTool",
52+
"requirements": [
53+
{
54+
"types": [
55+
{
56+
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params",
57+
"type": "record",
58+
"fields": [
59+
{
60+
"type": [
61+
"null",
62+
{
63+
"type": "enum",
64+
"symbols": [
65+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCh37",
66+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCh38",
67+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCm38"
68+
]
69+
}
70+
],
71+
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build"
72+
},
73+
{
74+
"type": [
75+
"null",
76+
{
77+
"type": "enum",
78+
"symbols": [
79+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species/homo_sapiens",
80+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species/mus_musculus"
81+
]
82+
}
83+
],
84+
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species"
85+
}
86+
]
87+
}
88+
],
89+
"class": "SchemaDefRequirement"
90+
}
91+
],
92+
"inputs": [
93+
{
94+
"type": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params",
95+
"id": "anon_enum_inside_array_inside_schemadef.cwl#first"
96+
}
97+
],
98+
"baseCommand": "echo",
99+
"outputs": [
100+
],
101+
"id": "anon_enum_inside_array_inside_schemadef.cwl",
102+
}]
103+
104+
105+
@pytest.mark.parametrize('snippet', snippet)
106+
def test_anon_types(snippet):
107+
CommandLineTool(snippet, LoadingContext())

‎typeshed/2and3/avro/__init__.pyi

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎typeshed/2and3/avro/schema.pyi

Lines changed: 0 additions & 117 deletions
This file was deleted.

0 commit comments

Comments
 (0)