Skip to content

Commit 816602b

Browse files
mergify[bot]mergify[bot]
authored
Merge pull request #711 from common-workflow-language/typedsl_multi_dimensional
Support nested typeDSL
2 parents d6a4085 + 77c2483 commit 816602b

File tree

11 files changed

+153
-84
lines changed

11 files changed

+153
-84
lines changed

‎schema_salad/codegen.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ def codegen(
5757
else ".".join(list(reversed(sp.netloc.split("."))) + sp.path.strip("/").split("/"))
5858
)
5959
info = parser_info or pkg
60+
salad_version = schema_metadata.get("saladVersion", "v1.1")
61+
6062
if lang in set(["python", "cpp", "dlang"]):
6163
if target:
6264
dest: Union[TextIOWrapper, TextIO] = open(target, mode="w", encoding="utf-8")
@@ -83,7 +85,9 @@ def codegen(
8385
)
8486
gen.parse(j)
8587
return
86-
gen = PythonCodeGen(dest, copyright=copyright, parser_info=info)
88+
gen = PythonCodeGen(
89+
dest, copyright=copyright, parser_info=info, salad_version=salad_version
90+
)
8791

8892
elif lang == "java":
8993
gen = JavaCodeGen(

‎schema_salad/main.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,10 @@ def main(argsl: Optional[List[str]] = None) -> int:
299299
raise ValidationException(f"Expected a CommentedSeq, got {type(schema_doc)}: {schema_doc}.")
300300

301301
# Create the loader that will be used to load the target document.
302-
document_loader = Loader(schema_ctx, skip_schemas=args.skip_schemas)
302+
schema_version = schema_metadata.get("saladVersion", None)
303+
document_loader = Loader(
304+
schema_ctx, skip_schemas=args.skip_schemas, salad_version=schema_version
305+
)
303306

304307
if args.codegen:
305308
codegen.codegen(

‎schema_salad/metaschema.py

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import logging
77
import os
88
import pathlib
9-
import re
109
import tempfile
1110
import uuid as _uuid__ # pylint: disable=unused-import # noqa: F401
1211
import xml.sax # nosec
@@ -574,41 +573,47 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
574573

575574

576575
class _TypeDSLLoader(_Loader):
577-
typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$")
578-
579-
def __init__(self, inner, refScope):
580-
# type: (_Loader, Union[int, None]) -> None
576+
def __init__(self, inner, refScope, salad_version):
577+
# type: (_Loader, Union[int, None], str) -> None
581578
self.inner = inner
582579
self.refScope = refScope
580+
self.salad_version = salad_version
583581

584582
def resolve(
585583
self,
586584
doc, # type: str
587585
baseuri, # type: str
588586
loadingOptions, # type: LoadingOptions
589587
):
590-
# type: (...) -> Union[List[Union[Dict[str, str], str]], Dict[str, str], str]
591-
m = self.typeDSLregex.match(doc)
592-
if m:
593-
group1 = m.group(1)
594-
assert group1 is not None # nosec
595-
first = expand_url(group1, baseuri, loadingOptions, False, True, self.refScope)
596-
second = third = None
597-
if bool(m.group(2)):
598-
second = {"type": "array", "items": first}
599-
# second = CommentedMap((("type", "array"),
600-
# ("items", first)))
601-
# second.lc.add_kv_line_col("type", lc)
602-
# second.lc.add_kv_line_col("items", lc)
603-
# second.lc.filename = filename
604-
if bool(m.group(3)):
605-
third = ["null", second or first]
606-
# third = CommentedSeq(["null", second or first])
607-
# third.lc.add_kv_line_col(0, lc)
608-
# third.lc.add_kv_line_col(1, lc)
609-
# third.lc.filename = filename
610-
return third or second or first
611-
return doc
588+
# type: (...) -> Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str]
589+
doc_ = doc
590+
optional = False
591+
if doc_.endswith("?"):
592+
optional = True
593+
doc_ = doc_[0:-1]
594+
595+
if doc_.endswith("[]"):
596+
salad_versions = [int(v) for v in self.salad_version[1:].split(".")]
597+
items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str]
598+
rest = doc_[0:-2]
599+
if salad_versions < [1, 3]:
600+
if rest.endswith("[]"):
601+
# To show the error message with the original type
602+
return doc
603+
else:
604+
items = expand_url(rest, baseuri, loadingOptions, False, True, self.refScope)
605+
else:
606+
items = self.resolve(rest, baseuri, loadingOptions)
607+
if isinstance(items, str):
608+
items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope)
609+
expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str]
610+
else:
611+
expanded = expand_url(doc_, baseuri, loadingOptions, False, True, self.refScope)
612+
613+
if optional:
614+
return ["null", expanded]
615+
else:
616+
return expanded
612617

613618
def load(self, doc, baseuri, loadingOptions, docRoot=None):
614619
# type: (Any, str, LoadingOptions, Optional[str]) -> Any
@@ -3576,6 +3581,7 @@ def save(
35763581
typedsl_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_2 = _TypeDSLLoader(
35773582
union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype,
35783583
2,
3584+
"v1.1",
35793585
)
35803586
array_of_RecordFieldLoader = _ArrayLoader(RecordFieldLoader)
35813587
union_of_None_type_or_array_of_RecordFieldLoader = _UnionLoader(
@@ -3588,7 +3594,7 @@ def save(
35883594
union_of_None_type_or_array_of_RecordFieldLoader, "name", "type"
35893595
)
35903596
Record_nameLoader = _EnumLoader(("record",), "Record_name")
3591-
typedsl_Record_nameLoader_2 = _TypeDSLLoader(Record_nameLoader, 2)
3597+
typedsl_Record_nameLoader_2 = _TypeDSLLoader(Record_nameLoader, 2, "v1.1")
35923598
union_of_None_type_or_strtype = _UnionLoader(
35933599
(
35943600
None_type,
@@ -3600,15 +3606,15 @@ def save(
36003606
)
36013607
uri_array_of_strtype_True_False_None = _URILoader(array_of_strtype, True, False, None)
36023608
Enum_nameLoader = _EnumLoader(("enum",), "Enum_name")
3603-
typedsl_Enum_nameLoader_2 = _TypeDSLLoader(Enum_nameLoader, 2)
3609+
typedsl_Enum_nameLoader_2 = _TypeDSLLoader(Enum_nameLoader, 2, "v1.1")
36043610
uri_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_False_True_2 = _URILoader(
36053611
union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype_or_array_of_union_of_PrimitiveTypeLoader_or_RecordSchemaLoader_or_EnumSchemaLoader_or_ArraySchemaLoader_or_strtype,
36063612
False,
36073613
True,
36083614
2,
36093615
)
36103616
Array_nameLoader = _EnumLoader(("array",), "Array_name")
3611-
typedsl_Array_nameLoader_2 = _TypeDSLLoader(Array_nameLoader, 2)
3617+
typedsl_Array_nameLoader_2 = _TypeDSLLoader(Array_nameLoader, 2, "v1.1")
36123618
union_of_None_type_or_booltype = _UnionLoader(
36133619
(
36143620
None_type,
@@ -3665,7 +3671,7 @@ def save(
36653671
union_of_None_type_or_array_of_SpecializeDefLoader, "specializeFrom", "specializeTo"
36663672
)
36673673
Documentation_nameLoader = _EnumLoader(("documentation",), "Documentation_name")
3668-
typedsl_Documentation_nameLoader_2 = _TypeDSLLoader(Documentation_nameLoader, 2)
3674+
typedsl_Documentation_nameLoader_2 = _TypeDSLLoader(Documentation_nameLoader, 2, "v1.1")
36693675
union_of_SaladRecordSchemaLoader_or_SaladEnumSchemaLoader_or_DocumentationLoader = (
36703676
_UnionLoader(
36713677
(

‎schema_salad/metaschema/typedsl_res.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,14 @@
66
77
* If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null`
88
* If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol
9-
* The type may end with both `[]?` to indicate it is an optional array.
9+
* The type may end with both square brackets with one question mark (`[]?`) to indicate it is an optional array.
1010
* Identifier resolution is applied after type DSL expansion.
1111
12+
Starting with Schema Salad version 1.3, fields tagged with `typeDSL: true` in `jsonldPredicate` have the following additional behavior:
13+
14+
* Square brackes `[]` can be repeated to indicate 2, 3, or more dimensional array types.
15+
* These multi-dimensional arrays, like 1-dimensional arrays, can be combined with `?` (for example, `[][]?`) to indicate that it is an optional multi-dimensional array.
16+
1217
### Type DSL example
1318
1419
Given the following schema:

‎schema_salad/python_codegen.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ def __init__(
8282
out: IO[str],
8383
copyright: Optional[str],
8484
parser_info: str,
85+
salad_version: str,
8586
) -> None:
8687
super().__init__()
8788
self.out = out
@@ -90,6 +91,7 @@ def __init__(
9091
self.idfield = ""
9192
self.copyright = copyright
9293
self.parser_info = parser_info
94+
self.salad_version = salad_version
9395

9496
@staticmethod
9597
def safe_name(name: str) -> str:
@@ -629,7 +631,7 @@ def typedsl_loader(self, inner: TypeDef, ref_scope: Optional[int]) -> TypeDef:
629631
return self.declare_type(
630632
TypeDef(
631633
f"typedsl_{self.safe_name(inner.name)}_{ref_scope}",
632-
f"_TypeDSLLoader({self.safe_name(inner.name)}, {ref_scope})",
634+
f"_TypeDSLLoader({self.safe_name(inner.name)}, {ref_scope}, '{self.salad_version}')",
633635
)
634636
)
635637

‎schema_salad/python_codegen_support.py

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import logging
44
import os
55
import pathlib
6-
import re
76
import tempfile
87
import uuid as _uuid__ # pylint: disable=unused-import # noqa: F401
98
import xml.sax # nosec
@@ -571,41 +570,47 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
571570

572571

573572
class _TypeDSLLoader(_Loader):
574-
typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$")
575-
576-
def __init__(self, inner, refScope):
577-
# type: (_Loader, Union[int, None]) -> None
573+
def __init__(self, inner, refScope, salad_version):
574+
# type: (_Loader, Union[int, None], str) -> None
578575
self.inner = inner
579576
self.refScope = refScope
577+
self.salad_version = salad_version
580578

581579
def resolve(
582580
self,
583581
doc, # type: str
584582
baseuri, # type: str
585583
loadingOptions, # type: LoadingOptions
586584
):
587-
# type: (...) -> Union[List[Union[Dict[str, str], str]], Dict[str, str], str]
588-
m = self.typeDSLregex.match(doc)
589-
if m:
590-
group1 = m.group(1)
591-
assert group1 is not None # nosec
592-
first = expand_url(group1, baseuri, loadingOptions, False, True, self.refScope)
593-
second = third = None
594-
if bool(m.group(2)):
595-
second = {"type": "array", "items": first}
596-
# second = CommentedMap((("type", "array"),
597-
# ("items", first)))
598-
# second.lc.add_kv_line_col("type", lc)
599-
# second.lc.add_kv_line_col("items", lc)
600-
# second.lc.filename = filename
601-
if bool(m.group(3)):
602-
third = ["null", second or first]
603-
# third = CommentedSeq(["null", second or first])
604-
# third.lc.add_kv_line_col(0, lc)
605-
# third.lc.add_kv_line_col(1, lc)
606-
# third.lc.filename = filename
607-
return third or second or first
608-
return doc
585+
# type: (...) -> Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str]
586+
doc_ = doc
587+
optional = False
588+
if doc_.endswith("?"):
589+
optional = True
590+
doc_ = doc_[0:-1]
591+
592+
if doc_.endswith("[]"):
593+
salad_versions = [int(v) for v in self.salad_version[1:].split(".")]
594+
items = "" # type: Union[List[Union[Dict[str, Any], str]], Dict[str, Any], str]
595+
rest = doc_[0:-2]
596+
if salad_versions < [1, 3]:
597+
if rest.endswith("[]"):
598+
# To show the error message with the original type
599+
return doc
600+
else:
601+
items = expand_url(rest, baseuri, loadingOptions, False, True, self.refScope)
602+
else:
603+
items = self.resolve(rest, baseuri, loadingOptions)
604+
if isinstance(items, str):
605+
items = expand_url(items, baseuri, loadingOptions, False, True, self.refScope)
606+
expanded = {"type": "array", "items": items} # type: Union[Dict[str, Any], str]
607+
else:
608+
expanded = expand_url(doc_, baseuri, loadingOptions, False, True, self.refScope)
609+
610+
if optional:
611+
return ["null", expanded]
612+
else:
613+
return expanded
609614

610615
def load(self, doc, baseuri, loadingOptions, docRoot=None):
611616
# type: (Any, str, LoadingOptions, Optional[str]) -> Any

‎schema_salad/ref_resolver.py

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@
5151
)
5252

5353
_logger = logging.getLogger("salad")
54-
typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$")
5554

5655

5756
def file_uri(path: str, split_frag: bool = False) -> str:
@@ -141,6 +140,7 @@ def SubLoader(loader: "Loader") -> "Loader":
141140
url_fields=loader.url_fields,
142141
allow_attachments=loader.allow_attachments,
143142
session=loader.session,
143+
salad_version=loader.salad_version,
144144
)
145145

146146

@@ -158,6 +158,7 @@ def __init__(
158158
url_fields: Optional[Set[str]] = None,
159159
allow_attachments: Optional[AttachmentsType] = None,
160160
doc_cache: Union[str, bool] = True,
161+
salad_version: Optional[str] = None,
161162
) -> None:
162163
self.idx: IdxType = (
163164
NormDict(lambda url: urllib.parse.urlsplit(url).geturl()) if idx is None else idx
@@ -207,6 +208,11 @@ def __init__(
207208
self.secondaryFile_dsl_fields: Set[str] = set()
208209
self.allow_attachments = allow_attachments
209210

211+
if salad_version:
212+
self.salad_version = salad_version
213+
else:
214+
self.salad_version = "v1.1"
215+
210216
self.add_context(ctx)
211217

212218
def expand_url(
@@ -631,23 +637,41 @@ def _type_dsl(
631637
if not isinstance(t, str):
632638
return t
633639

634-
m = typeDSLregex.match(t)
635-
if not m:
636-
return t
637-
first = m.group(1)
638-
assert first # nosec
639-
second = third = None
640-
if bool(m.group(2)):
641-
second = CommentedMap((("type", "array"), ("items", first)))
642-
second.lc.add_kv_line_col("type", lc)
643-
second.lc.add_kv_line_col("items", lc)
644-
second.lc.filename = filename
645-
if bool(m.group(3)):
646-
third = CommentedSeq(["null", second or first])
647-
third.lc.add_kv_line_col(0, lc)
648-
third.lc.add_kv_line_col(1, lc)
649-
third.lc.filename = filename
650-
return third or second or first
640+
t_ = t
641+
optional = False
642+
if t_.endswith("?"):
643+
optional = True
644+
t_ = t_[0:-1]
645+
646+
if t_.endswith("[]"):
647+
salad_versions = [int(v) for v in self.salad_version[1:].split(".")]
648+
rest = t_[0:-2]
649+
if salad_versions < [1, 3]:
650+
if rest.endswith("[]"):
651+
# To show the error message with the original type
652+
return t
653+
else:
654+
cmap = CommentedMap((("type", "array"), ("items", rest)))
655+
else:
656+
items = self._type_dsl(rest, lc, filename)
657+
cmap = CommentedMap((("type", "array"), ("items", items)))
658+
cmap.lc.add_kv_line_col("type", lc)
659+
cmap.lc.add_kv_line_col("items", lc)
660+
cmap.lc.filename = filename
661+
expanded: Union[str, CommentedMap, CommentedSeq] = cmap
662+
else:
663+
expanded = t_
664+
665+
if optional:
666+
cs = CommentedSeq(["null", expanded])
667+
cs.lc.add_kv_line_col(0, lc)
668+
cs.lc.add_kv_line_col(1, lc)
669+
cs.lc.filename = filename
670+
ret: Union[str, CommentedMap, CommentedSeq] = cs
671+
else:
672+
ret = expanded
673+
674+
return ret
651675

652676
def _secondaryFile_dsl(
653677
self,

‎schema_salad/schema.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,8 @@ def get_metaschema() -> Tuple[Names, List[Dict[str, str]], Loader]:
177177
},
178178
"typeDSL": saladp + "JsonldPredicate/typeDSL",
179179
"xsd": "http://www.w3.org/2001/XMLSchema#",
180-
}
180+
},
181+
salad_version="v1.3",
181182
)
182183

183184
for salad in SALAD_FILES:

‎schema_salad/tests/metaschema-pre.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
"```\n\nThis becomes:\n\n```\n",
4141
"{\n \"mapped\": [\n {\n \"value\": \"daphne\",\n \"key\": \"fred\"\n },\n {\n \"value\": \"scooby\",\n \"key\": \"shaggy\"\n }\n ]\n}\n",
4242
"```\n",
43-
"## Domain Specific Language for types\n\nFields may be tagged `typeDSL: true` in `jsonldPredicate`. If so, the field is expanded using the\nfollowing micro-DSL for schema salad types:\n\n* If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null`\n* If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol\n* The type may end with both `[]?` to indicate it is an optional array.\n* Identifier resolution is applied after type DSL expansion.\n\n### Type DSL example\n\nGiven the following schema:\n\n```\n",
43+
"## Domain Specific Language for types\n\nFields may be tagged `typeDSL: true` in `jsonldPredicate`. If so, the field is expanded using the\nfollowing micro-DSL for schema salad types:\n\n* If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null`\n* If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol\n* The type may end with both square brackets with one question mark (`[]?`) to indicate it is an optional array.\n* Identifier resolution is applied after type DSL expansion.\n\nStarting with Schema Salad version 1.3, fields tagged with `typeDSL: true` in `jsonldPredicate` have the following additional behavior:\n\n* Square brackes `[]` can be repeated to indicate 2, 3, or more dimensional array types.\n* These multi-dimensional arrays, like 1-dimensional arrays, can be combined with `?` (for example, `[][]?`) to indicate that it is an optional multi-dimensional array.\n\n### Type DSL example\n\nGiven the following schema:\n\n```\n",
4444
"{\n \"$graph\": [\n {\"$import\": \"metaschema_base.yml\"},\n {\n \"name\": \"TypeDSLExample\",\n \"type\": \"record\",\n \"documentRoot\": true,\n \"fields\": [{\n \"name\": \"extype\",\n \"type\": \"string\",\n \"jsonldPredicate\": {\n _type: \"@vocab\",\n \"typeDSL\": true\n }\n }]\n }]\n}\n",
4545
"```\n\nProcess the following example:\n\n```\n",
4646
"[{\n \"extype\": \"string\"\n}, {\n \"extype\": \"string?\"\n}, {\n \"extype\": \"string[]\"\n}, {\n \"extype\": \"string[]?\"\n}]\n",

0 commit comments

Comments
 (0)