Skip to content

Commit b99ddc6

Browse files
author
reidspencer
committed
TokenParser Enhancements
* Add a "literal code" statement to everything_full.riddl * Enhance the token parser to accept that syntax * Adjust test cases affected by the test input change * Add "LiteralCode" token case.
1 parent 9bc08ad commit b99ddc6

File tree

13 files changed

+670
-560
lines changed

13 files changed

+670
-560
lines changed

‎language/input/everything_full.riddl‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,15 @@ context full is {
9494
function whenUnderTheInfluence is {
9595
requires { n: Nothing }
9696
returns { b: Boolean }
97-
"aribtrary statement"
97+
"aribtrary statement"
98+
```scala
99+
// Simulate a creative state
100+
val randomFactor = Math.random() // A random value between 0 and 1
101+
val threshold = 0.7 // Threshold for creativity
102+
103+
// If the random factor exceeds the threshold, consider it a creative state
104+
b = randomFactor > threshold
105+
```
98106
} with {
99107
briefly as "Something is nothing interesting"
100108
}

‎language/jvm/src/test/scala/com/ossuminc/riddl/language/JVMASTTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class JVMASTTest extends AbstractTestingBasis {
2525
ud.format must be(url.toExternalForm)
2626
val lines: scala.collection.Seq[String] = ud.lines.map(_.s)
2727
val head = lines.head
28-
head must include("resolver")
28+
head must include("ossuminc")
2929
}
3030
}
3131
}

‎language/shared/src/main/scala/com/ossuminc/riddl/language/AST.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3926,7 +3926,7 @@ object AST:
39263926
case Predefined(at: At) extends Token(at)
39273927
case Keyword(at: At) extends Token(at)
39283928
case Comment(at: At) extends Token(at)
3929-
case LiteralString(at: At) extends Token(at)
3929+
case LiteralCode(at: At) extends Token(at)
39303930
case MarkdownLine(at: At) extends Token(at)
39313931
case Identifier(at: At) extends Token(at)
39323932
case Numeric(at: At) extends Token(at)

‎language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/CommonParser.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ private[parsing] trait CommonParser(using io: PlatformContext)
7171
}
7272

7373
def literalStrings[u: P]: P[Seq[LiteralString]] = { P(literalString.rep(1)) }
74-
74+
7575
def markdownLines[u: P]: P[Seq[LiteralString]] = {
7676
P(markdownLine.rep(1))
7777
}

‎language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/Keywords.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ object Keywords {
195195
def nebula[u: P]: P[Unit] = keyword(Keyword.nebula)
196196

197197
def on[u: P]: P[Unit] = keyword(Keyword.on)
198-
198+
199199
def onInit[u: P]: P[Unit] = keyword("on init")
200200

201201
def onOther[u: P]: P[Unit] = keyword("on other")

‎language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/NoWhiteSpaceParsers.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,12 @@ private[parsing] trait NoWhiteSpaceParsers {
9797

9898
private final val escape_chars = "\\\\\\\"aefnrt"
9999
private def shortcut[u: P]: P[String] = P("\\" ~ CharIn(escape_chars)).!
100+
100101
def escape[u: P]: P[String] = P(shortcut | hexEscape | unicodeEscape).!./
101102

102-
private def stringChars(c: Char): Boolean = c != '\"' && c != '\\'
103+
def stringChars(c: Char): Boolean = c != '\"' && c != '\\'
103104

104-
private def strChars[u: P]: P[String] = P(CharsWhile(stringChars)).!./
105+
def strChars[u: P]: P[String] = P(CharsWhile(stringChars)).!./
105106

106107
def literalString[u: P]: P[LiteralString] = {
107108
P(

‎language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/Punctuation.scala‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import fastparse.*
1010
object Punctuation {
1111
final val asterisk = "*"
1212
final val atSign = "@"
13+
final val codeQuote = "```"
1314
final val comma = ","
1415
final val colon = ":"
1516
final val curlyOpen = "{"
@@ -48,6 +49,31 @@ object Punctuation {
4849
)
4950

5051
def anyPunctuation[u: P]: P[Unit] = {
52+
P(
53+
StringIn(
54+
asterisk,
55+
atSign,
56+
codeQuote,
57+
comma,
58+
colon,
59+
curlyOpen,
60+
curlyClose,
61+
dot,
62+
equalsSign,
63+
plus,
64+
question,
65+
quote,
66+
roundOpen,
67+
roundClose,
68+
squareOpen,
69+
squareClose,
70+
verticalBar,
71+
undefinedMark
72+
)
73+
)
74+
}
75+
76+
def tokenPunctuation[u: P]: P[Unit] = {
5177
P(
5278
StringIn(
5379
asterisk,

‎language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/TokenParser.scala‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,29 +22,53 @@ import scala.util.control.NonFatal
2222

2323
trait TokenParser extends CommonParser with Readability {
2424

25-
private def numericToken[u:P]: P[Token.Numeric] = {
25+
private def numericToken[u: P]: P[Token.Numeric] = {
2626
P(Index ~~ integer ~~ Index)./.map { case (start, _, end) => Token.Numeric(at(start, end)) }
2727
}
2828

2929
private def punctuationToken[u: P]: P[Token.Punctuation] = {
30-
P(Index ~~ Punctuation.anyPunctuation ~~ Index)./.map { case (start, end) => Token.Punctuation(at(start, end)) }
30+
P(Index ~~ Punctuation.tokenPunctuation ~~ Index)./.map { case (start, end) =>
31+
Token.Punctuation(at(start, end))
32+
}
33+
}
34+
35+
private def notCodeQuote[u:P]: P[Unit] = {
36+
P(AnyChar.rep(1))
3137
}
38+
39+
private def literalCode[u: P]: P[Token.LiteralCode] = {
40+
P(
41+
Index ~~ Punctuation.codeQuote ~~ until3('`','`','`') ~~ Index
42+
).map { case (start: Int, _: String, end: Int) =>
43+
Token.LiteralCode(at(start, end))
44+
}
45+
}
46+
47+
private def stringContent[u: P]: P[Unit] = P(CharsWhile(stringChars) | escape)
3248

3349
private def quotedStringToken[u: P]: P[Token.QuotedString] = {
34-
P(literalString)./.map { case litStr: LiteralString => Token.QuotedString(litStr.loc) }
50+
P(
51+
Index ~~ Punctuation.quote ~~/ stringContent.rep ~~ Punctuation.quote ~~ Index
52+
)./.map { case (start: Int, end: Int) => Token.QuotedString(at(start, end)) }
3553
}
3654

3755
private def readabilityToken[u: P]: P[Token.Readability] = {
38-
P(Index ~~ anyReadability ~~ Index)./.map { case (start, end) => Token.Readability(at(start, end)) }
56+
P(Index ~~ anyReadability ~~ Index)./.map { case (start, end) =>
57+
Token.Readability(at(start, end))
58+
}
3959
}
4060

4161
private def predefinedToken[u: P]: P[Token.Predefined] = {
4262
import com.ossuminc.riddl.language.parsing.PredefType.*
43-
P(Index ~~ PredefTypes.anyPredefType ~~ Index)./.map { case (start, end) => Token.Predefined(at(start, end)) }
63+
P(Index ~~ Keywords.keywords(PredefTypes.anyPredefType) ~~ Index)./.map { case (start, end) =>
64+
Token.Predefined(at(start, end))
65+
}
4466
}
4567

4668
private def keywordToken[u: P]: P[Token.Keyword] = {
47-
P(Index ~~ Keywords.anyKeyword ~~ Index)./.map { case (start, end) => Token.Keyword(at(start, end)) }
69+
P(Index ~~ Keywords.anyKeyword ~~ Index)./.map { case (start, end) =>
70+
Token.Keyword(at(start, end))
71+
}
4872
}
4973

5074
private def commentToken[u: P]: P[Token.Comment] = {
@@ -62,17 +86,20 @@ trait TokenParser extends CommonParser with Readability {
6286
}
6387

6488
private def otherToken[u: P]: P[Token.Other] = {
65-
P(Index ~~ AnyChar.rep(1) ~~ Index)./.map { case (start, end) =>
89+
P(
90+
Index ~~ (!(CharIn(" \n\r") | End) ~~ AnyChar).rep(1) ~~ Index
91+
)./.map { case (start, end) =>
6692
Token.Other(at(start, end))
6793
}
6894
}
6995

7096
def parseAnyToken[u: P]: P[Token] = {
7197
P(
72-
keywordToken |
73-
punctuationToken |
74-
quotedStringToken |
98+
quotedStringToken |
7599
markdownLinesToken |
100+
literalCode |
101+
punctuationToken |
102+
keywordToken |
76103
readabilityToken |
77104
predefinedToken |
78105
identifierToken |
@@ -83,6 +110,6 @@ trait TokenParser extends CommonParser with Readability {
83110
}
84111

85112
def parseAllTokens[u: P]: P[List[Token]] = {
86-
P(Start ~ parseAnyToken.rep(0) ~ End).map(_.toList)
113+
P(Start ~ parseAnyToken.rep(1) ~ End).map(_.toList)
87114
}
88115
}

‎language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/TypeParser.scala‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -318,8 +318,9 @@ private[parsing] trait TypeParser {
318318

319319
private def uniqueIdType[u: P]: P[UniqueId] = {
320320
(Index ~ PredefType.Id ~ Punctuation.roundOpen ~/
321-
maybe(Keyword.entity) ~ pathIdentifier ~ Punctuation.roundClose ~/ Index) map { case (start, pid, end) =>
322-
UniqueId(at(start, end), pid)
321+
maybe(Keyword.entity) ~ pathIdentifier ~ Punctuation.roundClose ~/ Index) map {
322+
case (start, pid, end) =>
323+
UniqueId(at(start, end), pid)
323324
}
324325
}
325326

@@ -328,13 +329,16 @@ private[parsing] trait TypeParser {
328329
}
329330

330331
def enumerator[u: P]: P[Enumerator] = {
331-
P(Index ~~ identifier ~ enumValue ~ withMetaData ~~Index).map { case (start, id, value, metaData, end) =>
332-
Enumerator(at(start, end), id, value, metaData.toContents)
332+
P(Index ~~ identifier ~ enumValue ~ withMetaData ~~ Index).map {
333+
case (start, id, value, metaData, end) =>
334+
Enumerator(at(start, end), id, value, metaData.toContents)
333335
}
334336
}
335337

336338
private def enumerators[u: P]: P[Seq[Enumerator]] = {
337-
enumerator.rep(1, maybe(Punctuation.comma)) | undefined[u, Seq[Enumerator]](Seq.empty[Enumerator])
339+
enumerator.rep(1, maybe(Punctuation.comma)) | undefined[u, Seq[Enumerator]](
340+
Seq.empty[Enumerator]
341+
)
338342

339343
}
340344

@@ -348,7 +352,9 @@ private[parsing] trait TypeParser {
348352
P(
349353
Index ~ Keywords.one ~ of.? ~/ open ~
350354
(Punctuation.undefinedMark.!.map(_ => Seq.empty[AliasedTypeExpression]) |
351-
aliasedTypeExpression.rep(0, P("or" | "|" | ","))) ~ close ~/ Index
355+
aliasedTypeExpression
356+
.rep(0, P(Keywords.or | Punctuation.verticalBar | Punctuation.comma))) ~ close
357+
~/ Index
352358
).map { case (start, contents, end) =>
353359
Alternation(at(start, end), contents.toContents)
354360
}
@@ -525,16 +531,18 @@ private[parsing] trait TypeParser {
525531
Punctuation.plus
526532
).!.? ~/ Index
527533
).map {
528-
case (start, None, None, typ, Some("?"), end) => Optional(at(start, end), typ)
529-
case (start, None, None, typ, Some("+"), end) => OneOrMore(at(start, end), typ)
530-
case (start, None, None, typ, Some("*"), end) => ZeroOrMore(at(start, end), typ)
531-
case (start, Some("many"), None, typ, None, end) => OneOrMore(at(start, end), typ)
532-
case (start, None, Some("optional"), typ, None, end) => Optional(at(start, end), typ)
533-
case (start, Some("many"), Some("optional"), typ, None, end) => ZeroOrMore(at(start, end), typ)
534-
case (start, None, Some("optional"), typ, Some("?"), end) => Optional(at(start, end), typ)
535-
case (start, Some("many"), None, typ, Some("+"), end) => OneOrMore(at(start, end), typ)
536-
case (start, Some("many"), Some("optional"), typ, Some("*"), end) => ZeroOrMore(at(start, end), typ)
537-
case (_, None, None, typ, None, _) => typ
534+
case (start, None, None, typ, Some("?"), end) => Optional(at(start, end), typ)
535+
case (start, None, None, typ, Some("+"), end) => OneOrMore(at(start, end), typ)
536+
case (start, None, None, typ, Some("*"), end) => ZeroOrMore(at(start, end), typ)
537+
case (start, Some("many"), None, typ, None, end) => OneOrMore(at(start, end), typ)
538+
case (start, None, Some("optional"), typ, None, end) => Optional(at(start, end), typ)
539+
case (start, Some("many"), Some("optional"), typ, None, end) =>
540+
ZeroOrMore(at(start, end), typ)
541+
case (start, None, Some("optional"), typ, Some("?"), end) => Optional(at(start, end), typ)
542+
case (start, Some("many"), None, typ, Some("+"), end) => OneOrMore(at(start, end), typ)
543+
case (start, Some("many"), Some("optional"), typ, Some("*"), end) =>
544+
ZeroOrMore(at(start, end), typ)
545+
case (_, None, None, typ, None, _) => typ
538546
case (start, _, _, typ, _, end) =>
539547
error(at(start, end), s"Cannot determine cardinality for $typ")
540548
typ

0 commit comments

Comments
 (0)