allow comments

OFL.g4

@@ -1,7 +1,7 @@
 grammar OFL;
 
 
-root: expression? EOF;
+root: WS* expression? WS* EOF;
 
 expression
   : po expression pc
@@ -90,19 +90,19 @@ OR: 'or';
 NOT: 'not';
 IN: 'in';
 
-and: WS? AND WS?;
-or: WS? OR WS?;
-not: WS? NOT WS?;
-in : WS? IN WS?;
+and: WS* AND WS*;
+or: WS* OR WS*;
+not: WS* NOT WS*;
+in : WS* IN WS*;
 
-eq: WS? '=' WS?;
-ne: WS? '!=' WS?;
-po: WS? '(' WS?;
-pc: WS? ')' WS?;
-co: WS? ',' WS?;
-dd: WS? '..' WS?;
-cn: WS? ':' WS?;
-tl: WS? '~' WS?;
+eq: WS* '=' WS*;
+ne: WS* '!=' WS*;
+po: WS* '(' WS*;
+pc: WS* ')' WS*;
+co: WS* ',' WS*;
+dd: WS* '..' WS*;
+cn: WS* ':' WS*;
+tl: WS* '~' WS*;
 
 WILDCARD: '*';
 
@@ -131,10 +131,11 @@ QUOTED: '"' CHARACTER+ '"';
 range_int: po (NUMBER dd NUMBER | dd NUMBER | NUMBER dd ) pc;
 range_dec: po ((NUMBER | DECIMAL) dd (NUMBER | DECIMAL) | dd (NUMBER | DECIMAL) | (NUMBER | DECIMAL) dd) pc;
 
-WS: [ \t\r\n]+;
+COMMENT: '/*' .*? '*/' -> skip;
+LINE_COMMENT: '//' ~[\n\r]* ('\r'? '\n' | EOF) -> skip;
+WS: [ \t\r\n];
 
 fragment NUMERAL: [0-9];
 fragment LETTER: [-_a-zA-Z0-9];
-fragment CHARACTER: ~["\\\r\n] | EscapeSequence;
-fragment EscapeSequence: '\\' ["rn\\];
-
+fragment CHARACTER: ~["\\\r\n] | ESCAPE_SEQUENCE;
+fragment ESCAPE_SEQUENCE: '\\' ["rn\\];

tests/approvals/test_main.py--test_comments[3030543116].approved.txt

@@ -0,0 +1,8 @@
+waterway=stream /* inline
+comment
+with
+linebreaks */ or waterway=river
+
+tags @> '{"waterway": "stream"}' OR tags @> '{"waterway": "river"}'
+
+[<Record count=22746>]

tests/approvals/test_main.py--test_comments[3597371092].approved.txt

@@ -0,0 +1,5 @@
+highway in (unclassified, road /* outdated tag */, service)
+
+tags -> 'highway' IN ('"unclassified"', '"road"', '"service"')
+
+[<Record count=226328>]

tests/approvals/test_main.py--test_comments[505400233].approved.txt

@@ -0,0 +1,7 @@
+waterway=stream
+// full line comment
+or waterway=river
+
+tags @> '{"waterway": "stream"}' OR tags @> '{"waterway": "river"}'
+
+[<Record count=22746>]

tests/approvals/test_main.py--test_comments[516841388].approved.txt

@@ -0,0 +1,5 @@
+natural=tree // this is a tree
+
+tags @> '{"natural": "tree"}'
+
+[<Record count=98199>]

tests/approvals/test_main.py--test_tag_value_pattern_match_invalid[286069758].approved.txt

@@ -1,3 +1,3 @@
 key ~ foo*bar
 
-line 1:10 extraneous input 'bar' expecting <EOF>
+line 1:10 extraneous input 'bar' expecting {<EOF>, WS}

tests/approvals/test_main.py--test_type_id_match_invalid[1628443325].approved.txt

@@ -1,3 +1,3 @@
 id:node/ 4540889804
 
-line 1:7 token recognition error at: '/'
+line 1:7 token recognition error at: '/ '

tests/approvals/test_main.py--test_type_id_match_invalid[2155641098].approved.txt

@@ -1,3 +1,3 @@
 id:node /4540889804
 
-line 1:8 token recognition error at: '/'
+line 1:8 token recognition error at: '/4'

tests/approvals/test_main.py--test_type_id_match_invalid[2910051487].approved.txt

@@ -1,3 +1,3 @@
 id:node / 4540889804
 
-line 1:8 token recognition error at: '/'
+line 1:8 token recognition error at: '/ '

tests/approvals/test_main.py--test_untrimmed_whitespace[2155298507].approved.txt

@@ -0,0 +1,5 @@
+natural=tree 
+
+tags @> '{"natural": "tree"}'
+
+[<Record count=98199>]

tests/approvals/test_main.py--test_untrimmed_whitespace[2480699734].approved.txt

@@ -0,0 +1,5 @@
+ natural=tree 
+
+tags @> '{"natural": "tree"}'
+
+[<Record count=98199>]

tests/approvals/test_main.py--test_untrimmed_whitespace[3672947595].approved.txt

@@ -0,0 +1,5 @@
+ natural=tree
+
+tags @> '{"natural": "tree"}'
+
+[<Record count=98199>]

tests/cassettes/test_main.py--test_comments.cassette.json

@@ -0,0 +1 @@
+{"3915070601": {"results": [{"count": 22746}], "query": "SELECT COUNT(*) FROM contributions WHERE tags @> '{\"waterway\": \"stream\"}' OR tags @> '{\"waterway\": \"river\"}'", "args": [], "limit": 0, "timeout": null, "record_class": null}, "2406030824": {"results": [{"count": 226328}], "query": "SELECT COUNT(*) FROM contributions WHERE tags -> 'highway' IN ('\"unclassified\"', '\"road\"', '\"service\"')", "args": [], "limit": 0, "timeout": null, "record_class": null}, "1868929949": {"results": [{"count": 98199}], "query": "SELECT COUNT(*) FROM contributions WHERE tags @> '{\"natural\": \"tree\"}'", "args": [], "limit": 0, "timeout": null, "record_class": null}}

tests/cassettes/test_main.py--test_untrimmed_whitespace.cassette.json

@@ -0,0 +1 @@
+{"1868929949": {"results": [{"count": 98199}], "query": "SELECT COUNT(*) FROM contributions WHERE tags @> '{\"natural\": \"tree\"}'", "args": [], "limit": 0, "timeout": null, "record_class": null}, "550655163": {"results": [{"count": 98199}], "query": "SELECT COUNT(*) FROM contributions WHERE  tags @> '{\"natural\": \"tree\"}'", "args": [], "limit": 0, "timeout": null, "record_class": null}}

tests/test_main.py

@@ -62,6 +62,36 @@ async def test_build_tree(filter):
 #
 
 
+@pytest.mark.parametrize(
+    "filter",
+    (
+        "natural=tree // this is a tree",
+        "highway in (unclassified, road /* outdated tag */, service)",
+        "waterway=stream\n// full line comment\nor waterway=river",
+        "waterway=stream /* inline\ncomment\nwith\nlinebreaks */ or waterway=river",
+        # TODO: It would be nice to not have to quote urls
+        "website=https://heigit.org",  # unquoted
+        "long_name=name//with//slashes",  # unquoted
+    ),
+)
+async def test_comments(filter):
+    sql = ohsome_filter_to_sql(filter)
+    assert await validate_and_verify(sql, filter)
+
+
+@pytest.mark.parametrize(
+    "filter",
+    (
+        " natural=tree ",
+        " natural=tree",
+        "natural=tree ",
+    ),
+)
+async def test_untrimmed_whitespace(filter):
+    sql = ohsome_filter_to_sql(filter)
+    assert await validate_and_verify(sql, filter)
+
+
 async def test_expression_and_expression():
     filter = "natural=tree and leaf_type=broadleaved"
     sql = ohsome_filter_to_sql(filter)