fix comment lexing

SeeSharpSoft · SeeSharpSoft · commit 31e8d7d9d0bc · 2022-10-26T08:50:27.000+02:00
diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex
@@ -39,8 +39,9 @@ VALUE_SEPARATOR=[,:;|\t]
 RECORD_SEPARATOR=\n
 ESCAPED_QUOTE=\"\"|\\\"
 QUOTE=\"
-TEXT=[^ ,:;|\t\r\n\"\\]+
+TEXT=[^ ,:;|\t\r\n\"\\#]+
 BACKSLASH=\\+
+HASHTAG=#
 COMMENT=\#[^\n]*
 
 %state UNQUOTED
@@ -91,6 +92,11 @@ COMMENT=\#[^\n]*
     return CsvTypes.TEXT;
 }
 
+<UNQUOTED, QUOTED> {HASHTAG}
+{
+    return CsvTypes.TEXT;
+}
+
 <QUOTED> {TEXT}
 {
     return CsvTypes.TEXT;
diff --git a/src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java b/src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java
@@ -85,7 +85,6 @@ public void testCsvWithCommentsWithoutCommentSupport() {
     }
 
     public void testCsvWithCommentsWithoutCommentSupportCustom() {
-        setName("CsvWithCommentsWithoutCommentSupport");
         // enforce custom lexer
         CsvEditorSettings.getInstance().setDefaultValueSeparator(new CsvValueSeparator(","));
         CsvEditorSettings.getInstance().setCommentIndicator("");
diff --git a/src/test/resources/parser/CsvWithComments.csv b/src/test/resources/parser/CsvWithComments.csv
@@ -1,4 +1,5 @@
 header#1,header#2
   # just a comment with whitespace at the start, "yes" - still a comment
 "# not a comment",#and this also not
-#but here again:,|	a comment
+#but here again:,|	a comment
+#comment
diff --git a/src/test/resources/parser/CsvWithComments.txt b/src/test/resources/parser/CsvWithComments.txt
@@ -1,10 +1,14 @@
-CSV File(0,156)
+CSV File(0,165)
   CsvRecordImpl(RECORD)(0,17)
     CsvFieldImpl(FIELD)(0,8)
-      PsiElement(CsvTokenType.TEXT)('header#1')(0,8)
+      PsiElement(CsvTokenType.TEXT)('header')(0,6)
+      PsiElement(CsvTokenType.TEXT)('#')(6,7)
+      PsiElement(CsvTokenType.TEXT)('1')(7,8)
     PsiElement(CsvTokenType.COMMA)(',')(8,9)
     CsvFieldImpl(FIELD)(9,17)
-      PsiElement(CsvTokenType.TEXT)('header#2')(9,17)
+      PsiElement(CsvTokenType.TEXT)('header')(9,15)
+      PsiElement(CsvTokenType.TEXT)('#')(15,16)
+      PsiElement(CsvTokenType.TEXT)('2')(16,17)
   PsiElement(CsvTokenType.CRLF)('\n')(17,18)
   PsiWhiteSpace('  ')(18,20)
   CsvRecordImpl(RECORD)(20,90)
@@ -23,7 +27,8 @@ CSV File(0,156)
       PsiElement(CsvTokenType.QUOTE)('"')(107,108)
     PsiElement(CsvTokenType.COMMA)(',')(108,109)
     CsvFieldImpl(FIELD)(109,127)
-      PsiElement(CsvTokenType.TEXT)('#and')(109,113)
+      PsiElement(CsvTokenType.TEXT)('#')(109,110)
+      PsiElement(CsvTokenType.TEXT)('and')(110,113)
       PsiWhiteSpace(' ')(113,114)
       PsiElement(CsvTokenType.TEXT)('this')(114,118)
       PsiWhiteSpace(' ')(118,119)
@@ -32,4 +37,7 @@ CSV File(0,156)
       PsiElement(CsvTokenType.TEXT)('not')(124,127)
   PsiElement(CsvTokenType.CRLF)('\n')(127,128)
   CsvRecordImpl(RECORD)(128,156)
-    PsiElement(CsvTokenType.COMMENT)('#but here again:,|\ta comment')(128,156)
+    PsiElement(CsvTokenType.COMMENT)('#but here again:,|\ta comment')(128,156)
+  PsiElement(CsvTokenType.CRLF)('\n')(156,157)
+  CsvRecordImpl(RECORD)(157,165)
+    PsiElement(CsvTokenType.COMMENT)('#comment')(157,165)
diff --git a/src/test/resources/parser/CsvWithCommentsCustom.csv b/src/test/resources/parser/CsvWithCommentsCustom.csv
@@ -1,4 +1,5 @@
 header#1,header#2
   // just a comment with whitespace at the start, "yes" - still a comment
 "# not a comment",#and this also not
-//but here again:,|	a comment
+//but here again:,|	a comment
+//comment
diff --git a/src/test/resources/parser/CsvWithCommentsCustom.txt b/src/test/resources/parser/CsvWithCommentsCustom.txt
@@ -1,4 +1,4 @@
-CSV File(0,158)
+CSV File(0,168)
   CsvRecordImpl(RECORD)(0,17)
     CsvFieldImpl(FIELD)(0,8)
       PsiElement(CsvTokenType.TEXT)('header#1')(0,8)
@@ -32,4 +32,7 @@ CSV File(0,158)
       PsiElement(CsvTokenType.TEXT)('not')(125,128)
   PsiElement(CsvTokenType.CRLF)('\n')(128,129)
   CsvRecordImpl(RECORD)(129,158)
-    PsiElement(CsvTokenType.COMMENT)('//but here again:,|\ta comment')(129,158)
+    PsiElement(CsvTokenType.COMMENT)('//but here again:,|\ta comment')(129,158)
+  PsiElement(CsvTokenType.CRLF)('\n')(158,159)
+  CsvRecordImpl(RECORD)(159,168)
+    PsiElement(CsvTokenType.COMMENT)('//comment')(159,168)
diff --git a/src/test/resources/parser/CsvWithCommentsWithoutCommentSupport.csv b/src/test/resources/parser/CsvWithCommentsWithoutCommentSupport.csv
@@ -1,4 +1,5 @@
 header#1,header#2
   # this should not be recognized as a comment, this would be the second value
 "# not a comment",#and this also not
-# and here again.,"no comment, just values"
+# and here again.,"no comment, just values"
+#comment
diff --git a/src/test/resources/parser/CsvWithCommentsWithoutCommentSupport.txt b/src/test/resources/parser/CsvWithCommentsWithoutCommentSupport.txt
@@ -1,10 +1,14 @@
-CSV File(0,177)
+CSV File(0,186)
   CsvRecordImpl(RECORD)(0,17)
     CsvFieldImpl(FIELD)(0,8)
-      PsiElement(CsvTokenType.TEXT)('header#1')(0,8)
+      PsiElement(CsvTokenType.TEXT)('header')(0,6)
+      PsiElement(CsvTokenType.TEXT)('#')(6,7)
+      PsiElement(CsvTokenType.TEXT)('1')(7,8)
     PsiElement(CsvTokenType.COMMA)(',')(8,9)
     CsvFieldImpl(FIELD)(9,17)
-      PsiElement(CsvTokenType.TEXT)('header#2')(9,17)
+      PsiElement(CsvTokenType.TEXT)('header')(9,15)
+      PsiElement(CsvTokenType.TEXT)('#')(15,16)
+      PsiElement(CsvTokenType.TEXT)('2')(16,17)
   PsiElement(CsvTokenType.CRLF)('\n')(17,18)
   PsiWhiteSpace('  ')(18,20)
   CsvRecordImpl(RECORD)(20,96)
@@ -54,7 +58,8 @@ CSV File(0,177)
       PsiElement(CsvTokenType.QUOTE)('"')(113,114)
     PsiElement(CsvTokenType.COMMA)(',')(114,115)
     CsvFieldImpl(FIELD)(115,133)
-      PsiElement(CsvTokenType.TEXT)('#and')(115,119)
+      PsiElement(CsvTokenType.TEXT)('#')(115,116)
+      PsiElement(CsvTokenType.TEXT)('and')(116,119)
       PsiWhiteSpace(' ')(119,120)
       PsiElement(CsvTokenType.TEXT)('this')(120,124)
       PsiWhiteSpace(' ')(124,125)
@@ -82,4 +87,9 @@ CSV File(0,177)
       PsiElement(CsvTokenType.TEXT)('just')(165,169)
       PsiWhiteSpace(' ')(169,170)
       PsiElement(CsvTokenType.TEXT)('values')(170,176)
-      PsiElement(CsvTokenType.QUOTE)('"')(176,177)
+      PsiElement(CsvTokenType.QUOTE)('"')(176,177)
+  PsiElement(CsvTokenType.CRLF)('\n')(177,178)
+  CsvRecordImpl(RECORD)(178,186)
+    CsvFieldImpl(FIELD)(178,186)
+      PsiElement(CsvTokenType.TEXT)('#')(178,179)
+      PsiElement(CsvTokenType.TEXT)('comment')(179,186)
diff --git a/src/test/resources/parser/CsvWithCommentsWithoutCommentSupportCustom.csv b/src/test/resources/parser/CsvWithCommentsWithoutCommentSupportCustom.csv
@@ -0,0 +1,5 @@
+header#1,header#2
+  # this should not be recognized as a comment, this would be the second value
+"# not a comment",#and this also not
+# and here again.,"no comment, just values"
+#comment
diff --git a/src/test/resources/parser/CsvWithCommentsWithoutCommentSupportCustom.txt b/src/test/resources/parser/CsvWithCommentsWithoutCommentSupportCustom.txt
@@ -0,0 +1,89 @@
+CSV File(0,186)
+  CsvRecordImpl(RECORD)(0,17)
+    CsvFieldImpl(FIELD)(0,8)
+      PsiElement(CsvTokenType.TEXT)('header#1')(0,8)
+    PsiElement(CsvTokenType.COMMA)(',')(8,9)
+    CsvFieldImpl(FIELD)(9,17)
+      PsiElement(CsvTokenType.TEXT)('header#2')(9,17)
+  PsiElement(CsvTokenType.CRLF)('\n')(17,18)
+  PsiWhiteSpace('  ')(18,20)
+  CsvRecordImpl(RECORD)(20,96)
+    CsvFieldImpl(FIELD)(20,64)
+      PsiElement(CsvTokenType.TEXT)('#')(20,21)
+      PsiWhiteSpace(' ')(21,22)
+      PsiElement(CsvTokenType.TEXT)('this')(22,26)
+      PsiWhiteSpace(' ')(26,27)
+      PsiElement(CsvTokenType.TEXT)('should')(27,33)
+      PsiWhiteSpace(' ')(33,34)
+      PsiElement(CsvTokenType.TEXT)('not')(34,37)
+      PsiWhiteSpace(' ')(37,38)
+      PsiElement(CsvTokenType.TEXT)('be')(38,40)
+      PsiWhiteSpace(' ')(40,41)
+      PsiElement(CsvTokenType.TEXT)('recognized')(41,51)
+      PsiWhiteSpace(' ')(51,52)
+      PsiElement(CsvTokenType.TEXT)('as')(52,54)
+      PsiWhiteSpace(' ')(54,55)
+      PsiElement(CsvTokenType.TEXT)('a')(55,56)
+      PsiWhiteSpace(' ')(56,57)
+      PsiElement(CsvTokenType.TEXT)('comment')(57,64)
+    PsiElement(CsvTokenType.COMMA)(',')(64,65)
+    PsiWhiteSpace(' ')(65,66)
+    CsvFieldImpl(FIELD)(66,96)
+      PsiElement(CsvTokenType.TEXT)('this')(66,70)
+      PsiWhiteSpace(' ')(70,71)
+      PsiElement(CsvTokenType.TEXT)('would')(71,76)
+      PsiWhiteSpace(' ')(76,77)
+      PsiElement(CsvTokenType.TEXT)('be')(77,79)
+      PsiWhiteSpace(' ')(79,80)
+      PsiElement(CsvTokenType.TEXT)('the')(80,83)
+      PsiWhiteSpace(' ')(83,84)
+      PsiElement(CsvTokenType.TEXT)('second')(84,90)
+      PsiWhiteSpace(' ')(90,91)
+      PsiElement(CsvTokenType.TEXT)('value')(91,96)
+  PsiElement(CsvTokenType.CRLF)('\n')(96,97)
+  CsvRecordImpl(RECORD)(97,133)
+    CsvFieldImpl(FIELD)(97,114)
+      PsiElement(CsvTokenType.QUOTE)('"')(97,98)
+      PsiElement(CsvTokenType.TEXT)('#')(98,99)
+      PsiWhiteSpace(' ')(99,100)
+      PsiElement(CsvTokenType.TEXT)('not')(100,103)
+      PsiWhiteSpace(' ')(103,104)
+      PsiElement(CsvTokenType.TEXT)('a')(104,105)
+      PsiWhiteSpace(' ')(105,106)
+      PsiElement(CsvTokenType.TEXT)('comment')(106,113)
+      PsiElement(CsvTokenType.QUOTE)('"')(113,114)
+    PsiElement(CsvTokenType.COMMA)(',')(114,115)
+    CsvFieldImpl(FIELD)(115,133)
+      PsiElement(CsvTokenType.TEXT)('#and')(115,119)
+      PsiWhiteSpace(' ')(119,120)
+      PsiElement(CsvTokenType.TEXT)('this')(120,124)
+      PsiWhiteSpace(' ')(124,125)
+      PsiElement(CsvTokenType.TEXT)('also')(125,129)
+      PsiWhiteSpace(' ')(129,130)
+      PsiElement(CsvTokenType.TEXT)('not')(130,133)
+  PsiElement(CsvTokenType.CRLF)('\n')(133,134)
+  CsvRecordImpl(RECORD)(134,177)
+    CsvFieldImpl(FIELD)(134,151)
+      PsiElement(CsvTokenType.TEXT)('#')(134,135)
+      PsiWhiteSpace(' ')(135,136)
+      PsiElement(CsvTokenType.TEXT)('and')(136,139)
+      PsiWhiteSpace(' ')(139,140)
+      PsiElement(CsvTokenType.TEXT)('here')(140,144)
+      PsiWhiteSpace(' ')(144,145)
+      PsiElement(CsvTokenType.TEXT)('again.')(145,151)
+    PsiElement(CsvTokenType.COMMA)(',')(151,152)
+    CsvFieldImpl(FIELD)(152,177)
+      PsiElement(CsvTokenType.QUOTE)('"')(152,153)
+      PsiElement(CsvTokenType.TEXT)('no')(153,155)
+      PsiWhiteSpace(' ')(155,156)
+      PsiElement(CsvTokenType.TEXT)('comment')(156,163)
+      PsiElement(CsvTokenType.ESCAPED_TEXT)(',')(163,164)
+      PsiWhiteSpace(' ')(164,165)
+      PsiElement(CsvTokenType.TEXT)('just')(165,169)
+      PsiWhiteSpace(' ')(169,170)
+      PsiElement(CsvTokenType.TEXT)('values')(170,176)
+      PsiElement(CsvTokenType.QUOTE)('"')(176,177)
+  PsiElement(CsvTokenType.CRLF)('\n')(177,178)
+  CsvRecordImpl(RECORD)(178,186)
+    CsvFieldImpl(FIELD)(178,186)
+      PsiElement(CsvTokenType.TEXT)('#comment')(178,186)

Original file line number	Diff line number	Diff line change
`@@ -85,7 +85,6 @@ public void testCsvWithCommentsWithoutCommentSupport() {`
`85`	`85`	`}`
`86`	`86`
`87`	`87`	`public void testCsvWithCommentsWithoutCommentSupportCustom() {`
`88`		`- setName("CsvWithCommentsWithoutCommentSupport");`
`89`	`88`	`// enforce custom lexer`
`90`	`89`	`CsvEditorSettings.getInstance().setDefaultValueSeparator(new CsvValueSeparator(","));`
`91`	`90`	`CsvEditorSettings.getInstance().setCommentIndicator("");`