[FIX] multi character value separator

Author: SeeSharpSoft

gradle.properties

@@ -3,5 +3,5 @@
 # https://www.jetbrains.com/intellij-repository/snapshots
 
 name='CSV Plugin'
-javaVersion=8
-javaTargetVersion=8
+javaVersion=9
+javaTargetVersion=9

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerFactory.java

@@ -9,7 +9,7 @@
 public class CsvLexerFactory {
     protected static CsvLexerFactory INSTANCE = new CsvLexerFactory();
 
-    public static final CsvLexerFactory getInstance() {
+    public static CsvLexerFactory getInstance() {
         return INSTANCE;
     }
 

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSharpLexer.java

@@ -106,18 +106,18 @@ public CsvSharpLexer(Configuration configuration) {
         if (configuration.escapeCharacter.equals(configuration.quoteCharacter)) {
             tokenizer.add(TokenType.END_QUOTE, String.format("%s(?!%s)", configuration.quoteCharacter, configuration.quoteCharacter));
             tokenizer.add(TokenType.ESCAPED_CHARACTER, String.format("(%s%s|%s|%s)+", configuration.quoteCharacter, configuration.quoteCharacter, configuration.valueSeparator, configuration.recordSeparator));
-            tokenizer.add(TokenType.TEXT, String.format("[^ \f%s%s%s]+",configuration.quoteCharacter, configuration.valueSeparator, configuration.recordSeparator));
+            tokenizer.add(TokenType.TEXT, String.format("((?!%s)[^ \f%s%s])+", configuration.valueSeparator, configuration.quoteCharacter, configuration.recordSeparator));
         } else {
             tokenizer.add(TokenType.END_QUOTE, String.format("%s", configuration.quoteCharacter));
             tokenizer.add(TokenType.ESCAPED_CHARACTER, String.format("(%s%s|%s%s|%s|%s)+", configuration.escapeCharacter, configuration.quoteCharacter, configuration.escapeCharacter, configuration.escapeCharacter, configuration.valueSeparator, configuration.recordSeparator));
-            tokenizer.add(TokenType.TEXT, String.format("[^ \f%s%s%s%s]+", configuration.escapeCharacter, configuration.quoteCharacter, configuration.valueSeparator, configuration.recordSeparator));
+            tokenizer.add(TokenType.TEXT, String.format("((?!%s)[^ \f%s%s%s])+", configuration.valueSeparator, configuration.escapeCharacter, configuration.quoteCharacter, configuration.recordSeparator));
         }
 
         unquotedNextStateTokens = LexerState.Unquoted.getPossibleTokens().stream()
-                .map(tokenType -> tokenizer.getToken(tokenType))
+                .map(tokenizer::getToken)
                 .collect(Collectors.toList());
         quotedNextStateTokens = LexerState.Quoted.getPossibleTokens().stream()
-                .map(tokenType -> tokenizer.getToken(tokenType))
+                .map(tokenizer::getToken)
                 .collect(Collectors.toList());
     }
 

src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java

@@ -22,6 +22,12 @@ public void testParsingTestDataWithCustomParser() {
         CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
     }
 
+    public void testCustomMultiSymbolSeparator() {
+        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.create("~§"));
+        doTest(true);
+        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
+    }
+
     @Override
     protected String getTestDataPath() {
         return "./src/test/resources/parser";

src/test/resources/parser/CustomMultiSymbolSeparator.csv

@@ -0,0 +1,2 @@
+header ~1~§header §2~§ header ,3
+"value ~1"~§"value §2"~§" value ~§3"

src/test/resources/parser/CustomMultiSymbolSeparator.txt

@@ -0,0 +1,41 @@
+CSV File(0,69)
+  CsvRecordImpl(RECORD)(0,32)
+    CsvFieldImpl(FIELD)(0,9)
+      PsiElement(CsvTokenType.TEXT)('header')(0,6)
+      PsiWhiteSpace(' ')(6,7)
+      PsiElement(CsvTokenType.TEXT)('~1')(7,9)
+    PsiElement(CsvTokenType.COMMA)('~§')(9,11)
+    CsvFieldImpl(FIELD)(11,20)
+      PsiElement(CsvTokenType.TEXT)('header')(11,17)
+      PsiWhiteSpace(' ')(17,18)
+      PsiElement(CsvTokenType.TEXT)('§2')(18,20)
+    PsiElement(CsvTokenType.COMMA)('~§')(20,22)
+    PsiWhiteSpace(' ')(22,23)
+    CsvFieldImpl(FIELD)(23,32)
+      PsiElement(CsvTokenType.TEXT)('header')(23,29)
+      PsiWhiteSpace(' ')(29,30)
+      PsiElement(CsvTokenType.TEXT)(',3')(30,32)
+  PsiElement(CsvTokenType.CRLF)('\n')(32,33)
+  CsvRecordImpl(RECORD)(33,69)
+    CsvFieldImpl(FIELD)(33,43)
+      PsiElement(CsvTokenType.QUOTE)('"')(33,34)
+      PsiElement(CsvTokenType.TEXT)('value')(34,39)
+      PsiWhiteSpace(' ')(39,40)
+      PsiElement(CsvTokenType.TEXT)('~1')(40,42)
+      PsiElement(CsvTokenType.QUOTE)('"')(42,43)
+    PsiElement(CsvTokenType.COMMA)('~§')(43,45)
+    CsvFieldImpl(FIELD)(45,55)
+      PsiElement(CsvTokenType.QUOTE)('"')(45,46)
+      PsiElement(CsvTokenType.TEXT)('value')(46,51)
+      PsiWhiteSpace(' ')(51,52)
+      PsiElement(CsvTokenType.TEXT)('§2')(52,54)
+      PsiElement(CsvTokenType.QUOTE)('"')(54,55)
+    PsiElement(CsvTokenType.COMMA)('~§')(55,57)
+    CsvFieldImpl(FIELD)(57,69)
+      PsiElement(CsvTokenType.QUOTE)('"')(57,58)
+      PsiWhiteSpace(' ')(58,59)
+      PsiElement(CsvTokenType.TEXT)('value')(59,64)
+      PsiWhiteSpace(' ')(64,65)
+      PsiElement(CsvTokenType.ESCAPED_TEXT)('~§')(65,67)
+      PsiElement(CsvTokenType.TEXT)('3')(67,68)
+      PsiElement(CsvTokenType.QUOTE)('"')(68,69)