Merge pull request #175 from SeeSharpSoft/fb_flexible_escape_character

Customizable escape character

Author: SeeSharpSoft

README.md

@@ -12,6 +12,8 @@ Compatible with _IntelliJ IDEA  PhpStorm  WebStorm  PyCharm  RubyMine  AppCode
 This plugin introduces CSV (_Comma-Separated Values_) as a language to Jetbrains IDE with a syntax definition, structured language elements and associated file types (.csv/.tsv/.psv).
 This enables default editor features like syntax validation, highlighting and inspections for CSV-alike files.
 
+![CSV Plugin Example](./docs/example.png)
+
 ## Features
 
 - CSV/TSV/PSV file detection
@@ -299,15 +301,15 @@ Annasusanna,Amsterdam,         1
 
 ### Actions
 
-#### File specific value separator
+#### File specific value separator & escape character
 
 ![Context menu](./docs/contextmenu.png)
 
-The action to switch the value separator used for CSV syntax validation of a specific file is part of its editors context menu.
+The action to switch the value separator (or escape character) - *which is used for CSV syntax validation of a specific file* - is part of its editors context menu.
 
 
 This action defines how the parser/validator/highlighter/etc. behaves. It does intentionally not change the file content.
-To be more precise: It **does not replace** previous separator characters by new ones or adjust the escaped texts.
+To be more precise: It **does not replace** previous separator/escape characters by new ones or adjust the escaped texts.
 
 #### Adjust column widths (table editor only)
 

docs/contextmenu.png

@@ -15,8 +15,8 @@
   tokenTypeClass="net.seesharpsoft.intellij.plugins.csv.psi.CsvTokenType"
 
   tokens=[
-    TEXT='regexp:[^ ,;|\t\r\n"]+'
-    ESCAPED_TEXT='regexp:([,;|\t\r\n]|"")+'
+    TEXT='regexp:[^ ,;|\t\r\n"\\]+'
+    ESCAPED_TEXT='regexp:[,;|\t\r\n\\]|""|\\"'
     COMMA='regexp:[,;|\t]'
     QUOTE='"'
     CRLF='regexp:\n'
@@ -29,6 +29,6 @@ record ::= field ( << separator >> COMMA field)*
 
 field ::= (escaped | nonEscaped)
 
-private escaped ::= QUOTE ( TEXT | ESCAPED_TEXT)* QUOTE
+private escaped ::= QUOTE ( TEXT | << escapeCharacter >> ESCAPED_TEXT)* QUOTE
 
-private nonEscaped ::= TEXT*
+private nonEscaped ::= TEXT*

docs/example.png

@@ -17,6 +17,8 @@
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.util.PsiTreeUtil;
 import net.seesharpsoft.intellij.lang.FileParserDefinition;
+import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
+import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvField;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvRecord;
@@ -26,6 +28,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
 public final class CsvHelper {
 
@@ -149,14 +152,23 @@ public static int getFieldEndOffset(PsiElement field) {
         return separator == null ? field.getContainingFile().getTextLength() : separator.getTextOffset();
     }
 
+    public static CsvEditorSettings.EscapeCharacter getCurrentEscapeCharacter(CsvFile csvFile) {
+        return getCurrentEscapeCharacter(csvFile.getContainingFile());
+    }
+
+    public static CsvEditorSettings.EscapeCharacter getCurrentEscapeCharacter(PsiFile psiFile) {
+        return CsvFileAttributes.getInstance(psiFile.getProject()).getEscapeCharacter(psiFile);
+    }
+
     public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile) {
+        CsvEditorSettings.EscapeCharacter escapeCharacter = getCurrentEscapeCharacter(csvFile);
         Map<Integer, CsvColumnInfo<PsiElement>> columnInfoMap = new HashMap<>();
         CsvRecord[] records = PsiTreeUtil.getChildrenOfType(csvFile, CsvRecord.class);
         int row = 0;
         for (CsvRecord record : records) {
             int column = 0;
             for (CsvField field : record.getFieldList()) {
-                Integer length = CsvHelper.getMaxTextLineLength(unquoteCsvValue(field.getText()));
+                Integer length = CsvHelper.getMaxTextLineLength(unquoteCsvValue(field.getText(), escapeCharacter));
                 if (!columnInfoMap.containsKey(column)) {
                     columnInfoMap.put(column, new CsvColumnInfo(column, length, row));
                 } else if (columnInfoMap.get(column).getMaxLength() < length) {
@@ -170,28 +182,28 @@ public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile)
         return new CsvColumnInfoMap(columnInfoMap, PsiTreeUtil.hasErrorElements(csvFile));
     }
 
-    public static String unquoteCsvValue(String content) {
+    public static String unquoteCsvValue(String content, CsvEditorSettings.EscapeCharacter escapeCharacter) {
         if (content == null) {
             return "";
         }
         String result = content.trim();
         if (result.length() > 1 && result.startsWith("\"") && result.endsWith("\"")) {
             result = result.substring(1, result.length() - 1);
         }
-        result = result.replaceAll("(?:\")\"", "\"");
+        result = result.replaceAll("(?:" + Pattern.quote(escapeCharacter.getCharacter()) + ")\"", "\"");
         return result;
     }
 
     private static boolean isQuotingRequired(String content, String separator) {
         return content != null && (content.contains(separator) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
     }
 
-    public static String quoteCsvField(String content, String separator, boolean quotingEnforced) {
+    public static String quoteCsvField(String content, CsvEditorSettings.EscapeCharacter escapeCharacter, String separator, boolean quotingEnforced) {
         if (content == null) {
             return "";
         }
         if (quotingEnforced || isQuotingRequired(content, separator)) {
-            String result = content.replaceAll("\"", "\"\"");
+            String result = content.replaceAll("\"", escapeCharacter.getCharacter() + "\"");
             return "\"" + result + "\"";
         }
         return content;

src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf

@@ -2,10 +2,12 @@ package net.seesharpsoft.intellij.plugins.csv;
 
 import com.intellij.lexer.FlexLexer;
 import com.intellij.psi.tree.IElementType;
-import com.intellij.openapi.project.Project;
+import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
 import com.intellij.psi.TokenType;
 
+import java.util.regex.Pattern;
+
 %%
 
 %class CsvLexer
@@ -14,21 +16,25 @@ import com.intellij.psi.TokenType;
 %function advance
 %type IElementType
 %{
-  private String currentSeparator;
-
-  /**
-   * Provide constructor that supports a Project as parameter.
-   */
-  CsvLexer(java.io.Reader in, String separator) {
-    this(in);
-    this.currentSeparator = separator;
-  }
+    private String currentSeparator;
+    private CsvEditorSettings.EscapeCharacter myEscapeCharacter;
+
+    private static final Pattern ESCAPE_TEXT_PATTERN = Pattern.compile("[,;|\\t\\r\\n]");
+
+    /**
+     * Provide constructor that supports a Project as parameter.
+     */
+    CsvLexer(java.io.Reader in, String separator, CsvEditorSettings.EscapeCharacter escapeCharacter) {
+      this(in);
+      this.currentSeparator = separator;
+      myEscapeCharacter = escapeCharacter;
+    }
 %}
 %eof{  return;
 %eof}
 
-TEXT=[^ ,;|\t\r\n\"]+
-ESCAPED_TEXT=([,;|\t\r\n]|\"\")+
+TEXT=[^ ,;|\t\r\n\"\\]+
+ESCAPED_TEXT=[,;|\t\r\n\\]|\"\"|\\\"
 QUOTE=\"
 COMMA=[,;|\t]
 EOL=\n
@@ -65,7 +71,13 @@ WHITE_SPACE=[ \f]+
 
 <ESCAPED_TEXT> {ESCAPED_TEXT}
 {
-    return CsvTypes.ESCAPED_TEXT;
+    String text = yytext().toString();
+    if (myEscapeCharacter.isEscapedQuote(text)
+        || ESCAPE_TEXT_PATTERN.matcher(text).matches()
+     ) {
+        return CsvTypes.ESCAPED_TEXT;
+    }
+    return TokenType.BAD_CHARACTER;
 }
 
 <YYINITIAL, AFTER_TEXT, UNESCAPED_TEXT> {COMMA}

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java

@@ -1,9 +1,10 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
 import com.intellij.lexer.FlexAdapter;
+import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
 
 public class CsvLexerAdapter extends FlexAdapter {
-    public CsvLexerAdapter(String separator) {
-        super(new CsvLexer(null, separator));
+    public CsvLexerAdapter(String separator, CsvEditorSettings.EscapeCharacter escapeCharacter) {
+        super(new CsvLexer(null, separator, escapeCharacter));
     }
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex

@@ -11,6 +11,7 @@
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.TokenSet;
 import net.seesharpsoft.intellij.lang.FileParserDefinition;
+import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
 import net.seesharpsoft.intellij.plugins.csv.parser.CsvParser;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFileElementType;
@@ -76,11 +77,11 @@ public PsiElement createElement(ASTNode node) {
 
     @Override
     public Lexer createLexer(@NotNull PsiFile file) {
-        return new CsvLexerAdapter(CsvCodeStyleSettings.getCurrentSeparator(file));
+        return new CsvLexerAdapter(CsvCodeStyleSettings.getCurrentSeparator(file), CsvFileAttributes.getInstance(file.getProject()).getEscapeCharacter(file));
     }
 
     @Override
     public PsiParser createParser(@NotNull PsiFile file) {
         return createParser(file.getProject());
     }
-}
+}

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerAdapter.java

@@ -3,26 +3,44 @@
 import com.intellij.lang.PsiBuilder;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.impl.source.resolve.FileContextUtil;
+import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
 import net.seesharpsoft.intellij.plugins.csv.settings.CsvCodeStyleSettings;
 
+import java.util.regex.Pattern;
+
 public final class CsvParserUtil {
 
+    public static final Pattern ESCAPE_TEXT_PATTERN = Pattern.compile("[,;|\\t\\r\\n]");
+
     private CsvParserUtil() {
         // static utility class
     }
 
     public static boolean separator(PsiBuilder builder, int tokenType) {
         if (builder.getTokenType() == CsvTypes.COMMA) {
-            PsiFile currentFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
-            if (currentFile == null) {
+            PsiFile psiFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
+            if (psiFile == null) {
                 throw new UnsupportedOperationException("parser requires containing file");
             }
             return builder.getTokenText().equals(
-                    CsvCodeStyleSettings.getCurrentSeparator(currentFile)
+                    CsvCodeStyleSettings.getCurrentSeparator(psiFile)
             );
         }
         return false;
     }
 
+    public static boolean escapeCharacter(PsiBuilder builder, int tokenType) {
+        if (builder.getTokenType() == CsvTypes.ESCAPED_TEXT) {
+            PsiFile psiFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
+            if (psiFile == null) {
+                throw new UnsupportedOperationException("parser requires containing file");
+            }
+            String tokenText = builder.getTokenText();
+            return CsvFileAttributes.getInstance(psiFile.getProject()).getEscapeCharacter(psiFile).isEscapedQuote(tokenText) ||
+                    ESCAPE_TEXT_PATTERN.matcher(tokenText).matches();
+        }
+        return false;
+    }
+
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserDefinition.java

@@ -17,7 +17,7 @@ public static String getRelativeFileUrl(Project project, VirtualFile virtualFile
             return null;
         }
         String url = virtualFile.getUserData(RELATIVE_FILE_URL);
-        if (url == null) {
+        if (url == null && project.getBasePath() != null) {
             String projectDir = PathUtil.getLocalPath(project.getBasePath());
             url = PathUtil.getLocalPath(virtualFile.getPath())
                     .replaceFirst("^" + Pattern.quote(projectDir), "");