Merge pull request #176 from SeeSharpSoft/fb_value_separator_rework

Value separator rework

Author: SeeSharpSoft

gradle.properties

@@ -3,5 +3,5 @@
 # https://www.jetbrains.com/intellij-repository/snapshots
 
 name='CSV Plugin'
-javaVersion=1.8
-javaTargetVersion=1.8
+javaVersion=8
+javaTargetVersion=8

src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf

@@ -1,8 +1,6 @@
 {
   parserClass="net.seesharpsoft.intellij.plugins.csv.parser.CsvParser"
 
-  parserImports=["static net.seesharpsoft.intellij.plugins.csv.CsvParserUtil.*"]
-
   extends="com.intellij.extapi.psi.ASTWrapperPsiElement"
 
   psiClassPrefix="Csv"
@@ -25,10 +23,10 @@
 
 csvFile ::= record (CRLF record)* [CRLF]
 
-record ::= field ( << separator >> COMMA field)*
+record ::= field (COMMA field)*
 
 field ::= (escaped | nonEscaped)
 
-private escaped ::= QUOTE ( TEXT | << escapeCharacter >> ESCAPED_TEXT)* QUOTE
+private escaped ::= QUOTE (TEXT | ESCAPED_TEXT)* QUOTE
 
 private nonEscaped ::= TEXT*

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvEscapeCharacter.java

@@ -0,0 +1,30 @@
+package net.seesharpsoft.intellij.plugins.csv;
+
+import java.util.regex.Pattern;
+
+public enum CsvEscapeCharacter {
+    QUOTE("\"", "Double Quote (\")"),
+    BACKSLASH("\\", "Backslash (\\)");
+
+    private final String myCharacter;
+    private final String myDisplay;
+    private final Pattern myPattern;
+
+    CsvEscapeCharacter(String character, String display) {
+        myCharacter = character;
+        myDisplay = display;
+        myPattern = Pattern.compile(Pattern.quote(myCharacter + "\""));
+    }
+
+    public String getCharacter() {
+        return myCharacter;
+    }
+
+    public String getDisplay() {
+        return myDisplay;
+    }
+
+    public boolean isEscapedQuote(String text) {
+        return myPattern.matcher(text).matches();
+    }
+}

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java

@@ -18,7 +18,6 @@
 import com.intellij.psi.util.PsiTreeUtil;
 import net.seesharpsoft.intellij.lang.FileParserDefinition;
 import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
-import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvField;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvRecord;
@@ -152,16 +151,48 @@ public static int getFieldEndOffset(PsiElement field) {
         return separator == null ? field.getContainingFile().getTextLength() : separator.getTextOffset();
     }
 
-    public static CsvEditorSettings.EscapeCharacter getCurrentEscapeCharacter(CsvFile csvFile) {
-        return getCurrentEscapeCharacter(csvFile.getContainingFile());
+    public static VirtualFile getVirtualFile(PsiFile psiFile) {
+        return psiFile == null ? null : psiFile.getOriginalFile().getVirtualFile();
     }
 
-    public static CsvEditorSettings.EscapeCharacter getCurrentEscapeCharacter(PsiFile psiFile) {
-        return CsvFileAttributes.getInstance(psiFile.getProject()).getEscapeCharacter(psiFile);
+    public static Project getProject(PsiFile psiFile) {
+        return psiFile == null ? null : psiFile.getProject();
+    }
+
+    public static CsvValueSeparator getValueSeparator(CsvFile csvFile) {
+        return getValueSeparator(csvFile.getContainingFile());
+    }
+
+    public static CsvValueSeparator getValueSeparator(PsiFile psiFile) {
+        return getValueSeparator(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
+    public static CsvValueSeparator getValueSeparator(Project project, VirtualFile virtualFile) {
+        return CsvFileAttributes.getInstance(project).getValueSeparator(project, virtualFile);
+    }
+
+    public static boolean hasValueSeparatorAttribute(@NotNull PsiFile psiFile) {
+        return CsvFileAttributes.getInstance(getProject(psiFile)).hasValueSeparatorAttribute(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
+    public static CsvEscapeCharacter getEscapeCharacter(CsvFile csvFile) {
+        return getEscapeCharacter(csvFile.getContainingFile());
+    }
+
+    public static CsvEscapeCharacter getEscapeCharacter(PsiFile psiFile) {
+        return getEscapeCharacter(getProject(psiFile), getVirtualFile(psiFile));
+    }
+
+    public static CsvEscapeCharacter getEscapeCharacter(Project project, VirtualFile virtualFile) {
+        return CsvFileAttributes.getInstance(project).getEscapeCharacter(project, virtualFile);
+    }
+
+    public static boolean hasEscapeCharacterAttribute(@NotNull PsiFile psiFile) {
+        return CsvFileAttributes.getInstance(getProject(psiFile)).hasEscapeCharacterAttribute(getProject(psiFile), getVirtualFile(psiFile));
     }
 
     public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile) {
-        CsvEditorSettings.EscapeCharacter escapeCharacter = getCurrentEscapeCharacter(csvFile);
+        CsvEscapeCharacter escapeCharacter = getEscapeCharacter(csvFile);
         Map<Integer, CsvColumnInfo<PsiElement>> columnInfoMap = new HashMap<>();
         CsvRecord[] records = PsiTreeUtil.getChildrenOfType(csvFile, CsvRecord.class);
         int row = 0;
@@ -182,7 +213,7 @@ public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile)
         return new CsvColumnInfoMap(columnInfoMap, PsiTreeUtil.hasErrorElements(csvFile));
     }
 
-    public static String unquoteCsvValue(String content, CsvEditorSettings.EscapeCharacter escapeCharacter) {
+    public static String unquoteCsvValue(String content, CsvEscapeCharacter escapeCharacter) {
         if (content == null) {
             return "";
         }
@@ -194,15 +225,19 @@ public static String unquoteCsvValue(String content, CsvEditorSettings.EscapeCha
         return result;
     }
 
-    private static boolean isQuotingRequired(String content, String separator) {
-        return content != null && (content.contains(separator) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
+    private static boolean isQuotingRequired(String content, CsvValueSeparator valueSeparator) {
+        return content != null &&
+                (content.contains(valueSeparator.getCharacter()) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
     }
 
-    public static String quoteCsvField(String content, CsvEditorSettings.EscapeCharacter escapeCharacter, String separator, boolean quotingEnforced) {
+    public static String quoteCsvField(String content,
+                                       CsvEscapeCharacter escapeCharacter,
+                                       CsvValueSeparator valueSeparator,
+                                       boolean quotingEnforced) {
         if (content == null) {
             return "";
         }
-        if (quotingEnforced || isQuotingRequired(content, separator)) {
+        if (quotingEnforced || isQuotingRequired(content, valueSeparator)) {
             String result = content.replaceAll("\"", escapeCharacter.getCharacter() + "\"");
             return "\"" + result + "\"";
         }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex

@@ -1,10 +1,9 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
-import com.intellij.lexer.FlexLexer;
 import com.intellij.psi.tree.IElementType;
-import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
 import com.intellij.psi.TokenType;
+import com.intellij.lexer.FlexLexer;
 
 import java.util.regex.Pattern;
 
@@ -16,17 +15,17 @@ import java.util.regex.Pattern;
 %function advance
 %type IElementType
 %{
-    private String currentSeparator;
-    private CsvEditorSettings.EscapeCharacter myEscapeCharacter;
+    private CsvValueSeparator myValueSeparator;
+    private CsvEscapeCharacter myEscapeCharacter;
 
     private static final Pattern ESCAPE_TEXT_PATTERN = Pattern.compile("[,;|\\t\\r\\n]");
 
     /**
      * Provide constructor that supports a Project as parameter.
      */
-    CsvLexer(java.io.Reader in, String separator, CsvEditorSettings.EscapeCharacter escapeCharacter) {
+    CsvLexer(java.io.Reader in, CsvValueSeparator valueSeparator, CsvEscapeCharacter escapeCharacter) {
       this(in);
-      this.currentSeparator = separator;
+      myValueSeparator = valueSeparator;
       myEscapeCharacter = escapeCharacter;
     }
 %}
@@ -82,7 +81,7 @@ WHITE_SPACE=[ \f]+
 
 <YYINITIAL, AFTER_TEXT, UNESCAPED_TEXT> {COMMA}
 {
-    if (currentSeparator.equals(yytext().toString())) {
+    if (myValueSeparator.isValueSeparator(yytext().toString())) {
         yybegin(YYINITIAL);
         return CsvTypes.COMMA;
     }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerAdapter.java

@@ -1,10 +1,9 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
 import com.intellij.lexer.FlexAdapter;
-import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
 
 public class CsvLexerAdapter extends FlexAdapter {
-    public CsvLexerAdapter(String separator, CsvEditorSettings.EscapeCharacter escapeCharacter) {
+    public CsvLexerAdapter(CsvValueSeparator separator, CsvEscapeCharacter escapeCharacter) {
         super(new CsvLexer(null, separator, escapeCharacter));
     }
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserDefinition.java

@@ -11,12 +11,10 @@
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.TokenSet;
 import net.seesharpsoft.intellij.lang.FileParserDefinition;
-import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
 import net.seesharpsoft.intellij.plugins.csv.parser.CsvParser;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFileElementType;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
-import net.seesharpsoft.intellij.plugins.csv.settings.CsvCodeStyleSettings;
 import org.jetbrains.annotations.NotNull;
 
 public class CsvParserDefinition implements FileParserDefinition {
@@ -77,7 +75,7 @@ public PsiElement createElement(ASTNode node) {
 
     @Override
     public Lexer createLexer(@NotNull PsiFile file) {
-        return new CsvLexerAdapter(CsvCodeStyleSettings.getCurrentSeparator(file), CsvFileAttributes.getInstance(file.getProject()).getEscapeCharacter(file));
+        return new CsvLexerAdapter(CsvHelper.getValueSeparator(file), CsvHelper.getEscapeCharacter(file));
     }
 
     @Override

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserUtil.java

@@ -1,5 +1,5 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
 public interface CsvSeparatorHolder {
-    String getSeparator();
+    CsvValueSeparator getSeparator();
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvSeparatorHolder.java

@@ -0,0 +1,32 @@
+package net.seesharpsoft.intellij.plugins.csv;
+
+import java.util.regex.Pattern;
+
+public enum CsvValueSeparator {
+    COMMA(",", "Comma (,)"),
+    SEMICOLON(";", "Semicolon (;)"),
+    PIPE("|", "Pipe (|)"),
+    TAB("\t", "Tab (↹)");
+
+    private final String myCharacter;
+    private final String myDisplay;
+    private final Pattern myPattern;
+
+    CsvValueSeparator(String character, String display) {
+        myCharacter = character;
+        myDisplay = display;
+        myPattern = Pattern.compile(Pattern.quote(myCharacter));
+    }
+
+    public String getCharacter() {
+        return myCharacter;
+    }
+
+    public String getDisplay() {
+        return myDisplay;
+    }
+
+    public boolean isValueSeparator(String text) {
+        return myPattern.matcher(text).matches();
+    }
+}