Merge pull request #203 from SeeSharpSoft/master

Release 2.11.0

Author: SeeSharpSoft

.travis.yml

@@ -1,6 +1,6 @@
 language: java
 jdk:
-  - openjdk8
+  - openjdk9
 
 env:
   - IDEA_VERSION=PC-2019.3.3 GRAMMAR_KIT_VERSION=2019.3

CHANGELOG

@@ -1,3 +1,8 @@
+2.11.0
+May 8, 2020
+
+NEW: Adding custom separator #177
+
 2.10.0
 Apr 19, 2020
 

README.md

@@ -26,7 +26,8 @@ This enables default editor features like syntax validation, highlighting and in
 - quick fix inspections
 - intentions (Alt+Enter), e.g. Quote/Unquote (all), Shift Column Left/Right
 - structure view (header-entry layout)
-- support for ',', ';', '|' and '↹' as value separator
+- support for ',', ';', ':', '|' and '↹' as pre-defined value separator
+- support for freely defined value separators
 - highlight of active column values
 - tab (↹) separator highlighting
 
@@ -59,13 +60,16 @@ Please note that if a document is syntactically incorrect, other features like c
 CSV files provide a high degree of flexibility and can be used universally for all kind of data.
 This led to a variety of CSV derivatives like semicolon or pipe separated values, which share the common format but make use of a different separator.
 
-The plugin supports project specific separator setting.
-New separators can be added fairly easy in the parser definition of the source code.
+The plugin supports file-specific separator settings.
+Predefined separators, like Comma (,), semicolon (;), colon (:), tab (↹) and pipe (|), come with a precompiled lexer providing the best performance for working with CSV. 
+
+Since version 2.11.0, this plugin additionally comes with a runtime lexer implementation to support freely chosen separator. 
 
 #### TSV/PSV
 
-Comma (,), semicolon (;), tab (↹) and pipe (|) can be explicitly set as a separator for CSV files.
-Additionally the file types TSV (Tab-Separated-Values) and PSV (Pipe-Separated-Values) were introduced as a kind of CSV language.
+A different separator can be chosen for each CSV file. 
+
+Additionally, the file types TSV (Tab-Separated-Values) and PSV (Pipe-Separated-Values) were introduced as a kind of CSV language.
 For TSV and PSV files the same formatter and code style settings are applied as for CSV itself, but the separator is considered to be a tab or a pipe respectively.
 All functionality that is available for plain CSV files (inspections, intentions, structure view, etc.) can be used for TSV and PSV as well.
 
@@ -126,7 +130,7 @@ Enable zero-based column numbering. This affects the tooltip info of the text ed
 
 ##### Default Value Separator (CSV only)
 
-The following separators are currently supported: **,** (Comma), **;** (Semicolon), **|** (Pipe) and **↹** (Tab)
+The following separators are currently supported: **,** (Comma), **;** (Semicolon), **:** (Colon), **|** (Pipe) and **↹** (Tab)
 
 _Default Value Separator_ defines which separator is used as standard for each newly opened CSV file. The separator character can be changed for each file individually in its editors context menu.
 
@@ -326,9 +330,10 @@ Annasusanna,Amsterdam,         1
 ![Context menu](./docs/contextmenu.png)
 
 The action to switch the value separator (or escape character) - *which is used for CSV syntax validation of a specific file* - is part of its editors context menu.
+Since version 2.11.0, also custom separator setting is available via this action.
 
 
-This action defines how the parser/validator/highlighter/etc. behaves. It does intentionally not change the file content.
+**Please note:** This action defines how the parser/validator/highlighter/etc. behaves. It does intentionally not change the file content.
 To be more precise: It **does not replace** previous separator/escape characters by new ones or adjust the escaped texts.
 
 #### Adjust column widths (table editor only)
@@ -395,15 +400,6 @@ Besides source code contributions, feel free to open bug reports or just suggest
 
 - zoom table-editor cells with Ctrl+Mouse Wheel @royqh1979
 
-## FAQ
-
-> Why can't I choose the separator freely?
-
-Having clearly defined symbols enables the syntax parser and language lexer to do its job properly.
-The code for those is generated during build time by using the [Grammar-Kit](https://github.com/JetBrains/Grammar-Kit).
-Adding a new kind of separator during development is fairly easy (please feel free to request a new commonly used one) in comparison to the implementation effort and usefulness of a freely defined separator. 
-
-
 ## Jetbrains Repository
 
 JetBrains Plugin Repository Link: https://plugins.jetbrains.com/plugin/10037-csv-plugin

build.gradle

@@ -24,7 +24,7 @@ jacocoTestReport {
 }
 
 group 'net.seesharpsoft.intellij.plugins'
-version '2.10.0'
+version '2.11.0'
 
 apply plugin: 'java'
 sourceCompatibility = javaVersion
@@ -34,6 +34,7 @@ repositories {
     mavenCentral()
 }
 dependencies {
+    implementation 'net.seesharpsoft.sharping:sharping-commons:0.21.0'
     compileOnly 'org.apache.ant:ant:1.7.0'
     testCompile 'org.mockito:mockito-core:2.28.2'
 }

gradle.properties

@@ -3,5 +3,5 @@
 # https://www.jetbrains.com/intellij-repository/snapshots
 
 name='CSV Plugin'
-javaVersion=8
-javaTargetVersion=8
+javaVersion=9
+javaTargetVersion=9

src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf

@@ -13,11 +13,11 @@
   tokenTypeClass="net.seesharpsoft.intellij.plugins.csv.psi.CsvTokenType"
 
   tokens=[
-    TEXT='regexp:[^ ,;|\t\r\n"\\]+'
-    ESCAPED_TEXT='regexp:[,;|\t\r\n]|""|\\"'
+    TEXT='regexp:[^ ,:;|\t\r\n"\\]+'
+    ESCAPED_TEXT='regexp:[,:;|\t\r\n]|""|\\"'
     ESCAPE_CHARACTER='regexp:\\'
-    COMMA='regexp:[,;|\t]'
-    QUOTE='"'
+    COMMA='regexp:[,:;|\t]'
+    QUOTE='regexp:"'
     CRLF='regexp:\n'
    ]
 }

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java

@@ -1,6 +1,7 @@
 package net.seesharpsoft.intellij.plugins.csv;
 
 import com.intellij.ide.scratch.ScratchFileType;
+import com.intellij.ide.scratch.ScratchUtil;
 import com.intellij.lang.*;
 import com.intellij.lexer.Lexer;
 import com.intellij.openapi.fileTypes.FileType;
@@ -22,6 +23,7 @@
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvRecord;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
+import net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.HashMap;
@@ -50,7 +52,7 @@ public static PsiElement createEmptyCsvField(PsiFile psiFile) {
     public static boolean isCsvFile(Project project, VirtualFile file) {
         final FileType fileType = file.getFileType();
         return (fileType instanceof LanguageFileType && ((LanguageFileType) fileType).getLanguage().isKindOf(CsvLanguage.INSTANCE)) ||
-                (fileType == ScratchFileType.INSTANCE && LanguageUtil.getLanguageForPsi(project, file).isKindOf(CsvLanguage.INSTANCE));
+                (ScratchUtil.isScratch(file) && LanguageUtil.getLanguageForPsi(project, file).isKindOf(CsvLanguage.INSTANCE));
     }
 
     public static IElementType getElementType(PsiElement element) {
@@ -158,32 +160,36 @@ public static Project getProject(PsiFile psiFile) {
         return psiFile == null ? null : psiFile.getProject();
     }
 
-    public static CsvValueSeparator getValueSeparator(CsvFile csvFile) {
+    public static @NotNull CsvValueSeparator getValueSeparator(CsvFile csvFile) {
         return getValueSeparator(csvFile.getContainingFile());
     }
 
-    public static CsvValueSeparator getValueSeparator(PsiFile psiFile) {
+    public static @NotNull CsvValueSeparator getValueSeparator(PsiFile psiFile) {
         return getValueSeparator(getProject(psiFile), getVirtualFile(psiFile));
     }
 
-    public static CsvValueSeparator getValueSeparator(Project project, VirtualFile virtualFile) {
-        return CsvFileAttributes.getInstance(project).getValueSeparator(project, virtualFile);
+    public static @NotNull CsvValueSeparator getValueSeparator(Project project, VirtualFile virtualFile) {
+        return project == null ?
+                CsvEditorSettings.getInstance().getDefaultValueSeparator() :
+                CsvFileAttributes.getInstance(project).getValueSeparator(project, virtualFile);
     }
 
     public static boolean hasValueSeparatorAttribute(@NotNull PsiFile psiFile) {
         return CsvFileAttributes.getInstance(getProject(psiFile)).hasValueSeparatorAttribute(getProject(psiFile), getVirtualFile(psiFile));
     }
 
-    public static CsvEscapeCharacter getEscapeCharacter(CsvFile csvFile) {
+    public static @NotNull CsvEscapeCharacter getEscapeCharacter(CsvFile csvFile) {
         return getEscapeCharacter(csvFile.getContainingFile());
     }
 
-    public static CsvEscapeCharacter getEscapeCharacter(PsiFile psiFile) {
+    public static @NotNull CsvEscapeCharacter getEscapeCharacter(PsiFile psiFile) {
         return getEscapeCharacter(getProject(psiFile), getVirtualFile(psiFile));
     }
 
-    public static CsvEscapeCharacter getEscapeCharacter(Project project, VirtualFile virtualFile) {
-        return CsvFileAttributes.getInstance(project).getEscapeCharacter(project, virtualFile);
+    public static @NotNull CsvEscapeCharacter getEscapeCharacter(Project project, VirtualFile virtualFile) {
+        return project == null ?
+                CsvEditorSettings.getInstance().getDefaultEscapeCharacter() :
+                CsvFileAttributes.getInstance(project).getEscapeCharacter(project, virtualFile);
     }
 
     public static boolean hasEscapeCharacterAttribute(@NotNull PsiFile psiFile) {

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex

@@ -32,11 +32,11 @@ import java.util.regex.Pattern;
 %eof{  return;
 %eof}
 
-TEXT=[^ ,;|\t\r\n\"\\]+
-ESCAPED_TEXT=[,;|\t\r\n]|\"\"|\\\"
+TEXT=[^ ,:;|\t\r\n\"\\]+
+ESCAPED_TEXT=[,:;|\t\r\n]|\"\"|\\\"
 ESCAPE_CHAR=\\
 QUOTE=\"
-COMMA=[,;|\t]
+COMMA=[,:;|\t]
 EOL=\n
 WHITE_SPACE=[ \f]+
 

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerFactory.java

@@ -0,0 +1,34 @@
+package net.seesharpsoft.intellij.plugins.csv;
+
+import com.intellij.lexer.Lexer;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.PsiFile;
+import org.jetbrains.annotations.NotNull;
+
+public class CsvLexerFactory {
+    protected static CsvLexerFactory INSTANCE = new CsvLexerFactory();
+
+    public static CsvLexerFactory getInstance() {
+        return INSTANCE;
+    }
+
+    protected Lexer createLexer(@NotNull CsvValueSeparator separator, @NotNull CsvEscapeCharacter escapeCharacter) {
+        if (separator.isCustom()) {
+            return new CsvSharpLexer(new CsvSharpLexer.Configuration(
+                    separator.getCharacter(),
+                    "\n",
+                    escapeCharacter.getCharacter(),
+                    "\""));
+        }
+        return new CsvLexerAdapter(separator, escapeCharacter);
+    }
+
+    public Lexer createLexer(Project project, VirtualFile file) {
+        return createLexer(CsvHelper.getValueSeparator(project, file), CsvHelper.getEscapeCharacter(project, file));
+    }
+
+    public Lexer createLexer(@NotNull PsiFile file) {
+        return createLexer(CsvHelper.getValueSeparator(file), CsvHelper.getEscapeCharacter(file));
+    }
+}

src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserDefinition.java

@@ -75,7 +75,7 @@ public PsiElement createElement(ASTNode node) {
 
     @Override
     public Lexer createLexer(@NotNull PsiFile file) {
-        return new CsvLexerAdapter(CsvHelper.getValueSeparator(file), CsvHelper.getEscapeCharacter(file));
+        return CsvLexerFactory.getInstance().createLexer(file);
     }
 
     @Override