Merge pull request #188 from SeeSharpSoft/master

SeeSharpSoft · web-flow · commit 78bcc7991e76 · 2020-02-23T14:35:03.000+01:00
Release 2.9.2
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,9 @@
+2.9.2
+Feb 24, 2020
+
+FIX: Backslash in text is considered a special character #184
+FIX: NullPointerException thrown when trying to view CSV as table #185
+
 2.9.1
 Feb 15, 2020
 
diff --git a/build.gradle b/build.gradle
@@ -24,7 +24,7 @@ jacocoTestReport {
 }
 
 group 'net.seesharpsoft.intellij.plugins'
-version '2.9.1'
+version '2.9.2'
 
 apply plugin: 'java'
 sourceCompatibility = javaVersion
diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf b/src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf
@@ -14,7 +14,8 @@
 
   tokens=[
     TEXT='regexp:[^ ,;|\t\r\n"\\]+'
-    ESCAPED_TEXT='regexp:[,;|\t\r\n\\]|""|\\"'
+    ESCAPED_TEXT='regexp:[,;|\t\r\n]|""|\\"'
+    ESCAPE_CHARACTER='regexp:\\'
     COMMA='regexp:[,;|\t]'
     QUOTE='"'
     CRLF='regexp:\n'
@@ -27,6 +28,6 @@ record ::= field (COMMA field)*
 
 field ::= (escaped | nonEscaped)
 
-private escaped ::= QUOTE (TEXT | ESCAPED_TEXT)* QUOTE
+private escaped ::= QUOTE (TEXT | ESCAPE_CHARACTER | ESCAPED_TEXT)* QUOTE
 
-private nonEscaped ::= TEXT*
+private nonEscaped ::= (TEXT | ESCAPE_CHARACTER)*
diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvEscapeCharacter.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvEscapeCharacter.java
@@ -3,16 +3,18 @@
 import java.util.regex.Pattern;
 
 public enum CsvEscapeCharacter {
-    QUOTE("\"", "Double Quote (\")"),
-    BACKSLASH("\\", "Backslash (\\)");
+    QUOTE("\"", "Double Quote (\")", "\""),
+    BACKSLASH("\\", "Backslash (\\)", "\\\\");
 
     private final String myCharacter;
     private final String myDisplay;
     private final Pattern myPattern;
+    private final String myRegexPattern;
 
-    CsvEscapeCharacter(String character, String display) {
+    CsvEscapeCharacter(String character, String display, String regexPattern) {
         myCharacter = character;
         myDisplay = display;
+        myRegexPattern = regexPattern;
         myPattern = Pattern.compile(Pattern.quote(myCharacter + "\""));
     }
 
@@ -24,6 +26,10 @@ public String getDisplay() {
         return myDisplay;
     }
 
+    public String getRegexPattern() {
+        return myRegexPattern;
+    }
+
     public boolean isEscapedQuote(String text) {
         return myPattern.matcher(text).matches();
     }
diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java
@@ -27,7 +27,6 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
-import java.util.regex.Pattern;
 
 public final class CsvHelper {
 
@@ -221,13 +220,18 @@ public static String unquoteCsvValue(String content, CsvEscapeCharacter escapeCh
         if (result.length() > 1 && result.startsWith("\"") && result.endsWith("\"")) {
             result = result.substring(1, result.length() - 1);
         }
-        result = result.replaceAll("(?:" + Pattern.quote(escapeCharacter.getCharacter()) + ")\"", "\"");
+        if (escapeCharacter != CsvEscapeCharacter.QUOTE) {
+            result = result.replaceAll("(?:" + escapeCharacter.getRegexPattern() + ")" +
+                    escapeCharacter.getRegexPattern(), escapeCharacter.getRegexPattern());
+        }
+        result = result.replaceAll("(?:" + escapeCharacter.getRegexPattern() + ")\"", "\"");
         return result;
     }
 
-    private static boolean isQuotingRequired(String content, CsvValueSeparator valueSeparator) {
+    private static boolean isQuotingRequired(String content, CsvEscapeCharacter escapeCharacter, CsvValueSeparator valueSeparator) {
         return content != null &&
-                (content.contains(valueSeparator.getCharacter()) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
+                (content.contains(valueSeparator.getCharacter()) || content.contains("\"") || content.contains("\n") || content.contains(escapeCharacter.getCharacter()) ||
+                        content.startsWith(" ") || content.endsWith(" "));
     }
 
     public static String quoteCsvField(String content,
@@ -237,8 +241,13 @@ public static String quoteCsvField(String content,
         if (content == null) {
             return "";
         }
-        if (quotingEnforced || isQuotingRequired(content, valueSeparator)) {
-            String result = content.replaceAll("\"", escapeCharacter.getCharacter() + "\"");
+        if (quotingEnforced || isQuotingRequired(content, escapeCharacter, valueSeparator)) {
+            String result = content;
+            if (escapeCharacter != CsvEscapeCharacter.QUOTE) {
+                result = result.replaceAll(escapeCharacter.getRegexPattern(),
+                        escapeCharacter.getRegexPattern() + escapeCharacter.getRegexPattern());
+            }
+            result = result.replaceAll("\"", escapeCharacter.getRegexPattern() + "\"");
             return "\"" + result + "\"";
         }
         return content;
diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex b/src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex
@@ -3,7 +3,7 @@ package net.seesharpsoft.intellij.plugins.csv;
 import com.intellij.psi.tree.IElementType;
 import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
 import com.intellij.psi.TokenType;
-import com.intellij.lexer.FlexLexer;
+import com.intellij.lexer.FlexLexer;import org.intellij.grammar.livePreview.LivePreviewElementType;
 
 import java.util.regex.Pattern;
 
@@ -33,7 +33,8 @@ import java.util.regex.Pattern;
 %eof}
 
 TEXT=[^ ,;|\t\r\n\"\\]+
-ESCAPED_TEXT=[,;|\t\r\n\\]|\"\"|\\\"
+ESCAPED_TEXT=[,;|\t\r\n]|\"\"|\\\"
+ESCAPE_CHAR=\\
 QUOTE=\"
 COMMA=[,;|\t]
 EOL=\n
@@ -42,6 +43,7 @@ WHITE_SPACE=[ \f]+
 %state AFTER_TEXT
 %state ESCAPED_TEXT
 %state UNESCAPED_TEXT
+%state ESCAPING
 
 %%
 
@@ -68,6 +70,33 @@ WHITE_SPACE=[ \f]+
     return CsvTypes.TEXT;
 }
 
+<YYINITIAL, UNESCAPED_TEXT> {ESCAPE_CHAR}
+{
+    String text = yytext().toString();
+    if (myEscapeCharacter.getCharacter().equals(text)) {
+        return TokenType.BAD_CHARACTER;
+    }
+    yybegin(UNESCAPED_TEXT);
+    return CsvTypes.TEXT;
+}
+
+<ESCAPED_TEXT, ESCAPING> {ESCAPE_CHAR} {
+    String text = yytext().toString();
+    if (myEscapeCharacter.getCharacter().equals(text)) {
+        switch (yystate()) {
+            case ESCAPED_TEXT:
+                yybegin(ESCAPING);
+                break;
+            case ESCAPING:
+                yybegin(ESCAPED_TEXT);
+                break;
+            default:
+                throw new RuntimeException("unhandled state: " + yystate());
+        }
+    }
+    return CsvTypes.TEXT;
+}
+
 <ESCAPED_TEXT> {ESCAPED_TEXT}
 {
     String text = yytext().toString();
@@ -76,6 +105,11 @@ WHITE_SPACE=[ \f]+
      ) {
         return CsvTypes.ESCAPED_TEXT;
     }
+    if (!text.startsWith(CsvEscapeCharacter.QUOTE.getCharacter())) {
+        yypushback(1);
+        return CsvTypes.TEXT;
+    }
+
     return TokenType.BAD_CHARACTER;
 }
 
diff --git a/src/main/java/net/seesharpsoft/intellij/plugins/csv/editor/table/CsvTableEditor.java b/src/main/java/net/seesharpsoft/intellij/plugins/csv/editor/table/CsvTableEditor.java
@@ -343,12 +343,19 @@ public final void adjustAllColumnWidths() {
     }
 
     protected final void setAllColumnWidths(int[] widths) {
+        if (widths == null) {
+            return;
+        }
         getFileEditorState().setColumnWidths(widths);
         updateEditorLayout();
     }
 
     protected int[] calculateDistributedColumnWidths() {
-        Map<Integer, CsvColumnInfo<PsiElement>> columnInfos = this.getColumnInfoMap().getColumnInfos();
+        CsvColumnInfoMap csvColumnInfoMap = this.getColumnInfoMap();
+        if (csvColumnInfoMap == null) {
+            return null;
+        }
+        Map<Integer, CsvColumnInfo<PsiElement>> columnInfos = csvColumnInfoMap.getColumnInfos();
         Object[][] data = getDataHandler().getCurrentState();
         int[] widths = new int[columnInfos.size()];
         int tableAutoMaxColumnWidth = CsvEditorSettings.getInstance().getTableAutoMaxColumnWidth();
diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml
@@ -48,12 +48,8 @@
 
     <change-notes><![CDATA[
         <pre style="font-family: sans-serif">
-NEW: customizable escape character #159
-NEW: value separator setting moved from 'Code Style' to 'General'
-+ lots of code cleanup & rework
-
-FIX: update failed for AnAction #181
-FIX: selection indicator for default separator action
+FIX: Backslash in text is considered a special character #184
+FIX: NullPointerException thrown when trying to view CSV as table #185
       </pre>
     ]]>
     </change-notes>
diff --git a/src/test/java/net/seesharpsoft/intellij/plugins/csv/CsvHelperTest.java b/src/test/java/net/seesharpsoft/intellij/plugins/csv/CsvHelperTest.java
@@ -0,0 +1,26 @@
+package net.seesharpsoft.intellij.plugins.csv;
+
+import com.intellij.testFramework.PlatformLiteFixture;
+
+public class CsvHelperTest extends PlatformLiteFixture {
+
+    public void testUnquoteCsvValue() {
+        String csv = "\"Header\"\" \\\\1\\\\\"";
+        assertEquals("Header\" \\\\1\\\\", CsvHelper.unquoteCsvValue(csv, CsvEscapeCharacter.QUOTE));
+    }
+
+    public void testUnquoteCsvValueWithBackslash() {
+        String csv = "\"Header\\\" \\\\1\\\\\"";
+        assertEquals("Header\" \\1\\", CsvHelper.unquoteCsvValue(csv, CsvEscapeCharacter.BACKSLASH));
+    }
+
+    public void testQuoteCsvValue() {
+        String csv = "Header\" \\1\\";
+        assertEquals( "\"Header\"\" \\1\\\"", CsvHelper.quoteCsvField(csv, CsvEscapeCharacter.QUOTE, CsvValueSeparator.COMMA, false));
+    }
+
+    public void testQuoteCsvValueWithBackslash() {
+        String csv = "Header\" \\1\\";
+        assertEquals("\"Header\\\" \\\\1\\\\\"",  CsvHelper.quoteCsvField(csv, CsvEscapeCharacter.BACKSLASH, CsvValueSeparator.COMMA, false));
+    }
+}
diff --git a/src/test/java/net/seesharpsoft/intellij/plugins/csv/intention/CsvIntentionTest.java b/src/test/java/net/seesharpsoft/intellij/plugins/csv/intention/CsvIntentionTest.java
@@ -12,13 +12,37 @@ protected String getTestDataPath() {
         return "./src/test/resources/intention";
     }
 
+    @Override
+    protected void tearDown() throws Exception {
+        CsvEditorSettings.getInstance().setDefaultEscapeCharacter(CsvEditorSettings.ESCAPE_CHARACTER_DEFAULT);
+        super.tearDown();
+    }
+
     protected void doTestIntention(String testName, String hint) throws Throwable {
+        doTestIntention(testName, hint, false);
+    }
+
+    protected void doTestIntention(String testName, String hint, boolean expectError) throws Throwable {
         myFixture.configureByFile(testName + "/before.csv");
         final IntentionAction action = myFixture.filterAvailableIntentions(hint).stream()
                 .filter(intentionAction -> intentionAction.getText().equals(hint))
-                .findFirst().get();
-        myFixture.launchAction(action);
-        myFixture.checkResultByFile(testName + "/after.csv");
+                .findFirst().orElse(null);
+        if (action == null) {
+            assertTrue("action not found -> this was expected: " + expectError, expectError);
+        } else {
+            assertFalse("action was found -> this was expected: " + !expectError, expectError);
+            myFixture.launchAction(action);
+            myFixture.checkResultByFile(testName + "/after.csv");
+        }
+    }
+
+    public void testErroneousCsv() throws Throwable {
+        doTestIntention("Erroneous", "Quote All", true);
+    }
+
+    public void testErroneousBackslashCsv() throws Throwable {
+        CsvEditorSettings.getInstance().setDefaultEscapeCharacter(CsvEscapeCharacter.BACKSLASH);
+        doTestIntention("ErroneousBackslash", "Quote All", true);
     }
 
     public void testQuoteAllIntention() throws Throwable {
@@ -28,7 +52,6 @@ public void testQuoteAllIntention() throws Throwable {
     public void testQuoteAllBackslashIntention() throws Throwable {
         CsvEditorSettings.getInstance().setDefaultEscapeCharacter(CsvEscapeCharacter.BACKSLASH);
         doTestIntention("QuoteAllBackslash", "Quote All");
-        CsvEditorSettings.getInstance().setDefaultEscapeCharacter(CsvEditorSettings.ESCAPE_CHARACTER_DEFAULT);
     }
 
     public void testUnquoteAllIntention() throws Throwable {
@@ -38,7 +61,6 @@ public void testUnquoteAllIntention() throws Throwable {
     public void testUnquoteAllBackslashIntention() throws Throwable {
         CsvEditorSettings.getInstance().setDefaultEscapeCharacter(CsvEscapeCharacter.BACKSLASH);
         doTestIntention("UnquoteAllBackslash", "Unquote All");
-        CsvEditorSettings.getInstance().setDefaultEscapeCharacter(CsvEditorSettings.ESCAPE_CHARACTER_DEFAULT);
     }
 
     public void testQuoteIntention() throws Throwable {
diff --git a/src/test/resources/intention/Erroneous/before.csv b/src/test/resources/intention/Erroneous/before.csv
@@ -0,0 +1 @@
+"Value ""1"", "Value 2"
diff --git a/src/test/resources/intention/ErroneousBackslash/before.csv b/src/test/resources/intention/ErroneousBackslash/before.csv
@@ -0,0 +1 @@
+Header \\1\
diff --git a/src/test/resources/intention/QuoteAll/after.csv b/src/test/resources/intention/QuoteAll/after.csv
@@ -1,3 +1,3 @@
-"Header 1", "Header, 2"
+"\Header \\1\", "Header, 2"
 "Value ""1"""," Value 2"
-""
+"back\\""\""\slash\"
diff --git a/src/test/resources/intention/QuoteAll/before.csv b/src/test/resources/intention/QuoteAll/before.csv
@@ -1,2 +1,3 @@
-Header 1, "Header, 2"
+\Header \\1\, "Header, 2"
 "Value ""1""", Value 2
+"back\\""\""\slash\"
diff --git a/src/test/resources/intention/QuoteAllBackslash/after.csv b/src/test/resources/intention/QuoteAllBackslash/after.csv
@@ -1,3 +1,3 @@
-"Header 1", "Header, 2"
+"Header \\1\\", "Header, 2"
 "Value \"1\""," Value 2"
-""
+"back\\\"\"\\slash\\"
diff --git a/src/test/resources/intention/QuoteAllBackslash/before.csv b/src/test/resources/intention/QuoteAllBackslash/before.csv
@@ -1,2 +1,3 @@
-Header 1, "Header, 2"
+"Header \\1\\", "Header, 2"
 "Value \"1\"", Value 2
+"back\\\"\"\\slash\\"
diff --git a/src/test/resources/intention/UnquoteAll/after.csv b/src/test/resources/intention/UnquoteAll/after.csv
@@ -1,2 +1,3 @@
-Header 1, "Header, 2"
+Header \1\, "Header, 2"
 "Value ""1""", Value 2
+"back\\""\""\slash\"
diff --git a/src/test/resources/intention/UnquoteAll/before.csv b/src/test/resources/intention/UnquoteAll/before.csv
@@ -1,2 +1,3 @@
-"Header 1", "Header, 2"
+"Header \1\", "Header, 2"
 "Value ""1""", "Value 2"
+"back\\""\""\slash\"
diff --git a/src/test/resources/intention/UnquoteAllBackslash/after.csv b/src/test/resources/intention/UnquoteAllBackslash/after.csv
@@ -1,2 +1,3 @@
-Header 1, "Header, 2"
+Header \\1\\, "Header, 2"
 "Value \"1\"", Value 2
+"back\\\"\"\\slash\\"
diff --git a/src/test/resources/intention/UnquoteAllBackslash/before.csv b/src/test/resources/intention/UnquoteAllBackslash/before.csv
@@ -1,2 +1,3 @@
-"Header 1", "Header, 2"
+"Header \\1\\", "Header, 2"
 "Value \"1\"", "Value 2"
+"back\\\"\"\\slash\\"

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ jacocoTestReport {`
`24`	`24`	`}`
`25`	`25`
`26`	`26`	`group 'net.seesharpsoft.intellij.plugins'`
`27`		`-version '2.9.1'`
	`27`	`+version '2.9.2'`
`28`	`28`
`29`	`29`	`apply plugin: 'java'`
`30`	`30`	`sourceCompatibility = javaVersion`