Skip to content

Commit 4b98eb8

Browse files
SeeSharpSoftSeeSharpSoft
committed
[FEATURE] enable backslash (\) as alternative escape character
Fixes #159
1 parent 9bc0096 commit 4b98eb8

36 files changed

+628
-248
lines changed

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/Csv.bnf‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
tokenTypeClass="net.seesharpsoft.intellij.plugins.csv.psi.CsvTokenType"
1616

1717
tokens=[
18-
TEXT='regexp:[^ ,;|\t\r\n"]+'
19-
ESCAPED_TEXT='regexp:([,;|\t\r\n]|"")+'
18+
TEXT='regexp:[^ ,;|\t\r\n"\\]+'
19+
ESCAPED_TEXT='regexp:([,;|\t\r\n\\]|""|\\")+'
2020
COMMA='regexp:[,;|\t]'
2121
QUOTE='"'
2222
CRLF='regexp:\n'
@@ -29,6 +29,6 @@ record ::= field ( << separator >> COMMA field)*
2929

3030
field ::= (escaped | nonEscaped)
3131

32-
private escaped ::= QUOTE ( TEXT | ESCAPED_TEXT)* QUOTE
32+
private escaped ::= QUOTE ( TEXT | << escapeCharacter >> ESCAPED_TEXT)* QUOTE
3333

34-
private nonEscaped ::= TEXT*
34+
private nonEscaped ::= TEXT*

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvHelper.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import com.intellij.psi.tree.IElementType;
1818
import com.intellij.psi.util.PsiTreeUtil;
1919
import net.seesharpsoft.intellij.lang.FileParserDefinition;
20+
import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
21+
import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
2022
import net.seesharpsoft.intellij.plugins.csv.psi.CsvField;
2123
import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
2224
import net.seesharpsoft.intellij.plugins.csv.psi.CsvRecord;
@@ -149,14 +151,23 @@ public static int getFieldEndOffset(PsiElement field) {
149151
return separator == null ? field.getContainingFile().getTextLength() : separator.getTextOffset();
150152
}
151153

154+
public static CsvEditorSettings.EscapeCharacter getCurrentEscapeCharacter(CsvFile csvFile) {
155+
return getCurrentEscapeCharacter(csvFile.getContainingFile());
156+
}
157+
158+
public static CsvEditorSettings.EscapeCharacter getCurrentEscapeCharacter(PsiFile psiFile) {
159+
return CsvFileAttributes.getInstance(psiFile.getProject()).getEscapeCharacter(psiFile);
160+
}
161+
152162
public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile) {
163+
CsvEditorSettings.EscapeCharacter escapeCharacter = getCurrentEscapeCharacter(csvFile);
153164
Map<Integer, CsvColumnInfo<PsiElement>> columnInfoMap = new HashMap<>();
154165
CsvRecord[] records = PsiTreeUtil.getChildrenOfType(csvFile, CsvRecord.class);
155166
int row = 0;
156167
for (CsvRecord record : records) {
157168
int column = 0;
158169
for (CsvField field : record.getFieldList()) {
159-
Integer length = CsvHelper.getMaxTextLineLength(unquoteCsvValue(field.getText()));
170+
Integer length = CsvHelper.getMaxTextLineLength(unquoteCsvValue(field.getText(), escapeCharacter));
160171
if (!columnInfoMap.containsKey(column)) {
161172
columnInfoMap.put(column, new CsvColumnInfo(column, length, row));
162173
} else if (columnInfoMap.get(column).getMaxLength() < length) {
@@ -170,7 +181,7 @@ public static CsvColumnInfoMap<PsiElement> createColumnInfoMap(CsvFile csvFile)
170181
return new CsvColumnInfoMap(columnInfoMap, PsiTreeUtil.hasErrorElements(csvFile));
171182
}
172183

173-
public static String unquoteCsvValue(String content) {
184+
public static String unquoteCsvValue(String content, CsvEditorSettings.EscapeCharacter escapeCharacter) {
174185
if (content == null) {
175186
return "";
176187
}
@@ -186,12 +197,12 @@ private static boolean isQuotingRequired(String content, String separator) {
186197
return content != null && (content.contains(separator) || content.contains("\"") || content.contains("\n") || content.startsWith(" ") || content.endsWith(" "));
187198
}
188199

189-
public static String quoteCsvField(String content, String separator, boolean quotingEnforced) {
200+
public static String quoteCsvField(String content, CsvEditorSettings.EscapeCharacter escapeCharacter, String separator, boolean quotingEnforced) {
190201
if (content == null) {
191202
return "";
192203
}
193204
if (quotingEnforced || isQuotingRequired(content, separator)) {
194-
String result = content.replaceAll("\"", "\"\"");
205+
String result = content.replaceAll("\"", escapeCharacter.getCharacter() + "\"");
195206
return "\"" + result + "\"";
196207
}
197208
return content;

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexer.flex‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ package net.seesharpsoft.intellij.plugins.csv;
22

33
import com.intellij.lexer.FlexLexer;
44
import com.intellij.psi.tree.IElementType;
5-
import com.intellij.openapi.project.Project;
5+
import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
66
import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
77
import com.intellij.psi.TokenType;
88

9+
import java.util.regex.Pattern;
10+
911
%%
1012

1113
%class CsvLexer
@@ -14,21 +16,25 @@ import com.intellij.psi.TokenType;
1416
%function advance
1517
%type IElementType
1618
%{
17-
private String currentSeparator;
18-
19-
/**
20-
* Provide constructor that supports a Project as parameter.
21-
*/
22-
CsvLexer(java.io.Reader in, String separator) {
23-
this(in);
24-
this.currentSeparator = separator;
25-
}
19+
private String currentSeparator;
20+
private CsvEditorSettings.EscapeCharacter myEscapeCharacter;
21+
22+
private static final Pattern ESCAPE_TEXT_PATTERN = Pattern.compile("[,;|\\t\\r\\n]");
23+
24+
/**
25+
* Provide constructor that supports a Project as parameter.
26+
*/
27+
CsvLexer(java.io.Reader in, String separator, CsvEditorSettings.EscapeCharacter escapeCharacter) {
28+
this(in);
29+
this.currentSeparator = separator;
30+
myEscapeCharacter = escapeCharacter;
31+
}
2632
%}
2733
%eof{ return;
2834
%eof}
2935

30-
TEXT=[^ ,;|\t\r\n\"]+
31-
ESCAPED_TEXT=([,;|\t\r\n]|\"\")+
36+
TEXT=[^ ,;|\t\r\n\"\\]+
37+
ESCAPED_TEXT=([,;|\t\r\n\\]|\"\"|\\\")+
3238
QUOTE=\"
3339
COMMA=[,;|\t]
3440
EOL=\n
@@ -65,7 +71,13 @@ WHITE_SPACE=[ \f]+
6571

6672
<ESCAPED_TEXT> {ESCAPED_TEXT}
6773
{
68-
return CsvTypes.ESCAPED_TEXT;
74+
String text = yytext().toString();
75+
if (myEscapeCharacter.isEscapedQuote(text)
76+
|| ESCAPE_TEXT_PATTERN.matcher(text).matches()
77+
) {
78+
return CsvTypes.ESCAPED_TEXT;
79+
}
80+
return TokenType.BAD_CHARACTER;
6981
}
7082

7183
<YYINITIAL, AFTER_TEXT, UNESCAPED_TEXT> {COMMA}

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvLexerAdapter.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package net.seesharpsoft.intellij.plugins.csv;
22

33
import com.intellij.lexer.FlexAdapter;
4+
import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
45

56
public class CsvLexerAdapter extends FlexAdapter {
6-
public CsvLexerAdapter(String separator) {
7-
super(new CsvLexer(null, separator));
7+
public CsvLexerAdapter(String separator, CsvEditorSettings.EscapeCharacter escapeCharacter) {
8+
super(new CsvLexer(null, separator, escapeCharacter));
89
}
910
}

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserDefinition.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import com.intellij.psi.tree.IFileElementType;
1212
import com.intellij.psi.tree.TokenSet;
1313
import net.seesharpsoft.intellij.lang.FileParserDefinition;
14+
import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
1415
import net.seesharpsoft.intellij.plugins.csv.parser.CsvParser;
1516
import net.seesharpsoft.intellij.plugins.csv.psi.CsvFile;
1617
import net.seesharpsoft.intellij.plugins.csv.psi.CsvFileElementType;
@@ -76,11 +77,11 @@ public PsiElement createElement(ASTNode node) {
7677

7778
@Override
7879
public Lexer createLexer(@NotNull PsiFile file) {
79-
return new CsvLexerAdapter(CsvCodeStyleSettings.getCurrentSeparator(file));
80+
return new CsvLexerAdapter(CsvCodeStyleSettings.getCurrentSeparator(file), CsvFileAttributes.getInstance(file.getProject()).getEscapeCharacter(file));
8081
}
8182

8283
@Override
8384
public PsiParser createParser(@NotNull PsiFile file) {
8485
return createParser(file.getProject());
8586
}
86-
}
87+
}

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvParserUtil.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,44 @@
33
import com.intellij.lang.PsiBuilder;
44
import com.intellij.psi.PsiFile;
55
import com.intellij.psi.impl.source.resolve.FileContextUtil;
6+
import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
67
import net.seesharpsoft.intellij.plugins.csv.psi.CsvTypes;
78
import net.seesharpsoft.intellij.plugins.csv.settings.CsvCodeStyleSettings;
89

10+
import java.util.regex.Pattern;
11+
912
public final class CsvParserUtil {
1013

14+
public static final Pattern ESCAPE_TEXT_PATTERN = Pattern.compile("[,;|\\t\\r\\n]");
15+
1116
private CsvParserUtil() {
1217
// static utility class
1318
}
1419

1520
public static boolean separator(PsiBuilder builder, int tokenType) {
1621
if (builder.getTokenType() == CsvTypes.COMMA) {
17-
PsiFile currentFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
18-
if (currentFile == null) {
22+
PsiFile psiFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
23+
if (psiFile == null) {
1924
throw new UnsupportedOperationException("parser requires containing file");
2025
}
2126
return builder.getTokenText().equals(
22-
CsvCodeStyleSettings.getCurrentSeparator(currentFile)
27+
CsvCodeStyleSettings.getCurrentSeparator(psiFile)
2328
);
2429
}
2530
return false;
2631
}
2732

33+
public static boolean escapeCharacter(PsiBuilder builder, int tokenType) {
34+
if (builder.getTokenType() == CsvTypes.ESCAPED_TEXT) {
35+
PsiFile psiFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
36+
if (psiFile == null) {
37+
throw new UnsupportedOperationException("parser requires containing file");
38+
}
39+
String tokenText = builder.getTokenText();
40+
return CsvFileAttributes.getInstance(psiFile.getProject()).getEscapeCharacter(psiFile).isEscapedQuote(tokenText) ||
41+
ESCAPE_TEXT_PATTERN.matcher(tokenText).matches();
42+
}
43+
return false;
44+
}
45+
2846
}

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/CsvStorageHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public static String getRelativeFileUrl(Project project, VirtualFile virtualFile
1717
return null;
1818
}
1919
String url = virtualFile.getUserData(RELATIVE_FILE_URL);
20-
if (url == null) {
20+
if (url == null && project.getBasePath() != null) {
2121
String projectDir = PathUtil.getLocalPath(project.getBasePath());
2222
url = PathUtil.getLocalPath(virtualFile.getPath())
2323
.replaceFirst("^" + Pattern.quote(projectDir), "");

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/actions/CsvChangeEscapeCharacterAction.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package net.seesharpsoft.intellij.plugins.csv.actions;
2+
3+
import com.intellij.openapi.actionSystem.AnActionEvent;
4+
import com.intellij.openapi.actionSystem.CommonDataKeys;
5+
import com.intellij.openapi.actionSystem.PlatformDataKeys;
6+
import com.intellij.openapi.actionSystem.ToggleAction;
7+
import com.intellij.openapi.fileEditor.FileEditor;
8+
import com.intellij.psi.PsiFile;
9+
import com.intellij.util.FileContentUtil;
10+
import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
11+
import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
12+
import org.jetbrains.annotations.NotNull;
13+
14+
public class CsvChangeEscapeCharacterAction extends ToggleAction {
15+
private CsvEditorSettings.EscapeCharacter myEscapeCharacter;
16+
17+
CsvChangeEscapeCharacterAction(CsvEditorSettings.EscapeCharacter escapeCharacter) {
18+
super(escapeCharacter.getDisplay());
19+
myEscapeCharacter = escapeCharacter;
20+
}
21+
22+
@Override
23+
public boolean isSelected(@NotNull AnActionEvent anActionEvent) {
24+
PsiFile psiFile = anActionEvent.getData(CommonDataKeys.PSI_FILE);
25+
if (psiFile == null) {
26+
return false;
27+
}
28+
CsvFileAttributes csvFileAttributes = CsvFileAttributes.getInstance(psiFile.getProject());
29+
return csvFileAttributes.hasEscapeCharacterAttribute(psiFile) && csvFileAttributes.getEscapeCharacter(psiFile).equals(myEscapeCharacter);
30+
}
31+
32+
@Override
33+
public void setSelected(@NotNull AnActionEvent anActionEvent, boolean selected) {
34+
PsiFile psiFile = anActionEvent.getData(CommonDataKeys.PSI_FILE);
35+
if (psiFile == null) {
36+
return;
37+
}
38+
CsvFileAttributes.getInstance(psiFile.getProject()).setEscapeCharacter(psiFile, this.myEscapeCharacter);
39+
FileContentUtil.reparseFiles(psiFile.getVirtualFile());
40+
41+
FileEditor fileEditor = anActionEvent.getData(PlatformDataKeys.FILE_EDITOR);
42+
if (fileEditor != null) {
43+
fileEditor.selectNotify();
44+
}
45+
}
46+
}

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/actions/CsvChangeEscapeCharacterActionGroup.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package net.seesharpsoft.intellij.plugins.csv.actions;
2+
3+
import com.intellij.lang.Language;
4+
import com.intellij.openapi.actionSystem.ActionGroup;
5+
import com.intellij.openapi.actionSystem.AnAction;
6+
import com.intellij.openapi.actionSystem.AnActionEvent;
7+
import com.intellij.openapi.actionSystem.CommonDataKeys;
8+
import com.intellij.psi.PsiFile;
9+
import net.seesharpsoft.intellij.plugins.csv.CsvLanguage;
10+
import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
11+
import net.seesharpsoft.intellij.plugins.csv.editor.CsvEditorSettings;
12+
import org.jetbrains.annotations.NotNull;
13+
import org.jetbrains.annotations.Nullable;
14+
15+
public class CsvChangeEscapeCharacterActionGroup extends ActionGroup {
16+
17+
private static final AnAction[] CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS;
18+
19+
static {
20+
CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS = new AnAction[CsvEditorSettings.EscapeCharacter.values().length + 1];
21+
for (int i = 0; i < CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS.length - 1; ++i) {
22+
CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS[i] = new CsvChangeEscapeCharacterAction(CsvEditorSettings.EscapeCharacter.values()[i]);
23+
}
24+
CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS[CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS.length - 1] = new CsvDefaultEscapeCharacterAction();
25+
}
26+
27+
@Override
28+
public void update(AnActionEvent anActionEvent) {
29+
PsiFile psiFile = anActionEvent.getData(CommonDataKeys.PSI_FILE);
30+
Language language = psiFile == null ? null : psiFile.getLanguage();
31+
anActionEvent.getPresentation().setEnabledAndVisible(psiFile != null && language != null && language.isKindOf(CsvLanguage.INSTANCE));
32+
33+
if (psiFile != null) {
34+
CsvEditorSettings.EscapeCharacter escapeCharacter = CsvFileAttributes.getInstance(psiFile.getProject()).getEscapeCharacter(psiFile);
35+
anActionEvent.getPresentation().setText(String.format("CSV Escape Character: %s", escapeCharacter.getDisplay()));
36+
}
37+
}
38+
39+
@NotNull
40+
@Override
41+
public AnAction[] getChildren(@Nullable AnActionEvent anActionEvent) {
42+
return CSV_ESCAPE_CHARACTER_CHANGE_ACTIONS;
43+
}
44+
}

‎src/main/java/net/seesharpsoft/intellij/plugins/csv/actions/CsvDefaultEscapeCharacterAction.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package net.seesharpsoft.intellij.plugins.csv.actions;
2+
3+
import com.intellij.lang.Language;
4+
import com.intellij.openapi.actionSystem.AnActionEvent;
5+
import com.intellij.openapi.actionSystem.CommonDataKeys;
6+
import com.intellij.openapi.actionSystem.PlatformDataKeys;
7+
import com.intellij.openapi.actionSystem.ToggleAction;
8+
import com.intellij.openapi.components.ServiceManager;
9+
import com.intellij.openapi.fileEditor.FileEditor;
10+
import com.intellij.psi.PsiFile;
11+
import com.intellij.util.FileContentUtil;
12+
import net.seesharpsoft.intellij.plugins.csv.CsvLanguage;
13+
import net.seesharpsoft.intellij.plugins.csv.CsvSeparatorHolder;
14+
import net.seesharpsoft.intellij.plugins.csv.components.CsvFileAttributes;
15+
import org.jetbrains.annotations.NotNull;
16+
17+
public class CsvDefaultEscapeCharacterAction extends ToggleAction {
18+
CsvDefaultEscapeCharacterAction() {
19+
super("Project Default");
20+
}
21+
22+
@Override
23+
public boolean isSelected(@NotNull AnActionEvent anActionEvent) {
24+
PsiFile psiFile = anActionEvent.getData(CommonDataKeys.PSI_FILE);
25+
if (psiFile == null) {
26+
return false;
27+
}
28+
return !CsvFileAttributes.getInstance(psiFile.getProject()).hasEscapeCharacterAttribute(psiFile);
29+
}
30+
31+
@Override
32+
public void setSelected(@NotNull AnActionEvent anActionEvent, boolean selected) {
33+
PsiFile psiFile = anActionEvent.getData(CommonDataKeys.PSI_FILE);
34+
if (psiFile == null) {
35+
return;
36+
}
37+
CsvFileAttributes.getInstance(psiFile.getProject()).resetEscapeSeparator(psiFile);
38+
FileContentUtil.reparseFiles(psiFile.getVirtualFile());
39+
40+
FileEditor fileEditor = anActionEvent.getData(PlatformDataKeys.FILE_EDITOR);
41+
if (fileEditor != null) {
42+
fileEditor.selectNotify();
43+
}
44+
}
45+
}

0 commit comments

Comments
 (0)