[FEATURE] auto detect value separator

Author: SeeSharpSoft

build.gradle

@@ -27,8 +27,8 @@ group 'net.seesharpsoft.intellij.plugins'
 version '2.14.3'
 
 apply plugin: 'java'
-sourceCompatibility = javaVersion
-targetCompatibility = javaTargetVersion
+project.sourceCompatibility = JavaVersion.VERSION_11
+project.targetCompatibility = JavaVersion.VERSION_11
 tasks.withType(JavaCompile) { options.encoding = 'UTF-8' }
 repositories {
     mavenCentral()
@@ -60,8 +60,8 @@ sourceSets {
 apply plugin: 'idea'
 idea {
     project {
-        jdkName = javaVersion
-        languageLevel = javaVersion
+        jdkName = JavaVersion.VERSION_11
+        languageLevel = JavaVersion.VERSION_11
         vcs = 'Git'
     }
     module {

gradle.properties

@@ -3,5 +3,3 @@
 # https://www.jetbrains.com/intellij-repository/snapshots
 
 name='CSV Plugin'
-javaVersion=11
-javaTargetVersion=11

src/main/java/net/seesharpsoft/intellij/plugins/csv/components/CsvFileAttributes.java

@@ -5,17 +5,22 @@
 import com.intellij.openapi.components.ServiceManager;
 import com.intellij.openapi.components.State;
 import com.intellij.openapi.components.Storage;
+import com.intellij.openapi.editor.Document;
+import com.intellij.openapi.fileEditor.FileDocumentManager;
 import com.intellij.openapi.fileTypes.LanguageFileType;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiFile;
 import com.intellij.util.xmlb.XmlSerializerUtil;
 import com.intellij.util.xmlb.annotations.OptionTag;
+import net.seesharpsoft.commons.collection.Pair;
 import net.seesharpsoft.intellij.plugins.csv.*;
 import net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings;
+import org.apache.commons.lang.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -87,12 +92,18 @@ public boolean canChangeValueSeparator(@NotNull PsiFile psiFile) {
         return language.isKindOf(CsvLanguage.INSTANCE) && !(language instanceof CsvSeparatorHolder);
     }
 
+    private void setFileSeparator(@NotNull Project project, @NotNull VirtualFile virtualFile, @NotNull CsvValueSeparator separator) {
+        Attribute attribute = getFileAttribute(project, virtualFile, true);
+        if (attribute != null) {
+            attribute.separator = separator;
+        }
+    }
+
     public void setFileSeparator(@NotNull PsiFile psiFile, @NotNull CsvValueSeparator separator) {
         if (!canChangeValueSeparator(psiFile)) {
             return;
         }
-        Attribute attribute = getFileAttribute(psiFile.getProject(), psiFile.getOriginalFile().getVirtualFile(), true);
-        attribute.separator = separator;
+        setFileSeparator(psiFile.getProject(), psiFile.getOriginalFile().getVirtualFile(), separator);
     }
 
     public void resetValueSeparator(@NotNull PsiFile psiFile) {
@@ -105,18 +116,51 @@ public void resetValueSeparator(@NotNull PsiFile psiFile) {
         }
     }
 
+    private @NotNull
+    CsvValueSeparator guessOrGetValueSeparator(Project project, VirtualFile virtualFile) {
+        return CsvEditorSettings.getInstance().isGuessValueSeparator() ?
+                guessValueSeparator(project, virtualFile) :
+                CsvEditorSettings.getInstance().getDefaultValueSeparator();
+    }
+
+    private @NotNull
+    CsvValueSeparator guessValueSeparator(Project project, VirtualFile virtualFile) {
+        Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
+        final String text = document == null ? "" : document.getText();
+        Pair<CsvValueSeparator, Integer> separatorWithCount =
+                Arrays.stream(CsvValueSeparator.values())
+                        // count
+                        .map(separator -> {
+                            String character = separator.getCharacter();
+                            return Pair.of(separator, StringUtils.countMatches(text, character));
+                        })
+                        // ignore non-matched separators
+                        .filter(p -> p.getSecond() > 0)
+                        // get the one with most hits
+                        .max((p1, p2) -> p1.getSecond() - p2.getSecond())
+                        // failsafe (e.g. empty document)
+                        .orElse(null);
+
+        CsvValueSeparator valueSeparator = separatorWithCount != null ?
+                separatorWithCount.getFirst() :
+                CsvEditorSettings.getInstance().getDefaultValueSeparator();
+
+        setFileSeparator(project, virtualFile, valueSeparator);
+        return valueSeparator;
+    }
+
     public @NotNull
     CsvValueSeparator getValueSeparator(Project project, VirtualFile virtualFile) {
         if (project == null || virtualFile == null || !(virtualFile.getFileType() instanceof LanguageFileType)) {
             return CsvEditorSettings.getInstance().getDefaultValueSeparator();
         }
         Language language = ((LanguageFileType) virtualFile.getFileType()).getLanguage();
         if (language instanceof CsvSeparatorHolder) {
-                return ((CsvSeparatorHolder) language).getSeparator();
+            return ((CsvSeparatorHolder) language).getSeparator();
         }
         Attribute attribute = getFileAttribute(project, virtualFile);
         return attribute == null || attribute.separator == null ?
-                CsvEditorSettings.getInstance().getDefaultValueSeparator() :
+                guessOrGetValueSeparator(project, virtualFile) :
                 attribute.separator;
     }
 
@@ -127,7 +171,9 @@ public boolean hasValueSeparatorAttribute(@NotNull Project project, @NotNull Vir
 
     public void setEscapeCharacter(@NotNull PsiFile psiFile, @NotNull CsvEscapeCharacter escapeCharacter) {
         Attribute attribute = getFileAttribute(psiFile.getProject(), psiFile.getOriginalFile().getVirtualFile(), true);
-        attribute.escapeCharacter = escapeCharacter;
+        if (attribute != null) {
+            attribute.escapeCharacter = escapeCharacter;
+        }
     }
 
     public void resetEscapeSeparator(@NotNull PsiFile psiFile) {

src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettings.java

@@ -84,6 +84,7 @@ public static final class OptionSet {
         public boolean KEEP_TRAILING_SPACES = false;
         public String COMMENT_INDICATOR = COMMENT_INDICATOR_DEFAULT;
         public ValueColoring VALUE_COLORING = ValueColoring.RAINBOW;
+        public boolean GUESS_VALUE_SEPARATOR = true;
 
         public OptionSet() {
             EditorSettingsExternalizable editorSettingsExternalizable = EditorSettingsExternalizable.getInstance();
@@ -312,6 +313,14 @@ public void setHeaderRowFixed(boolean headerRowFixed) {
         getState().TABLE_HEADER_ROW_FIXED = headerRowFixed;
     }
 
+    public boolean isGuessValueSeparator() {
+        return getState().GUESS_VALUE_SEPARATOR;
+    }
+
+    public void setGuessValueSeparator(boolean guessValueSeparator) {
+        getState().GUESS_VALUE_SEPARATOR = guessValueSeparator;
+    }
+
     public boolean checkCurrentPluginVersion(String actualVersion) {
         if (!actualVersion.equals(getState().CURRENT_PLUGIN_VERSION)) {
             getState().CURRENT_PLUGIN_VERSION = actualVersion;

src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.form

@@ -343,7 +343,7 @@
           </component>
         </children>
       </grid>
-      <grid id="29095" layout-manager="GridLayoutManager" row-count="1" column-count="3" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">
+      <grid id="29095" layout-manager="GridLayoutManager" row-count="1" column-count="4" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">
         <margin top="0" left="0" bottom="0" right="0"/>
         <constraints>
           <grid row="1" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="0" fill="3" indent="0" use-parent-layout="false"/>
@@ -363,7 +363,7 @@
           </component>
           <hspacer id="f2aa1">
             <constraints>
-              <grid row="0" column="2" row-span="1" col-span="1" vsize-policy="1" hsize-policy="6" anchor="0" fill="1" indent="0" use-parent-layout="false"/>
+              <grid row="0" column="3" row-span="1" col-span="1" vsize-policy="1" hsize-policy="6" anchor="0" fill="1" indent="0" use-parent-layout="false"/>
             </constraints>
           </hspacer>
           <component id="65f7" class="javax.swing.JComboBox" binding="comboValueSeparator" custom-create="true">
@@ -372,6 +372,14 @@
             </constraints>
             <properties/>
           </component>
+          <component id="aed54" class="javax.swing.JCheckBox" binding="cbGuessSeparator">
+            <constraints>
+              <grid row="0" column="2" row-span="1" col-span="1" vsize-policy="0" hsize-policy="3" anchor="8" fill="0" indent="0" use-parent-layout="false"/>
+            </constraints>
+            <properties>
+              <text value="Initial separator guessing"/>
+            </properties>
+          </component>
         </children>
       </grid>
       <grid id="7eea1" layout-manager="GridLayoutManager" row-count="1" column-count="3" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">

src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.java

@@ -43,6 +43,7 @@ public class CsvEditorSettingsProvider implements EditorOptionsProvider {
     private JTextField tfCommentIndicator;
     private JComboBox comboValueColoring;
     private JCheckBox cbHeaderRowFixed;
+    private JCheckBox cbGuessSeparator;
 
     @NotNull
     @Override
@@ -94,7 +95,8 @@ public boolean isModified() {
                 isModified(cbKeepTrailingWhitespaces, csvEditorSettings.getKeepTrailingSpaces()) ||
                 isModified(tfCommentIndicator, csvEditorSettings.getCommentIndicator()) ||
                 !Objects.equals(comboValueColoring.getSelectedItem(), csvEditorSettings.getValueColoring()) ||
-                isModified(cbHeaderRowFixed, csvEditorSettings.isHeaderRowFixed());
+                isModified(cbHeaderRowFixed, csvEditorSettings.isHeaderRowFixed()) ||
+                isModified(cbGuessSeparator, csvEditorSettings.isGuessValueSeparator());
     }
 
     @Override
@@ -120,6 +122,7 @@ public void reset() {
         tfCommentIndicator.setText(csvEditorSettings.getCommentIndicator());
         comboValueColoring.setSelectedItem(csvEditorSettings.getValueColoring());
         cbHeaderRowFixed.setSelected(csvEditorSettings.isHeaderRowFixed());
+        cbGuessSeparator.setSelected(csvEditorSettings.isGuessValueSeparator());
     }
 
     @Override
@@ -145,6 +148,7 @@ public void apply() throws ConfigurationException {
         csvEditorSettings.setCommentIndicator(tfCommentIndicator.getText());
         csvEditorSettings.setValueColoring((CsvEditorSettings.ValueColoring) comboValueColoring.getSelectedItem());
         csvEditorSettings.setHeaderRowFixed(cbHeaderRowFixed.isSelected());
+        csvEditorSettings.setGuessValueSeparator(cbGuessSeparator.isSelected());
     }
 
     protected void createUIComponents() {