Much more stable dialect sniffer

The parser can now infer CSV dialects in a much more stable way, omitting the propagation of average errors to the table record scores.

Author: ws-garcia

src/Access_version.zip

@@ -527,7 +527,7 @@ End Function
 ''' <summary>
 ''' Calculates a score for the imported data based on the congruence
 ''' of the detected data and the uniformity of the fields contained
-''' in each record. The score is in the range 0 < x <= 100.
+''' in each record. The score is in the range 0 < x <= inf.
 ''' </summary>
 ''' <param name="ArrayList">CSV imported data.</param>
 Public Function TableScore(ByRef ArrayList As CSVArrayList) As Double
@@ -540,7 +540,7 @@ Public Function TableScore(ByRef ArrayList As CSVArrayList) As Double
             For L0 = 0 To ArrayList.count - 1
                 SumRecScores = SumRecScores + RecordScore(ArrayList(L0))
             Next L0
-            TableScore = RecordsConsistencyFactor(ArrayList) * SumRecScores / ArrayList.count
+            TableScore = RecordsConsistencyFactor(ArrayList) * SumRecScores
         End If
     End If
 End Function

src/All_Host_version.zip

@@ -3149,9 +3149,7 @@ Private Function SniffInString(ByRef confObject As CSVparserConfig, _
         Set ScoreArray = New CSVArrayList
         ScoreArray.indexing = True
         For i = LBound(TmpDelimiters) To UBound(TmpDelimiters)
-            If InStrB(1, TmpCSVstr, TmpDelimiters(i)) Then
                 For j = LBound(LinesEnds) To UBound(LinesEnds)
-                    If InStrB(1, TmpCSVstr, LinesEnds(j)) Then
                         For k = LBound(QuoteChar) To UBound(QuoteChar)
                             '@--------------------------------------------------------------------------------
                             'Set CSV dialect
@@ -3168,9 +3166,7 @@ Private Function SniffInString(ByRef confObject As CSVparserConfig, _
                                 ScoreArray.AddIndexedItem DialectToString(.dialect) & CHR_CARET, GuesserHelper.TableScore(ImportedTable) / 2
                             End If
                         Next k
-                    End If
                 Next j
-            End If
         Next i
     End With
     With ScoreArray
@@ -3183,6 +3179,26 @@ Private Function SniffInString(ByRef confObject As CSVparserConfig, _
         If InStrB(1, TmpCSVstr, CHR_BACKSLASH) Then
             SniffInString.escapeMode = unix
         End If
+        '@--------------------------------------------------------------------------------
+        '2nd pass for unique record CSV file
+        Set ImportedTable = New CSVArrayList
+        Set tmpConfig.dialect = SniffInString
+        ParseCSVstring TmpCSVstr, tmpConfig, ImportedTable, EmptyParam
+        If ImportedTable.count = 1 Then
+            If InStrB(1, TmpCSVstr, vbCrLf) Then
+                SniffInString.recordsDelimiter = vbCrLf
+            Else
+                If InStrB(1, TmpCSVstr, vbCr) Then
+                    SniffInString.recordsDelimiter = vbCr
+                Else
+                    If InStrB(1, TmpCSVstr, vbLf) Then
+                        SniffInString.recordsDelimiter = vbLf
+                    Else
+                        SniffInString.recordsDelimiter = vbCrLf
+                    End If
+                End If
+            End If
+        End If
     End With
     Erase EmptyParam
     Erase LinesEnds

src/CSVSniffer.cls

@@ -0,0 +1,21 @@
+=== Delimiters guessing test ===
++ Mixed comma and semicolon
++ File with multi-line field
++ Optional quoted fields
++ Mixed comma and semicolon - file B
++ Geometric CSV
++ Table embedded in the last record
++ Table embedded in the second record
++ Multiple commas in fields
++ Uncommon char as field delimiter
++ Wrong delimiters have been added to guessing operation
++ FEC data - [clevercsv issue #15]
++ Mixed comma and colon - [clevercsv issue #35]
++ Json data type - [clevercsv issue #37]
++ Undefined field delimiter
++ Rainbow CSV [issue #92]
++ Pipe character is more frequent than the comma
++ Pipe character is more frequent than the semicolon
++ Short pipe separated table embedded
+= PASS (18 of 18 passed) = 3/4/2023 11:55:13 p.�m. =
+