Added more delimiter guessing tests. Renamed some auxiliary guessing methods.

Author: ws-garcia

src/Access_version.zip

@@ -210,45 +210,6 @@ Private Sub Class_Terminate()
     Set P_CSV_DATA = Nothing
     Set P_CSV_HEADER = Nothing
 End Sub
-Private Function ArrayListAVGStdDev(ByRef ArrayList As ECPArrayList) As Double
-    Dim L0 As Long
-    Dim StdDevSUM As Double
-    
-    StdDevSUM = 0
-    For L0 = 0 To ArrayList.count - 1
-        StdDevSUM = StdDevSUM + ArrayStrStdDev(ArrayList(L0))
-    Next L0
-    ArrayListAVGStdDev = StdDevSUM / ArrayList.count
-End Function
-Private Function ArrayStrLenAVG(ByRef strArray As Variant) As Double
-    Dim L0 As Long
-    Dim tmpSUM As Double
-    
-    tmpSUM = 0
-    For L0 = LBound(strArray) To UBound(strArray)
-        tmpSUM = tmpSUM + LenB(strArray(L0))
-    Next L0
-    ArrayStrLenAVG = tmpSUM / (1 + UBound(strArray) - LBound(strArray))
-End Function
-Private Function ArrayStrStdDev(ByRef strArray As Variant) As Double
-    Dim arrLB As Long
-    Dim arrUB As Long
-    Dim L0 As Long
-    Dim LengthsAVG As Double
-    Dim CumulativeAVGsquaredDiff As Double
-    Dim LengthsSum As Double
-    
-    On Error GoTo ArrStrStdDev_ErrHandler
-    LengthsAVG = ArrayStrLenAVG(strArray)
-    CumulativeAVGsquaredDiff = 0
-    For L0 = LBound(strArray) To UBound(strArray)
-        CumulativeAVGsquaredDiff = CumulativeAVGsquaredDiff + (LenB(strArray(L0)) - LengthsAVG) ^ 2
-    Next L0
-    ArrayStrStdDev = (CumulativeAVGsquaredDiff / (UBound(strArray) - LBound(strArray))) ^ 0.5
-    Exit Function
-ArrStrStdDev_ErrHandler:
-    ArrayStrStdDev = 1.6E+308
-End Function
 Public Sub ClearData()
     Set P_CSV_DATA = New ECPArrayList
     Set P_CSV_HEADER = New ECPArrayList
@@ -661,6 +622,16 @@ ErrHandler_ExportToCSV:
     P_SUCCESSFUL_EXPORT = False
     P_ERROR_DESC = "[CSV file Export]: " & P_ERROR_DESC
 End Sub
+Private Function FieldScore(ByRef strArray As Variant) As Double
+    Dim L0 As Long
+    Dim tmpSUM As Double
+    
+    tmpSUM = 0
+    For L0 = LBound(strArray) To UBound(strArray)
+        tmpSUM = tmpSUM + LenB(strArray(L0))
+    Next L0
+    FieldScore = tmpSUM / (1 + UBound(strArray) - LBound(strArray))
+End Function
 Private Function FileExists(ByVal FilePath As String) As Boolean
     FileExists = CBool(LenB(Dir(FilePath, vbHidden + vbNormal + vbSystem + vbReadOnly + vbArchive)))
 End Function
@@ -1305,7 +1276,7 @@ Private Sub GuessInString(ByRef confObject As parserConfig, _
     Dim emptyParam() As Variant
     Dim StartRec As Long
     Dim EndRec As Long
-    Dim FieldDelStandardDEV() As Double
+    Dim CtableScore() As Double
     Dim QuoteChar() As EscapeTokens
     Dim L0_FieldDelimiter As Long
     Dim L1_FieldDelimiter As Long
@@ -1332,7 +1303,7 @@ Private Sub GuessInString(ByRef confObject As parserConfig, _
     tmpDelimiters() = confObject.DelimitersToGuess
     '@--------------------------------------------------------------------------------
     'StdDev array
-    ReDim FieldDelStandardDEV(LBound(tmpDelimiters) To UBound(tmpDelimiters))
+    ReDim CtableScore(LBound(tmpDelimiters) To UBound(tmpDelimiters))
     '@--------------------------------------------------------------------------------
     'String to guess
     If Not Buffered Then
@@ -1376,23 +1347,23 @@ Private Sub GuessInString(ByRef confObject As parserConfig, _
                 Set guessResults = New ECPArrayList
                 ParseCSVstring CSVstring, confObject, guessResults, emptyParam
                 If Not guessResults Is Nothing Then
-                    FieldDelStandardDEV(delIdx) = ArrayListAVGStdDev(guessResults)
+                    CtableScore(delIdx) = TableScore(guessResults)
                 Else
-                    FieldDelStandardDEV(delIdx) = 1.6E+308
+                    CtableScore(delIdx) = 1.6E+308
                 End If
             Else
-                FieldDelStandardDEV(delIdx) = 1.6E+308
+                CtableScore(delIdx) = 1.6E+308
             End If
         Next delIdx
         '@--------------------------------------------------------------------------------
         'Delimiters discrimination
-        L1_FieldDelimiter = MinIndexVal(FieldDelStandardDEV)
+        L1_FieldDelimiter = MinIndexVal(CtableScore)
         .fieldsDelimiter = tmpDelimiters(L1_FieldDelimiter)
         .startingRecord = StartRec
         .endingRecord = EndRec
     End With
     Erase emptyParam
-    Erase FieldDelStandardDEV
+    Erase CtableScore
     Erase LinesEnds
     Erase QuoteChar
     Erase tmpDelimiters
@@ -2446,6 +2417,24 @@ ParseCSVstring_NoSignificantData:
                                     "the String has only empty or commented lines that can be omitted."
     Resume ParseCSVstring_Error_Handler
 End Sub
+Private Function RecordScore(ByRef strArray As Variant) As Double
+    Dim arrLB As Long
+    Dim arrUB As Long
+    Dim L0 As Long
+    Dim CfieldScore As Double
+    Dim CumulativeScore As Double
+    
+    On Error GoTo RecordScore_ErrHandler
+    CfieldScore = FieldScore(strArray)
+    CumulativeScore = 0
+    For L0 = LBound(strArray) To UBound(strArray)
+        CumulativeScore = CumulativeScore + (LenB(strArray(L0)) - CfieldScore) ^ 2
+    Next L0
+    RecordScore = (CumulativeScore / (UBound(strArray) - LBound(strArray))) ^ 0.5
+    Exit Function
+RecordScore_ErrHandler:
+    RecordScore = 1.6E+308
+End Function
 Private Function RequestedField(ByVal FieldIndex As Long, _
                                 ByVal mxReq As Long) As Boolean
     Select Case RequestedFieldsArray(0)
@@ -3129,6 +3118,16 @@ AdvanceLine:
         End If
     End If
 End Sub
+Private Function TableScore(ByRef ArrayList As ECPArrayList) As Double
+    Dim L0 As Long
+    Dim StdDevSUM As Double
+    
+    StdDevSUM = 0
+    For L0 = 0 To ArrayList.count - 1
+        StdDevSUM = StdDevSUM + RecordScore(ArrayList(L0))
+    Next L0
+    TableScore = StdDevSUM / ArrayList.count
+End Function
 Public Sub TwoDimToJaggedArray(ByRef TwoDimArray() As Variant, ByRef JaggedArray() As Variant)
     Dim UBj1 As Long, LBj1 As Long
     Dim UBj2 As Long, LBj2 As Long
@@ -3240,5 +3239,4 @@ Private Function UnixToStandardEscapeSeq(ByRef UnixEscapedString As String, _
     Else
         UnixToStandardEscapeSeq = UnixEscapedString
     End If
-End Function
-
+End Function

src/CSVinterface.cls

@@ -142,6 +142,30 @@ Function DelimitersGuessingTests(FullFileName As String) As TestSuite
                 "Expected: (" & "[" & ExpectedResult(0) & "]" & " & " & "[" & ExpectedResult(2) & "]" & ")" & _
                 "Actual: (" & "[" & ActualResult(0) & "]" & " & " & "[" & ActualResult(2) & "]" & ")"
     End With
+    '@--------------------------------------------------------------------------------
+    'FEC data - [clevercsv issue #15]
+    With DelimitersGuessingTests.test("FEC data - [clevercsv issue #15]")
+        FECdata_clevercsvIssue15
+        .IsEqual ActualResult, ExpectedResult, _
+                "Expected: (" & "[" & ExpectedResult(0) & "]" & " & " & "[" & ExpectedResult(2) & "]" & ")" & _
+                "Actual: (" & "[" & ActualResult(0) & "]" & " & " & "[" & ActualResult(2) & "]" & ")"
+    End With
+    '@--------------------------------------------------------------------------------
+    'Mixed comma and colon - [clevercsv issue #35]
+    With DelimitersGuessingTests.test("Mixed comma and colon - [clevercsv issue #35]")
+        MixedCommaAndColon_clevercsvIssue35
+        .IsEqual ActualResult, ExpectedResult, _
+                "Expected: (" & "[" & ExpectedResult(0) & "]" & " & " & "[" & ExpectedResult(2) & "]" & ")" & _
+                "Actual: (" & "[" & ActualResult(0) & "]" & " & " & "[" & ActualResult(2) & "]" & ")"
+    End With
+    '@--------------------------------------------------------------------------------
+    'Json data type - [clevercsv issue #37]
+    With DelimitersGuessingTests.test("Json data type - [clevercsv issue #37]")
+        JsonDataType_clevercsvIssue37
+        .IsEqual ActualResult, ExpectedResult, _
+                "Expected: (" & "[" & ExpectedResult(0) & "]" & " & " & "[" & ExpectedResult(2) & "]" & ")" & _
+                "Actual: (" & "[" & ActualResult(0) & "]" & " & " & "[" & ActualResult(2) & "]" & ")"
+    End With
     Set DelimitersGuessingTests = Nothing
 End Function
 Sub GetActualAndExpectedResults(FileName As String, _
@@ -228,6 +252,21 @@ Sub WrongDelimitersHaveBeenAddedToGuessingOperation()
 
     GetActualAndExpectedResults "Wrong delimiters have been added to guessing operation.csv", ",", vbLf, DoubleQuotes
 End Sub
+Sub FECdata_clevercsvIssue15()
+    Set confObj = New parserConfig
+    
+    GetActualAndExpectedResults "FEC data - [clevercsv issue #15].csv", "|", vbLf, DoubleQuotes
+End Sub
+Sub MixedCommaAndColon_clevercsvIssue35()
+    Set confObj = New parserConfig
+    
+    GetActualAndExpectedResults "Mixed comma and colon - [clevercsv issue #35].csv", ",", vbLf, Apostrophe
+End Sub
+Sub JsonDataType_clevercsvIssue37()
+    Set confObj = New parserConfig
+    
+    GetActualAndExpectedResults "Json data type - [clevercsv issue #37].csv", ",", vbLf, DoubleQuotes
+End Sub
 '/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 '#
 

src/Tests/CSVDelimitersGuessingTESTS.bas

@@ -0,0 +1,5 @@
+C00078279|A|M11|P|80031492155|22Y||MCKENNON, K R|MIDLAND|MI|00000|||10031979|400|||||CONTRIBUTION REF TO INDIVIDUAL|3062020110011466469
+C00078279|A|M11||79031415137|15||OREFFICE, P|MIDLAND|MI|00000|DOW CHEMICAL CO||10261979|1500||||||3061920110000382948
+C00078279|A|M11||79031415137|15||DOWNEY, J|MIDLAND|MI|00000|DOW CHEMICAL CO||10261979|300||||||3061920110000382949
+C00078279|A|M11||79031415137|15||BLAIR, E|MIDLAND|MI|00000|DOW CHEMICAL CO||10261979|1000||||||3061920110000382950
+C00078287|A|Q1||79031231889|15||BLANCHARD, JOHN A|CHICAGO|IL|60685|||03201979|200||||||3061920110000383914

testing/tests/Test runner [CSVinterface-TDD].xlsm

@@ -0,0 +1,8 @@
+"{""fake"": ""json"", ""fake2"":""json2""}",13:31:38,06:00:04+01:00
+"{""fake"": ""json"", ""fake2"":""json2""}",22:13:29,14:20:11+02:00
+"{""fake"": ""json"", ""fake2"":""json2""}",04:37:27,22:04:28+03:00
+"{""fake"": ""json"", ""fake2"":""json2""}",04:25:28,23:12:53+01:00
+"{""fake"": ""json"", ""fake2"":""json2""}",21:04:15,08:23:58+02:00
+"{""fake"": ""json"", ""fake2"":""json2""}",10:37:03,11:06:42+05:30
+"{""fake"": ""json"", ""fake2"":""json2""}",10:17:24,23:38:47+06:00
+"{""fake"": ""json"", ""fake2"":""json2""}",00:02:51,20:04:45-06:00

testing/tests/delimiters-guessing/FEC data - [clevercsv issue #15].csv

@@ -0,0 +1,22 @@
+bytearray(b'fake data'),20:53:06,2019-09-01T19:28:21
+bytearray(b'fake data'),19:33:15,2005-02-15T19:10:31
+bytearray(b'fake data'),10:43:05,1992-10-12T14:49:24
+bytearray(b'fake data'),10:36:49,1999-07-18T17:27:55
+bytearray(b'fake data'),03:33:35,1982-04-24T17:38:45
+bytearray(b'fake data'),14:49:47,1983-01-05T22:17:42
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45
+bytearray(b'fake data'),10:35:30,2006-10-27T02:30:45