bpo-30825: guess lineterminator in csv.Sniffer

Lib/csv.py

@@ -12,6 +12,7 @@
 from _csv import Dialect as _Dialect
 
 from io import StringIO
+import os
 
 __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
            "Error", "Dialect", "__doc__", "excel", "excel_tab",
@@ -164,7 +165,7 @@ def writerows(self, rowdicts):
 
 class Sniffer:
     '''
-    "Sniffs" the format of a CSV file (i.e. delimiter, quotechar)
+    "Sniffs" the format of a CSV file (i.e. delimiter, quotechar, lineterminator)
     Returns a Dialect object.
     '''
     def __init__(self):
@@ -177,18 +178,19 @@ def sniff(self, sample, delimiters=None):
         Returns a dialect (or None) corresponding to the sample
         """
 
+        lineterminator = self._guess_lineterminator(sample)
         quotechar, doublequote, delimiter, skipinitialspace = \
                    self._guess_quote_and_delimiter(sample, delimiters)
         if not delimiter:
             delimiter, skipinitialspace = self._guess_delimiter(sample,
-                                                                delimiters)
+                                                                delimiters,
+                                                                lineterminator)
 
         if not delimiter:
             raise Error("Could not determine delimiter")
 
         class dialect(Dialect):
             _name = "sniffed"
-            lineterminator = '\r\n'
             quoting = QUOTE_MINIMAL
             # escapechar = ''
 
@@ -197,6 +199,7 @@ class dialect(Dialect):
         # _csv.reader won't accept a quotechar of ''
         dialect.quotechar = quotechar or '"'
         dialect.skipinitialspace = skipinitialspace
+        dialect.lineterminator = lineterminator
 
         return dialect
 
@@ -277,7 +280,7 @@ def _guess_quote_and_delimiter(self, data, delimiters):
         return (quotechar, doublequote, delim, skipinitialspace)
 
 
-    def _guess_delimiter(self, data, delimiters):
+    def _guess_delimiter(self, data, delimiters, lineterminator):
         """
         The delimiter /should/ occur the same number of times on
         each row. However, due to malformed data, it may not. We don't want
@@ -296,7 +299,7 @@ def _guess_delimiter(self, data, delimiters):
         additional chunks as necessary.
         """
 
-        data = list(filter(None, data.split('\n')))
+        data = list(filter(None, data.split(lineterminator)))
 
         ascii = [chr(c) for c in range(127)] # 7-bit ASCII
 
@@ -446,3 +449,14 @@ def has_header(self, sample):
                     hasHeader -= 1
 
         return hasHeader > 0
+
+
+    def _guess_lineterminator(self, sample):
+        # Guess line terminator based on presence
+        terminators = ('\r\n', '\n', '\r')
+
+        for terminator in terminators:
+            if terminator in sample:
+                return terminator
+
+        return os.linesep

Lib/test/test_csv.py

@@ -9,6 +9,7 @@
 import csv
 import gc
 import pickle
+import os
 from test import support
 from itertools import permutations
 from textwrap import dedent
@@ -1051,6 +1052,20 @@ def test_doublequote(self):
         dialect = sniffer.sniff(self.sample9)
         self.assertTrue(dialect.doublequote)
 
+    def test_guess_lineterminator(self):
+        sniffer = csv.Sniffer()
+        dialect = sniffer.sniff('Date;Value\r\n2010-01-01;10')
+        self.assertEqual(dialect.lineterminator, '\r\n')
+        dialect = sniffer.sniff('Date;Value\n2010-01-01;10')
+        self.assertEqual(dialect.lineterminator, '\n')
+        dialect = sniffer.sniff('Date;Value\r2010-01-01;10')
+        self.assertEqual(dialect.lineterminator, '\r')
+        dialect = sniffer.sniff('Date;Value\v2010-01-01;10')
+        self.assertEqual(dialect.lineterminator, os.linesep)
+        dialect = sniffer.sniff('Date;Value')
+        self.assertEqual(dialect.lineterminator, os.linesep)
+
+
 class NUL:
     def write(s, *args):
         pass

Misc/NEWS.d/next/Library/2018-02-12-05-10-28.bpo-30825.t0RXql.rst

@@ -0,0 +1 @@
+csv.Sniffer now detects line terminator instead of defaulting to '\r\n'