- TypedColumnReader now supports STDIN

Author: Alan Christie

src/python/pipelines_utils/TypedColumnReader.py

@@ -25,7 +25,6 @@
 """
 
 import csv
-import gzip
 
 
 class Error(Exception):
@@ -118,13 +117,15 @@ class TypedColumnReader(object):
         "smiles,comment:string,hac:int,ratio:float"
     """
 
-    def __init__(self, filename,
+    def __init__(self, csv_file,
                  column_sep='\t',
                  type_sep=':',
                  header=None):
         """Basic initialiser.
 
-        :param filename: The typed CSV file name
+        :param csvfile: The typed CSV file. csvfile can be any object which
+                        supports the iterator protocol and returns a string
+                        each time its next() method is called
         :param column_sep: The file column separator
         :param type_sep: The type separator
         :param header: An optional header. If provided the must not have
@@ -138,15 +139,10 @@ def __init__(self, filename,
                        encouraged to add a header line to all new files.
         """
 
-        self._filename = filename
+        self._csv_file = csv_file
         self._type_sep = type_sep
         self._header = header
 
-        # Open the CSV file (which may be compressed)
-        if filename.endswith('.gz'):
-            self._csv_file = gzip.open(filename, 'rt')
-        else:
-            self._csv_file = open(filename, 'rt')
         self._c_reader = csv.reader(self._csv_file,
                                     delimiter=column_sep,
                                     skipinitialspace=True,
@@ -242,10 +238,3 @@ def _handle_hdr(self, hdr):
             self._converters.append([name, CONVERTERS[column_type]])
             self._column_names.append(name)
             column_number += 1
-
-    def __del__(self):
-        """Delete method.
-        """
-        if self._csv_file:
-            self._csv_file.close()
-            self._csv_file = None

src/python/test/python2_3/pipelines_utils/test_TypedColumnReader.py

@@ -1,3 +1,4 @@
+import gzip
 import os
 import unittest
 
@@ -12,7 +13,8 @@ def test_basic_example_a(self):
         """Test loading of a simple CSV file
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.a.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file, column_sep=',')
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file, column_sep=',')
         num_lines = 0
         first_row = {}
         for row in test_file:
@@ -22,34 +24,40 @@ def test_basic_example_a(self):
         self.assertEqual(2, num_lines)
         self.assertEqual('A string', first_row['one'])
         self.assertEqual('and finally', first_row['four'])
+        csv_file.close()
 
     def test_basic_example_a_with_supplied_header(self):
         """Test loading of a simple CSV file with a provided header
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.a-no-header.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file,
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file,
                                                         column_sep=',',
                                                         header='one,two:int,three:float,four:string')
         num_lines = 0
         for _ in test_file:
             num_lines += 1
         self.assertEqual(2, num_lines)
+        csv_file.close()
 
     def test_basic_example_a_gzip(self):
         """Test loading of a simple CSV file (gzipped)
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.a.csv.gz')
-        test_file = TypedColumnReader.TypedColumnReader(test_file, column_sep=',')
+        csv_file = gzip.open(test_file, 'rt')
+        test_file = TypedColumnReader.TypedColumnReader(csv_file, column_sep=',')
         num_lines = 0
         for _ in test_file:
             num_lines += 1
         self.assertEqual(2, num_lines)
+        csv_file.close()
 
     def test_basic_example_b_unknown_type(self):
         """Test loading of a simple CSV file with a column type that is unknown
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.b.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file, column_sep=',')
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file, column_sep=',')
         num_lines = 0
         got_exception = False
         try:
@@ -61,12 +69,14 @@ def test_basic_example_b_unknown_type(self):
             got_exception = True
         self.assertTrue(got_exception)
         self.assertEqual(0, num_lines)
+        csv_file.close()
 
     def test_basic_example_c_too_many_colons(self):
         """Test loading of a simple CSV file with a column that has too many colons
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.c.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file, column_sep=',')
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file, column_sep=',')
         num_lines = 0
         got_exception = False
         try:
@@ -79,12 +89,14 @@ def test_basic_example_c_too_many_colons(self):
             got_exception = True
         self.assertTrue(got_exception)
         self.assertEqual(0, num_lines)
+        csv_file.close()
 
     def test_basic_example_d_wrong_type(self):
         """Test loading of a simple CSV file with a column that has a string as an int
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.d.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file, column_sep=',')
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file, column_sep=',')
         num_lines = 0
         got_exception = False
         try:
@@ -98,22 +110,26 @@ def test_basic_example_d_wrong_type(self):
             got_exception = True
         self.assertTrue(got_exception)
         self.assertEqual(0, num_lines)
+        csv_file.close()
 
     def test_basic_example_d_tabs(self):
         """Test loading of a simple CSV file with tab (default) separators
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.e.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file)
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file)
         num_lines = 0
         for _ in test_file:
             num_lines += 1
         self.assertEqual(2, num_lines)
+        csv_file.close()
 
     def test_basic_example_d_too_many_values(self):
         """Test loading of a simple CSV file with too many values
         """
         test_file = os.path.join(DATA_DIR, 'TypedCsvReader.example.f.csv')
-        test_file = TypedColumnReader.TypedColumnReader(test_file, column_sep=',')
+        csv_file = open(test_file)
+        test_file = TypedColumnReader.TypedColumnReader(csv_file, column_sep=',')
         num_lines = 0
         got_exception = False
         try:
@@ -126,3 +142,4 @@ def test_basic_example_d_too_many_values(self):
             got_exception = True
         self.assertTrue(got_exception)
         self.assertEqual(0, num_lines)
+        csv_file.close()