Add integer tests and options for setting missing values in integer-based columns

Author: Kevin D Smith

swat/cas/datamsghandlers.py

@@ -28,6 +28,7 @@
 import datetime
 import numpy as np
 import pandas as pd
+import warnings
 from .utils.datetime import (str2cas_timestamp, str2cas_datetime, str2cas_date,
                              str2cas_time, str2sas_timestamp, str2sas_datetime,
                              str2sas_date, str2sas_time, cas2python_timestamp,
@@ -40,6 +41,7 @@
                              python2sas_date, python2sas_time, python2cas_timestamp,
                              python2cas_datetime, python2cas_date, python2cas_time)
 from .. import clib
+from ..config import get_option
 from ..clib import errorcheck
 from ..exceptions import SWATError
 from ..utils.compat import a2n, text_types, binary_types, int32, int64, float64
@@ -333,6 +335,13 @@ def get(arr, idx, default=0):
                     errorcheck(self._sw_databuffer.setString(row, offset, a2n('')),
                                self._sw_databuffer)
             elif vrtype == 'NUMERIC' and vtype in ['INT32', 'DATE']:
+                if pd.isnull(value):
+                    value = get_option('cas.%s_missing' % vtype.lower())
+                    warnings.warn(("Missing value found in 32-bit integer-based column '%s'.\n" %
+                                   v['name']) +
+                                  ("Substituting cas.%s_missing option value (%s)." %
+                                   (vtype.lower(), value)),
+                                  RuntimeWarning)
                 if length > 4:
                     for i in range(int64(length / 4)):
                         errorcheck(self._sw_databuffer.setInt32(row, offset + (i * 4),
@@ -343,6 +352,13 @@ def get(arr, idx, default=0):
                                                             int32(transformer(value))),
                                self._sw_databuffer)
             elif vrtype == 'NUMERIC' and vtype in ['INT64', 'DATETIME', 'TIME']:
+                if pd.isnull(value):
+                    value = get_option('cas.%s_missing' % vtype.lower())
+                    warnings.warn(("Missing value found in 64-bit integer-based column '%s'.\n"
+                                   % v['name']) +
+                                  ("Substituting cas.%s_missing option value (%s)." %
+                                   (vtype.lower(), value)),
+                                  RuntimeWarning)
                 if length > 8:
                     for i in range(int64(length / 8)):
                         errorcheck(self._sw_databuffer.setInt64(row, offset + (i * 8),

swat/config.py

@@ -136,6 +136,25 @@ def check_severity(sev):
                 '1 would raise exceptions on warnings.  2 would raise exceptions\n' +
                 'on errors.')
 
+#
+# Integer missing value substitutions
+#
+
+register_option('cas.int64_missing', 'int', check_int, -2**(64-1),
+                'Sets substitution value for int64 missing values.')
+
+register_option('cas.int32_missing', 'int', check_int, -2**(32-1),
+                'Sets substitution value for int32 missing values.')
+
+register_option('cas.date_missing', 'int', check_int, -2**(32-1),
+                'Sets substitution value for date missing values.')
+
+register_option('cas.time_missing', 'int', check_int, -2**(64-1),
+                'Sets substitution value for time missing values.')
+
+register_option('cas.datetime_missing', 'int', check_int, -2**(64-1),
+                'Sets substitution value for datetime missing values.')
+
 #
 # Tabular data options
 #

swat/tests/cas/test_basics.py

@@ -52,6 +52,7 @@ def setUp(self):
         swat.reset_option()
         swat.options.cas.print_messages = False
         swat.options.interactive_mode = True
+        swat.options.int64_missing = -999999
 
         self.s = swat.CAS(HOST, PORT, USER, PASSWD, protocol=PROTOCOL)
 
@@ -412,7 +413,7 @@ def test_addtable(self):
         self.assertEqual(len(data), 15)
         self.assertEqual(data.ix[:,'Column'].tolist(),
                          'Make,Model,Type,Origin,DriveTrain,MSRP,Invoice,EngineSize,Cylinders,Horsepower,MPG_City,MPG_Highway,Weight,Wheelbase,Length'.split(','))
-        self.assertEqual(data.ix[:,'Type'].tolist(), (['varchar'] * 5) + (['double'] * 10))
+        self.assertEqual(data.ix[:,'Type'].tolist(), ['varchar', 'varchar', 'varchar', 'varchar', 'varchar', 'int64', 'int64', 'double', 'int64', 'int64', 'int64', 'int64', 'int64', 'int64', 'int64'])
 
         self.assertTablesEqual(cars, self.s.CASTable('cars', caslib=srcLib), sortby=SORT_KEYS)
 

swat/tests/cas/test_datamsg.py

@@ -52,6 +52,7 @@ def setUp(self):
         swat.reset_option()
         swat.options.cas.print_messages = False
         swat.options.interactive_mode = False
+        swat.options.int64_missing = -999999
 
         self.s = swat.CAS(HOST, PORT, USER, PASSWD, protocol=PROTOCOL)
 
@@ -104,7 +105,7 @@ def test_csv(self):
 
         self.assertEqual(len(data), 15)
         self.assertEqual(data.ix[:,'Column'].tolist(), 'Make,Model,Type,Origin,DriveTrain,MSRP,Invoice,EngineSize,Cylinders,Horsepower,MPG_City,MPG_Highway,Weight,Wheelbase,Length'.split(','))
-        self.assertEqual(data.ix[:,'Type'].tolist(), (['varchar'] * 5) + (['double'] * 10))
+        self.assertEqual(data.ix[:,'Type'].tolist(), ['varchar', 'varchar', 'varchar', 'varchar', 'varchar', 'int64', 'int64', 'double', 'int64', 'int64', 'int64', 'int64', 'int64', 'int64', 'int64'])
 
         self.assertTablesEqual(cars, self.s.CASTable('cars', caslib=srcLib), sortby=SORT_KEYS)
 

swat/tests/cas/test_table.py

@@ -59,6 +59,7 @@ def setUp(self):
         swat.options.cas.trace_actions = False
         swat.options.cas.trace_ui_actions = False
         swat.options.interactive_mode = False
+        swat.options.int64_missing = -999999
 
         self.s = swat.CAS(HOST, PORT, USER, PASSWD, protocol=PROTOCOL)
 

swat/tests/test_config.py

@@ -245,8 +245,11 @@ def test_doc(self):
 
     def test_suboptions(self):
         self.assertEqual(list(sorted(get_suboptions('cas').keys())), 
-                         ['dataset', 'exception_on_severity', 'hostname',
-                          'port', 'print_messages', 'protocol', 'trace_actions',
+                         ['dataset', 'date_missing', 'datetime_missing', 
+                          'exception_on_severity', 'hostname',
+                          'int32_missing', 'int64_missing',
+                          'port', 'print_messages', 'protocol',
+                          'time_missing', 'trace_actions',
                           'trace_ui_actions'])
 
         with self.assertRaises(SWATOptionError):