Use sockets to detect protocol

Author: Kevin D Smith

swat/cas/connection.py

@@ -396,21 +396,25 @@ def _detect_protocol(self, hostname, port, protocol=None):
         # Try to detect the proper protocol
         if protocol == 'auto':
 
-            from six.moves import urllib
+            import socket
 
 #           for ptype in ['http', 'https']:
             for ptype in ['http']:
                 try:
-                    req = urllib.request.Request('%s://%s:%s/cas' %
-                                                 (ptype, hostname, port))
-                    with urllib.request.urlopen(req) as res:
-                        pass
-                except urllib.error.HTTPError as err:
-                    protocol = ptype
-                    break
+                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                    sock.connect((hostname, port))
+                    sock.send(bytes('GET /cas HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nUser-Agent: Python-SWAT\r\nCache-Control: no-cache\r\n\r\n' % hostname, 'utf8'))
+
+                    if sock.recv(4).decode('utf-8').lower() == 'http':
+                        protocol = ptype
+                        break
+
                 except Exception as err:
                     pass
 
+                finally:
+                    sock.close()
+
             if protocol == 'auto':
                 protocol = 'cas'
 
@@ -1993,7 +1997,7 @@ def _read_any(self, _method_, *args, **kwargs):
 
         '''
         import pandas as pd
-        use_addtable = kwargs.pop('use_addtable', False) 
+        use_addtable = kwargs.pop('use_addtable', False)
         table, kwargs = self._get_table_args(**kwargs)
         dframe = getattr(pd, _method_)(*args, **kwargs)
         # REST doesn't support table.addtable
@@ -2457,7 +2461,7 @@ def read_html(self, io, casout=None, **kwargs):
         '''
         import pandas as pd
         from swat import datamsghandlers as dmh
-        use_addtable = kwargs.pop('use_addtable', False) 
+        use_addtable = kwargs.pop('use_addtable', False)
         out = []
         table, kwargs = self._get_table_args(casout=casout, **kwargs)
         for i, dframe in enumerate(pd.read_html(io, **kwargs)):

swat/cas/rest/table.py

@@ -24,6 +24,7 @@
 from __future__ import print_function, division, absolute_import, unicode_literals
 
 import base64
+import decimal
 import numpy as np
 import pandas as pd
 from ..utils.datetime import cas2python_date, cas2python_time, cas2python_datetime
@@ -286,17 +287,17 @@ def toTuples(self, errors, cas2python_datetime, cas2python_date,
                             outrow.append(base64.b64decode(item['data']+'=='))
                 # Check for datetime, date, time
                 elif dtype == 'datetime':
-                    if item == -9223372036854775808:
+                    if item < decimal.Decimal('-9223372036854775807.5'):
                         outrow.append(pd.NaT)
                     else:
                         outrow.append(cas2python_datetime(item))
                 elif dtype == 'date':
-                    if item == -2147483648:
+                    if item < decimal.Decimal('-2147483647.5'):
                         outrow.append(pd.NaT)
                     else:
                         outrow.append(cas2python_date(item))
                 elif dtype == 'time':
-                    if item == -9223372036854775808:
+                    if item < decimal.Decimal('-9223372036854775807.5'):
                         outrow.append(pd.NaT)
                     else:
                         outrow.append(cas2python_time(item))

swat/cas/transformers.py

@@ -461,6 +461,22 @@ def py2cas(soptions, _sw_error, **kwargs):
        CASValueList representation of Python object
 
     '''
+
+    def remove_unsupported(params):
+        ''' Remove any unsupported parameter types '''
+        for key in list(params.keys()):
+            if isinstance(params[key], dict_types):
+                remove_unsupported(params[key])
+            elif not isinstance(params[key], (binary_types, text_types, bool,
+                                              blob, int64_types, int32_types,
+                                              float64_types, items_types, ParamManager,
+                                              datetime.datetime, datetime.date,
+                                              datetime.time, type(nil))):
+                del params[key]
+
+    # In-place operation
+    remove_unsupported(kwargs)
+
     _sw_values = errorcheck(clib.SW_CASValueList(len(kwargs), a2n(soptions),
                                                  _sw_error), _sw_error)
 

swat/tests/cas/test_table.py

@@ -3162,6 +3162,7 @@ def test_read_html(self):
         else:
            self.assertEqual(set(tbl.dtypes.unique()), set(['double', 'int64', 'varchar']))
 
+    @unittest.skip('Need way to verify HDF installation')
     def test_read_hdf(self):
         df = self.get_cars_df(all_doubles=False)
 
@@ -3230,6 +3231,7 @@ def test_column_to_xarray(self):
 
         self.assertEqual(list(dfx.values)[:200], list(tblx.values)[:200])
 
+    @unittest.skip('Need way to verify HDF installation')
     def test_column_to_hdf(self):
         df = self.get_cars_df(all_doubles=False)
         tbl = self.table
@@ -3642,6 +3644,7 @@ def test_to_csv(self):
 
         os.remove(tmp.name)
 
+    @unittest.skip('Need way to verify HDF installation')
     def test_to_hdf(self):
         df = self.get_cars_df()
         tbl = self.table