Merge pull request #2 from sassoftware/master

sync to master repo

Author: jameskochubasas

CHANGELOG.md

@@ -3,6 +3,27 @@ Unreleased
 ----------
  - 
 
+v1.2.3 (2019-8-23)
+------------------
+**Changes**
+ - `put` request will take an `item` parameter that's used to automatically populate headers for updates.  
+
+**Bugfixes**
+ - Convert NaN values to null (None) when calling `microanalytic_score.execute_module_step`.
+
+
+v1.2.2 (2019-8-21)
+------------------
+**Bugfixes**
+ - `register_model` task should now correctly identify columns when registering a Sci-kit pipeline.
+ 
+
+v1.2.1 (2019-8-20)
+------------------
+**Improvements**
+ - Added the ability for `register_model` to correctly handle CAS tables containing data step
+ score code.
+ 
 
 v1.2.0 (2019-8-16)
 ------------------

README.md

@@ -81,7 +81,7 @@ which can handle all necessary communication with the SAS Viya server:
 
 >>> with Session('example.com', authinfo=<authinfo file>):
 ...    model = lm.LogisticRegression()
-...    register_model('Sklearn Model', model, 'My Project')
+...    register_model(model, 'Sklearn Model', 'My Project')
 ```
 
 
@@ -165,7 +165,7 @@ Register a pure Python model in Model Manager:
 
 >>> with Session(host, authinfo=<authinfo file>):
 ...    model = lm.LogisticRegression()
-...    register_model('Sklearn Model', model, 'My Project')
+...    register_model(model, 'Sklearn Model', 'My Project')
 ```
 
 Register a CAS model in Model Manager:

src/sasctl/__init__.py

@@ -4,7 +4,7 @@
 # Copyright © 2019, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-__version__ = '1.2.0'
+__version__ = '1.2.3'
 __author__ = 'SAS'
 __credits__ = ['Yi Jian Ching, Lucas De Paula, James Kochuba, Peter Tobac, '
                'Chris Toth, Jon Walker']

src/sasctl/_services/microanalytic_score.py

@@ -8,6 +8,7 @@
 
 import re
 from collections import OrderedDict
+from math import isnan
 
 import six
 
@@ -127,9 +128,17 @@ def execute_module_step(self, module, step, return_dict=True, **kwargs):
             elif type_name == 'int64':
                 kwargs[k] = int(kwargs[k])
 
-
         body = {'inputs': [{'name': k, 'value': v}
                            for k, v in six.iteritems(kwargs)]}
+
+        # Convert NaN to None (null) before calling MAS
+        for input in body['inputs']:
+            try:
+                if isnan(input['value']):
+                    input['value']  = None
+            except TypeError:
+                pass
+
         r = self.post('/modules/{}/steps/{}'.format(module, step), json=body)
 
         # Convert list of name/value pair dictionaries to single dict

src/sasctl/core.py

@@ -632,30 +632,115 @@ def is_uuid(id):
         return False
 
 
-def get(*args, **kwargs):
+def get(path, **kwargs):
+    """Send a GET request.
+
+    Parameters
+    ----------
+    path : str
+        The path portion of the URL.
+    kwargs : any
+        Passed to `request`.
+
+    Returns
+    -------
+    RestObj or None
+        The results or None if the resource was not found.
+
+    """
     try:
-        return request('get', *args, **kwargs)
+        return request('get', path, **kwargs)
     except HTTPError as e:
         if e.code == 404:
             return None  # Resource not found
         else:
             raise e
 
 
-def head(*args, **kwargs):
-    return request('head', *args, **kwargs)
+def head(path, **kwargs):
+    """Send a HEAD request.
+
+    Parameters
+    ----------
+    path : str
+        The path portion of the URL.
+    kwargs : any
+        Passed to `request`.
+
+    Returns
+    -------
+    RestObj
 
+    """
+    return request('head', path, **kwargs)
 
-def post(*args, **kwargs):
-    return request('post', *args, **kwargs)
 
+def post(path, **kwargs):
+    """Send a POST request.
 
-def put(*args, **kwargs):
-    return request('put', *args, **kwargs)
+    Parameters
+    ----------
+    path : str
+        The path portion of the URL.
+    kwargs : any
+        Passed to `request`.
 
+    Returns
+    -------
+    RestObj
 
-def delete(*args, **kwargs):
-    return request('delete', *args, **kwargs)
+    """
+    return request('post', path, **kwargs)
+
+
+def put(path, item=None, **kwargs):
+    """Send a PUT request.
+
+    Parameters
+    ----------
+    path : str
+        The path portion of the URL.
+    item : RestObj, optional
+        A existing object to PUT.  If provided, ETag and Content-Type headers
+        will automatically be specified.
+    kwargs : any
+        Passed to `request`.
+
+    Returns
+    -------
+    RestObj
+
+    """
+    # If call is in the format put(url, RestObj), automatically fill in header
+    # information
+    if item is not None and isinstance(item, RestObj):
+        get_headers = getattr(item, '_headers', None)
+        if get_headers is not None:
+            # Update the headers param if it was specified
+            headers = kwargs.pop('headers', {})
+            headers.setdefault('If-Match', get_headers.get('etag'))
+            headers.setdefault('Content-Type', get_headers.get('content-type'))
+            return request('put', path, json=item, headers=headers)
+
+    return request('put', path, **kwargs)
+
+
+def delete(path, **kwargs):
+    """Send a DELETE request.
+
+    Parameters
+    ----------
+    path : str
+        The path portion of the URL.
+    kwargs : any
+        Passed to `request`.
+
+    Returns
+    -------
+    RestObj
+
+    """
+    return request('delete', path, **kwargs)
 
 
 def request(verb, path, session=None, raw=False, **kwargs):

src/sasctl/tasks.py

@@ -187,7 +187,7 @@ def get_version(x):
     # If model is a CASTable then assume it holds an ASTORE model.
     # Import these via a ZIP file.
     if 'swat.cas.table.CASTable' in str(type(model)):
-        zipfile = utils.create_package_from_astore(model)
+        zipfile = utils.create_package(model)
 
         if create_project:
             project = mr.create_project(project, repository)

src/sasctl/utils/__init__.py

@@ -4,6 +4,6 @@
 # Copyright © 2019, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from .astore import create_package_from_astore
+from .astore import create_package, create_package_from_astore
 
 

src/sasctl/utils/astore.py

@@ -12,12 +12,77 @@
 import uuid
 import zipfile
 
+import six
+
 try:
     import swat
 except ImportError:
     swat = None
 
 
+def create_package(table):
+    """Create an importable model package from a CAS table.
+
+    Parameters
+    ----------
+    table : swat.CASTable
+        The CAS table containing an ASTORE or score code.
+
+    Returns
+    -------
+    BytesIO
+        A byte stream representing a ZIP archive which can be imported.
+
+    See Also
+    --------
+    :meth:`model_repository.import_model_from_zip <.ModelRepository.import_model_from_zip>`
+
+    """
+    if swat is None:
+        raise RuntimeError(
+            "The 'swat' package is required to work with SAS models.")
+
+    assert isinstance(table, swat.CASTable)
+
+    if 'DataStepSrc' in table.columns:
+        return create_package_from_datastep(table)
+    else:
+        return create_package_from_astore(table)
+
+
+def create_package_from_datastep(table):
+    """Create an importable model package from a score code table.
+
+    Parameters
+    ----------
+    table : swat.CASTable
+        The CAS table containing the score code.
+
+    Returns
+    -------
+    BytesIO
+        A byte stream representing a ZIP archive which can be imported.
+
+    See Also
+    --------
+    :meth:`model_repository.import_model_from_zip <.ModelRepository.import_model_from_zip>`
+
+    """
+    assert 'DataStepSrc' in table.columns
+    sess = table.session.get_connection()
+
+    dscode = table.to_frame().loc[0, 'DataStepSrc']
+
+    file_metadata = [{'role': 'score', 'name': 'dmcas_scorecode.sas'}]
+
+    zip_file = _build_zip_from_files({
+        'fileMetadata.json': file_metadata,
+        'dmcas_scorecode.sas': dscode
+    })
+
+    return zip_file
+
+
 def create_package_from_astore(table):
     """Create an importable model package from an ASTORE.
 
@@ -81,45 +146,62 @@ def create_package_from_astore(table):
                         'uri': '/dataTables/dataSources/cas~fs~cas-shared-default~fs~ModelStore/tables/{}'.format(astore_filename),
                         'key': astore_key}]
 
+    zip_file = _build_zip_from_files({
+        'dmcas_packagescorecode.sas': '\n'.join(package_ds2),
+        'dmcas_epscorecode.sas': ep_ds2,
+        astore_filename: astore,
+        'ModelProperties.json': model_properties,
+        'fileMetadata.json': file_metadata,
+        'AstoreMetadata.json': astore_metadata,
+        'inputVar.json': input_vars,
+        'outputVar.json': output_vars
+    })
+
+    return zip_file
+
+
+def _build_zip_from_files(files):
+    """Create a ZIP file containing the provided files.
+
+    Parameters
+    ----------
+    files : dict
+        Dictionary of filename: content to be added to the .zip file.
+
+    Returns
+    -------
+    BytesIO
+        Byte stream representation of the .zip file.
+
+    """
     try:
         # Create a temp folder
         folder = tempfile.mkdtemp()
 
-        # Closure for easily adding JSON files
-        def json_file(data, filename):
-            filename = os.path.join(folder, filename)
-            with open(filename, 'w') as f:
-                json.dump(data, f, indent=1)
+        for k, v in six.iteritems(files):
+            filename = os.path.join(folder, k)
 
-        filename = os.path.join(folder, 'dmcas_packagescorecode.sas')
-        with open(filename, 'w') as f:
-            f.write('\n'.join(package_ds2))
+            # Write JSON file
+            if os.path.splitext(k)[-1].lower() == '.json':
+                with open(filename, 'w') as f:
+                    json.dump(v, f, indent=1)
+            else:
+                mode = 'wb' if isinstance(v, bytes) else 'w'
 
-        filename = os.path.join(folder, 'dmcas_epscorecode.sas')
-        with open(filename, 'w') as f:
-            f.write(ep_ds2)
-
-        filename = os.path.join(folder, astore_filename)
-        with open(filename, 'wb') as f:
-            f.write(astore)
-
-        json_file(model_properties, 'ModelProperties.json')
-        json_file(file_metadata, 'fileMetadata.json')
-        json_file(astore_metadata, 'AstoreMetadata.json')
-        json_file(input_vars, 'inputVar.json')
-        json_file(output_vars, 'outputVar.json')
+                with open(filename, mode) as f:
+                    f.write(v)
 
         files = os.listdir(folder)
 
         with zipfile.ZipFile(os.path.join(folder, 'model.zip'), 'w') as z:
             for file in files:
                 z.write(os.path.join(folder, file), file)
 
-        # Need to return the ZIP file data but also need to ensure the directory is cleaned up.
+        # Need to return the ZIP file data but also need to ensure the
+        # directory is cleaned up.
         # Read the bytes from disk and return an in memory "file".
         with open(os.path.join(folder, 'model.zip'), 'rb') as z:
             return io.BytesIO(z.read())
-
     finally:
         shutil.rmtree(folder)
 

src/sasctl/utils/pymas/core.py

@@ -276,7 +276,6 @@ def _build_pymas(obj, func_name=None, input_types=None, array_input=False,
         # Run one observation through the model and use the result to
         # determine output variables
         output = target_func(input_types.head(1))
-        # output = target_func(input_types.iloc[0, :].values.reshape((1, -1)))
         output_vars = ds2_variables(output, output_vars=True)
         vars.extend(output_vars)
     elif isinstance(input_types, type):