pymas fix for python files

Author: jlwalke2

CHANGELOG.md

@@ -1,6 +1,6 @@
 Unreleased
 ----------
- - 
+ - Fixed PyMAS utilities to correctly work functions not bound to pickled objects. 
 
 v1.5 (2020-2-23)
 ----------------

src/sasctl/utils/misc.py

@@ -4,6 +4,11 @@
 # Copyright © 2019, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import random
+import string
+
+from .decorators import versionadded
+
 
 def installed_packages():
     """List Python packages installed in the current environment.
@@ -25,4 +30,21 @@ def installed_packages():
             freeze = None
 
     if freeze is not None:
-        return list(freeze.freeze())
+        return list(freeze.freeze())
+
+
+@versionadded(version='1.5.1')
+def random_string(length):
+    """Generates a random alpha-numeric string of a given length.
+
+    Parameters
+    ----------
+    length : int
+        The length of the generate string.
+
+    Returns
+    -------
+    str
+
+    """
+    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))

src/sasctl/utils/pymas/core.py

@@ -9,17 +9,17 @@
 from __future__ import print_function
 import base64
 import importlib
-import pickle  # skipcq BAN-B301
+import pickle                                                                                          # skipcq BAN-B301
 import os
 import re
 import sys
 from collections import OrderedDict
-
 import six
 
 from .ds2 import DS2Thread, DS2Variable, DS2PyMASPackage
 from .python import ds2_variables
 from ..decorators import versionadded, versionchanged
+from ..misc import random_string
 
 
 @versionchanged(reason='Added `name` parameter.', version='1.5')
@@ -88,19 +88,18 @@ def build_wrapper_function(func, variables, array_input,
                 string_input += ("        if {0}: {0} = {0}.strip()".format(v.name), )
             else:
                 string_input += (
-                "        if {0} is None: {0} = np.nan".format(v.name),)
-
+                 "        if {0} is None: {0} = np.nan".format(v.name),)
 
     # Statement to execute the function w/ provided parameters
     if array_input:
         middle = string_input + \
                  (
-                 '        input_array = np.array([{}]).reshape((1, -1))'.format(
+                  '        input_array = np.array([{}]).reshape((1, -1))'.format(
                      ', '.join(args)),
-                 '        columns = [{}]'.format(
+                  '        columns = [{}]'.format(
                      ', '.join('"{0}"'.format(w) for w in args)),
-                 '        input_df = pd.DataFrame(data=input_array, columns=columns)',
-                 '        result = {}(input_df)'.format(func))
+                  '        input_df = pd.DataFrame(data=input_array, columns=columns)',
+                  '        result = {}(input_df)'.format(func))
     else:
         func_call = '{}({})'.format(func, ','.join(args))
         middle = ('        result = {}'.format(func_call),)
@@ -131,8 +130,8 @@ def build_wrapper_function(func, variables, array_input,
                   '        import pandas as pd') + \
                  middle + \
                  (
-                 '        result = tuple(result.ravel()) if hasattr(result, '
-                 '"ravel") else tuple(result)',
+                  '        result = tuple(result.ravel()) if hasattr(result, '
+                  '"ravel") else tuple(result)',
                   '        if len(result) == 0:',
                   '            result = tuple(None for i in range(%s))' % len(
                       output_names),
@@ -142,9 +141,9 @@ def build_wrapper_function(func, variables, array_input,
                   '        from traceback import format_exc',
                   '        msg = str(e) + format_exc()' if return_msg else '',
                   '        if result is None:',
-                 '            result = tuple(None for i in range(%s))' % len(
+                  '            result = tuple(None for i in range(%s))' % len(
                      output_names),
-                 '    return result + (msg, )')
+                  '    return result + (msg, )')
 
     return '\n'.join(definition)
 
@@ -158,7 +157,7 @@ def wrap_predict_method(func, variables, **kwargs):
     func : function or str
         Function name or an instance of Function which will be wrapped.  Assumed
         to behave as `.predict()` methods.
-     variables : list of DS2Variable
+    variables : list of DS2Variable
         Input and output variables for the function
     kwargs : any
         Will be passed to `build_wrapper_function`.
@@ -188,7 +187,7 @@ def wrap_predict_proba_method(func, variables, **kwargs):
     func : function or str
         Function name or an instance of Function which will be wrapped.  Assumed
         to behave as `.predict_proba()` methods.
-     variables : list of DS2Variable
+    variables : list of DS2Variable
         Input and output variables for the function
     kwargs : any
         Will be passed to `build_wrapper_function`.
@@ -312,8 +311,7 @@ def from_python_file(file, func_name=None, input_types=None, array_input=False,
     with open(file, 'r') as f:
         code = [line.strip('\n') for line in f.readlines()]
 
-    return _build_pymas(target_func, None, input_types, array_input,
-                        return_code, return_message, code)
+    return _build_pymas(target_func, None, input_types, array_input, code=code)
 
 
 @versionchanged('Return code and message are disabled by default.', version='1.5')
@@ -382,11 +380,12 @@ def from_pickle(file, func_name=None, input_types=None, array_input=False,
             'bytes = {}'.format(pkl).replace("'", '"'),
             'obj = pickle.loads(base64.b64decode(bytes))')
 
-    return _build_pymas(obj, func_name, input_types, array_input, return_code,
-                        return_message, code)
+    return _build_pymas(obj, func_name, input_types, array_input,
+                        func_prefix='obj.', code=code)
 
 
 def _build_pymas(obj, func_name=None, input_types=None, array_input=False,
+                 func_prefix=None,
                  return_code=None, return_message=None, code=None):
     """
 
@@ -465,7 +464,7 @@ def parse_function(obj, func_name):
     variables = list(variables)
 
     return PyMAS(target_func, variables, code,
-                 array_input=array_input, func_prefix='obj.')
+                 array_input=array_input, func_prefix=func_prefix)
 
 
 class PyMAS:
@@ -521,17 +520,23 @@ def __init__(self,
             return_msg = [return_msg] * len(target_function)
 
         self.wrapper = []
+        wrapper_names = []
         for func, vars, code, msg in zip(target_function, variables, return_code, return_msg):
+            wrapper_names.append('_' + random_string(20))
+
             if func.lower() == 'predict':
                 lines = wrap_predict_method(func_prefix + func, vars,
-                                            setup=python_source, **kwargs)
+                                            setup=python_source,
+                                            name=wrapper_names[-1],
+                                            **kwargs)
             elif func.lower() == 'predict_proba':
                 lines = wrap_predict_proba_method(func_prefix + func, vars,
+                                                  name=wrapper_names[-1],
                                                   setup=python_source, **kwargs)
             else:
                 lines = build_wrapper_function(func_prefix+func,
                                                vars,
-                                               name=func,
+                                               name=wrapper_names[-1],
                                                setup=python_source,
                                                **kwargs)
 
@@ -556,7 +561,8 @@ def __init__(self,
         self.package = DS2PyMASPackage(self.wrapper)
 
         for idx, func in enumerate(target_function):
-            self.package.add_method(func, func,
+            self.package.add_method(func,
+                                    wrapper_names[idx],
                                     variables[idx])
 
     @versionchanged(version='1.4', reason="Added `dest='Python'` option")

tests/integration/test_pymas.py

@@ -140,12 +140,14 @@ def test_from_pickle(train_data, pickle_file):
 
     X, y = train_data
 
-    with mock.patch('uuid.uuid4') as mocked:
-        mocked.return_value.hex = 'DF74A4B18C9E41A2A34B0053E123AA67'
-        p = from_pickle(pickle_file,
-                        func_name='predict',
-                        input_types=X,
-                        array_input=True)
+    with mock.patch('sasctl.utils.pymas.core.random_string') as mock_rnd_string:
+        mock_rnd_string.return_value = 'randomMethodName'
+        with mock.patch('uuid.uuid4') as mocked:
+            mocked.return_value.hex = 'DF74A4B18C9E41A2A34B0053E123AA67'
+            p = from_pickle(pickle_file,
+                            func_name='predict',
+                            input_types=X,
+                            array_input=True)
 
     target = """
 package _DF74A4B18C9E41A2A34B0053E123AA6 / overwrite=yes;
@@ -169,7 +171,7 @@ def test_from_pickle(train_data, pickle_file):
                 rc = py.appendSrcLine('except Exception as e:');
                 rc = py.appendSrcLine('    _compile_error = e');
                 rc = py.appendSrcLine('');
-                rc = py.appendSrcLine('def predict(SepalLength, SepalWidth, PetalLength, PetalWidth):');
+                rc = py.appendSrcLine('def _randomMethodName(SepalLength, SepalWidth, PetalLength, PetalWidth):');
                 rc = py.appendSrcLine('    "Output: var1, msg"');
                 rc = py.appendSrcLine('    result = None');
                 rc = py.appendSrcLine('    msg = None');
@@ -219,7 +221,7 @@ def test_from_pickle(train_data, pickle_file):
     
         dcl integer rc;
         dcl varchar(4068) msg;
-        rc = py.useMethod('predict');
+        rc = py.useMethod('_randomMethodName');
         if rc then return;
         rc = py.setDouble('SepalLength', SepalLength);    if rc then return;
         rc = py.setDouble('SepalWidth', SepalWidth);    if rc then return;
@@ -251,10 +253,12 @@ def test_from_pickle_2(train_data, pickle_file):
 
     X, _ = train_data
 
-    with mock.patch('uuid.uuid4') as mocked:
-        mocked.return_value.hex = 'DF74A4B18C9E41A2A34B0053E123AA67'
-        p = from_pickle(pickle_file, func_name=['predict', 'predict_proba'],
-                        input_types=X, array_input=True)
+    with mock.patch('sasctl.utils.pymas.core.random_string') as mock_rnd_string:
+        mock_rnd_string.side_effect = ['randomMethodName1', 'randomMethodName2']
+        with mock.patch('uuid.uuid4') as mocked:
+            mocked.return_value.hex = 'DF74A4B18C9E41A2A34B0053E123AA67'
+            p = from_pickle(pickle_file, func_name=['predict', 'predict_proba'],
+                            input_types=X, array_input=True)
 
     target = """
 package _DF74A4B18C9E41A2A34B0053E123AA6 / overwrite=yes;
@@ -278,7 +282,7 @@ def test_from_pickle_2(train_data, pickle_file):
                 rc = py.appendSrcLine('except Exception as e:');
                 rc = py.appendSrcLine('    _compile_error = e');
                 rc = py.appendSrcLine('');
-                rc = py.appendSrcLine('def predict(SepalLength, SepalWidth, PetalLength, PetalWidth):');
+                rc = py.appendSrcLine('def _randomMethodName1(SepalLength, SepalWidth, PetalLength, PetalWidth):');
                 rc = py.appendSrcLine('    "Output: var1, msg"');
                 rc = py.appendSrcLine('    result = None');
                 rc = py.appendSrcLine('    msg = None');
@@ -309,7 +313,7 @@ def test_from_pickle_2(train_data, pickle_file):
                 rc = py.appendSrcLine('            result = tuple(None for i in range(1))');
                 rc = py.appendSrcLine('    return result + (msg, )');
                 rc = py.appendSrcLine('');
-                rc = py.appendSrcLine('def predict_proba(SepalLength, SepalWidth, PetalLength, PetalWidth):');
+                rc = py.appendSrcLine('def _randomMethodName2(SepalLength, SepalWidth, PetalLength, PetalWidth):');
                 rc = py.appendSrcLine('    "Output: P_1, P_2, P_3, msg"');
                 rc = py.appendSrcLine('    result = None');
                 rc = py.appendSrcLine('    msg = None');
@@ -359,7 +363,7 @@ def test_from_pickle_2(train_data, pickle_file):
     
         dcl integer rc;
         dcl varchar(4068) msg;
-        rc = py.useMethod('predict');
+        rc = py.useMethod('_randomMethodName1');
         if rc then return;
         rc = py.setDouble('SepalLength', SepalLength);    if rc then return;
         rc = py.setDouble('SepalWidth', SepalWidth);    if rc then return;
@@ -383,7 +387,7 @@ def test_from_pickle_2(train_data, pickle_file):
     
         dcl integer rc;
         dcl varchar(4068) msg;
-        rc = py.useMethod('predict_proba');
+        rc = py.useMethod('_randomMethodName2');
         if rc then return;
         rc = py.setDouble('SepalLength', SepalLength);    if rc then return;
         rc = py.setDouble('SepalWidth', SepalWidth);    if rc then return;
@@ -419,11 +423,99 @@ def test_from_pickle_stream(train_data, pickle_stream):
     assert isinstance(p, PyMAS)
 
 
-def test_from_python_file(python_file):
-    from sasctl.utils.pymas import PyMAS, from_python_file
+def test_from_python_file(tmpdir):
+    from sasctl.utils.pymas import from_python_file
 
-    p = from_python_file(python_file, func_name='predict')
-    assert isinstance(p, PyMAS)
+    code = """
+def hello_world():
+    print('Hello World!')
+
+"""
+
+    target = """
+package _DF74A4B18C9E41A2A34B0053E123AA6 / overwrite=yes;
+    dcl package pymas py;
+    dcl package logger logr('App.tk.MAS');
+    dcl varchar(67108864) character set utf8 pycode;
+    dcl int revision;
+
+    method init();
+    
+        dcl integer rc;
+        if null(py) then do;
+            py = _new_ pymas();
+            rc = py.useModule('DF74A4B18C9E41A2A34B0053E123AA67', 1);
+            if rc then do;
+                rc = py.appendSrcLine('try:');
+                rc = py.appendSrcLine('    ');
+                rc = py.appendSrcLine('    def hello_world():');
+                rc = py.appendSrcLine('        print('Hello World!')');
+                rc = py.appendSrcLine('    ');
+                rc = py.appendSrcLine('    _compile_error = None');
+                rc = py.appendSrcLine('except Exception as e:');
+                rc = py.appendSrcLine('    _compile_error = e');
+                rc = py.appendSrcLine('');
+                rc = py.appendSrcLine('def _randomMethodName():');
+                rc = py.appendSrcLine('    "Output: result, msg"');
+                rc = py.appendSrcLine('    result = None');
+                rc = py.appendSrcLine('    msg = None');
+                rc = py.appendSrcLine('    try:');
+                rc = py.appendSrcLine('        global _compile_error');
+                rc = py.appendSrcLine('        if _compile_error is not None:');
+                rc = py.appendSrcLine('            raise _compile_error');
+                rc = py.appendSrcLine('        import numpy as np');
+                rc = py.appendSrcLine('        import pandas as pd');
+                rc = py.appendSrcLine('        result = hello_world()');
+                rc = py.appendSrcLine('        result = tuple(result.ravel()) if hasattr(result, "ravel") else tuple(result)');
+                rc = py.appendSrcLine('        if len(result) == 0:');
+                rc = py.appendSrcLine('            result = tuple(None for i in range(1))');
+                rc = py.appendSrcLine('        elif "numpy" in str(type(result[0])):');
+                rc = py.appendSrcLine('            result = tuple(np.asscalar(i) for i in result)');
+                rc = py.appendSrcLine('    except Exception as e:');
+                rc = py.appendSrcLine('        from traceback import format_exc');
+                rc = py.appendSrcLine('        msg = str(e) + format_exc()');
+                rc = py.appendSrcLine('        if result is None:');
+                rc = py.appendSrcLine('            result = tuple(None for i in range(1))');
+                rc = py.appendSrcLine('    return result + (msg, )');
+                pycode = py.getSource();
+                revision = py.publish(pycode, 'DF74A4B18C9E41A2A34B0053E123AA67');
+                if revision lt 1 then do;
+                    logr.log('e', 'py.publish() failed.');
+                    rc = -1;
+                end;
+            end;
+        end;
+    end;
+    
+    method hello_world(
+        in_out double result
+        );
+    
+        dcl integer rc;
+        dcl varchar(4068) msg;
+        rc = py.useMethod('_randomMethodName');
+        if rc then return;
+        rc = py.execute();    if rc then return;
+        result = py.getDouble('result');
+        msg = py.getString('msg');
+        if not null(msg) then logr.log('e', 'Error executing Python function "hello_world": $s', msg);
+    end;
+    
+endpackage;
+""".lstrip('\n')
+
+    f = tmpdir.join('model.py')
+    f.write(code)
+
+    with mock.patch('sasctl.utils.pymas.core.random_string') as mock_rnd_string:
+        mock_rnd_string.return_value = 'randomMethodName'
+        with mock.patch('uuid.uuid4') as mocked:
+            mocked.return_value.hex = 'DF74A4B18C9E41A2A34B0053E123AA67'
+            p = from_python_file(str(f), 'hello_world')
+
+    result = p.score_code()
+
+    assert result == target
 
 
 def test_with_sklearn_pipeline(train_data, sklearn_pipeline):