merged changes from v1.1.2

Author: jlwalke2

CHANGELOG.md

@@ -7,7 +7,17 @@ Unreleased
 
 **Bugfixes**
 - Fixed an issue where `model_repository` did not find models by name once pagination limits were reached. 
-
+ 
+v1.1.2 (2019-8-12)
+-----------------
+**Improvements**
+ - CAS model table automatically reloaded on `publish_model` task.
+ 
+**Bugfixes**
+ - Fixed DS2 score code for CAS that was generated when registering a Python model.
+ - `PyMAS.score_code(dest='ESP')` corrected to `dest='EP'`
+ - Fixed an issue where long user-defined properties prevented model registration.
+ 
 
 v1.1.1 (2019-8-6)
 -----------------

src/sasctl/__init__.py

@@ -6,8 +6,8 @@
 
 __version__ = '1.2.0'
 __author__ = 'SAS'
-__credits__ = ['Yi Jian Ching, Lucas De Paula, Peter Tobac, Chris Toth, Jon '
-               'Walker']
+__credits__ = ['Yi Jian Ching, Lucas De Paula, James Kochuba, Peter Tobac, '
+               'Chris Toth, Jon Walker']
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright © 2019, SAS Institute Inc., ' \
                 'Cary, NC, USA.  All Rights Reserved.'

src/sasctl/_services/model_management.py

@@ -20,7 +20,30 @@ class ModelManagement(Service):
                             'performance tasks')
 
     # TODO:  set ds2MultiType
-    def publish_model(self, model, destination, name=None, force=False):
+    def publish_model(self,
+                      model,
+                      destination,
+                      name=None,
+                      force=False,
+                      reload_model_table=False):
+        """
+
+        Parameters
+        ----------
+        model
+        destination
+        name
+        force : bool, optional
+            Whether to overwrite the model if it already exists in the
+            publishing `destination`.
+        reload_model_table : bool, optional
+            Whether the model table in CAS should be reloaded.  Defaults to
+            False.
+
+        Returns
+        -------
+
+        """
         from .model_repository import ModelRepository
         from .model_publish import ModelPublish
 
@@ -37,7 +60,8 @@ def publish_model(self, model, destination, name=None, force=False):
 
         # TODO: Verify allowed formats by destination type.
         # As of 19w04 MAS throws HTTP 500 if name is in invalid format.
-        model_name = name or '{}_{}'.format(model_obj['name'].replace(' ', ''), model_obj['id']).replace('-', '')
+        model_name = name or '{}_{}'.format(model_obj['name'].replace(' ', ''),
+                                            model_obj['id']).replace('-', '')
 
         request = {
             "name": model_obj.get('name'),
@@ -56,8 +80,12 @@ def publish_model(self, model, destination, name=None, force=False):
         # Publishes a model that has already been registered in the model
         # repository.
         # Unlike model_publish service, does not require Code to be specified.
-        r = self.post('/publish', json=request, params=dict(force=force),
-                      headers={'Content-Type': 'application/vnd.sas.models.publishing.request.asynchronous+json'})
+        r = self.post('/publish',
+                      json=request,
+                      params=dict(force=force,
+                                  reloadModelTable=reload_model_table),
+                      headers={'Content-Type':
+                                   'application/vnd.sas.models.publishing.request.asynchronous+json'})
         return r
 
     def create_performance_definition(self,

src/sasctl/tasks.py

@@ -25,27 +25,43 @@
 
 
 def _sklearn_to_dict(model):
+    # As of Viya 3.4 model registration fails if character fields are longer
+    # than 1024 characters
+    DESC_MAXLEN = 1024
+
+    # As of Viya 3.4 model registration fails if user-defined properties are
+    # longer than 512 characters.
+    PROP_MAXLEN = 512
+
     # Convert Scikit-learn values to built-in Model Manager values
     mappings = {'LogisticRegression': 'Logistic regression',
                 'LinearRegression': 'Linear regression',
                 'SVC': 'Support vector machine',
                 'GradientBoostingClassifier': 'Gradient boosting',
+                'XGBClassifier': 'Gradient boosting',
+                'XGBRegressor': 'Gradient boosting',
                 'RandomForestClassifier': 'Forest',
                 'DecisionTreeClassifier': 'Decision tree',
                 'DecisionTreeRegressor': 'Decision tree',
                 'classifier': 'Classification',
                 'regressor': 'Prediction'}
 
+    if hasattr(model, '_final_estimator'):
+        estimator = type(model._final_estimator)
+    else:
+        estimator = type(model)
+
     # Can tell if multi-class .multi_class
     result = dict(
-        description=str(model),
-        algorithm=mappings.get(type(model).__name__, type(model).__name__),
+        description=str(model)[:DESC_MAXLEN],
+        algorithm=mappings.get(estimator.__name__, estimator.__name__),
         scoreCodeType='ds2MultiType',
         trainCodeType='Python',
         function=mappings.get(model._estimator_type, model._estimator_type),
         tool='Python %s.%s'
              % (sys.version_info.major, sys.version_info.minor),
-        properties=[{'name': k, 'value': v}
+        properties=[{'name': str(k)[:PROP_MAXLEN],
+                     'value': str(v)[:PROP_MAXLEN]}
                     for k, v in model.get_params().items()]
     )
 
@@ -198,9 +214,9 @@ def get_version(x):
             files.append({'name': 'dmcas_packagescorecode.sas',
                           'file': mas_module.score_code(),
                           'role': 'Score Code'})
-            files.append({'name': 'dmcas_espscorecode.sas',
-                          'file': mas_module.score_code(dest='ESP'),
-                          'role': 'Score Code'})
+            files.append({'name': 'dmcas_epscorecode.sas',
+                          'file': mas_module.score_code(dest='CAS'),
+                          'role': 'score'})
 
             model['inputVariables'] = [var.as_model_metadata()
                                        for var in mas_module.variables
@@ -245,7 +261,10 @@ def get_version(x):
     return model
 
 
-def publish_model(model, destination, code=None, max_retries=60,
+def publish_model(model,
+                  destination,
+                  code=None,
+                  max_retries=60,
                   replace=False, **kwargs):
     """Publish a model to a configured publishing destination.
 
@@ -290,7 +309,15 @@ def publish_model(model, destination, code=None, max_retries=60,
     def submit_request():
         # Submit a publishing request
         if code is None:
-            publish_req = mm.publish_model(model, destination, **kwargs)
+            dest_obj = mp.get_destination(destination)
+
+            if dest_obj and dest_obj.destinationType == "cas":
+                publish_req = mm.publish_model(model, destination,
+                                               force=replace,
+                                               reload_model_table=True)
+            else:
+                publish_req = mm.publish_model(model, destination,
+                                               force=replace)
         else:
             publish_req = mp.publish_model(model, destination,
                                            code=code, **kwargs)
@@ -304,6 +331,11 @@ def submit_request():
     # Submit and wait for status
     job = submit_request()
 
+    # If model was successfully published and it isn't a MAS module, we're done
+    if job.state.lower() == 'completed' \
+            and job.destination.destinationType != 'microAnalyticService':
+            return request_link(job,'self')
+
     # If MAS publish failed and replace=True, attempt to delete the module
     # and republish
     if job.state.lower() == 'failed' and replace and \

src/sasctl/utils/pymas/core.py

@@ -345,7 +345,7 @@ def score_code(self, input_table=None, output_table=None, columns=None, dest='MA
             The name of the table where execution results will be written
         columns : list of str
             Names of the columns from `table` that will be passed to `func`
-        dest : str {'MAS', 'ESP', 'CAS'}
+        dest : str {'MAS', 'EP', 'CAS'}
 
         Returns
         -------
@@ -365,28 +365,24 @@ def score_code(self, input_table=None, output_table=None, columns=None, dest='MA
         # Get package code
         code = tuple(self.package.code().split('\n'))
 
-        if dest == 'ESP':
+        if dest == 'EP':
             code = ('data sasep.out;', ) + code + ('   method run();',
                                                    '      set SASEP.IN;',
                                                    '   end;',
                                                    '   method term();',
                                                    '   end;',
                                                    'enddata;')
         elif dest == 'CAS':
-            if input_table is None:
-                raise ValueError('An input table must be specified when executing the code in CAS.')
-
-            output_table = output_table or input_table + '_pymas'
             thread = DS2Thread(self.variables, input_table, column_names=columns, return_message=self.return_message,
                                package=self.package)
 
             code += (str(thread),
-                     '    data {} (overwrite=yes);'.format(output_table),
-                     '    dcl thread {} t;'.format(thread.name),
-                     '    method run();',
-                     '       set from t;',
-                     '       output {} ;'.format(output_table),
-                     '    end;',
-                     '  enddata;')
-
-        return '\n'.join(code)
+                     'data SASEP.out;',
+                     '  dcl thread {} t;'.format(thread.name),
+                     '  method run();',
+                     '    set from t;',
+                     '    output;',
+                     '  end;',
+                     'enddata;')
+
+        return '\n'.join(code)

src/sasctl/utils/pymas/ds2.py

@@ -252,9 +252,6 @@ def name(self):
         return 'pyMasThread'
 
     def __str__(self):
-        extras = ['"rc"'] if self.return_code else []
-        extras += ['"msg"'] if self.return_message else []
-
         array_input = any([v.is_array for v in self.variables if not v.out])
 
         # If passing column data into Python as an array, need extra assignment statements to set values
@@ -272,25 +269,18 @@ def __str__(self):
         declarations = '\n'.join([v.as_declaration() for v in self.variables if
                                   v.out or v.is_array])
 
-        keep_vars = [v.name for v in self.variables] + extras
+        keep_vars = [v.name for v in self.variables]
 
-        code = ("        thread {} / inline;".format(self.name),
-                "            dcl package {} decisionPackage();".format(
-                    self.package.name),
+        code = ("thread {} / inline;".format(self.name),
+                "  dcl package {} pythonPackage();".format(self.package.name),
                 declarations,
-                '            dcl double "rc";',
-                "            dcl char(4096) character set utf8 msg;",
-                # "            keep {};".format(' '.join(keep_vars)),
-                "            method run();",
-                "                dcl integer localRC;",
-                "                set {} ( );".format(self.table),
+                "  method run();",
+                "    set SASEP.in;",
                 var_assignments,
-                "                decisionPackage.{}({});".format(
-                    self.package.methods[0].name,
-                    ','.join(keep_vars)),
-                "                output;",
-                "            end;",
-                "        endthread;")
+                "    pythonPackage.{}({});".format(self.package.methods[0].name,','.join(keep_vars)),
+                "    output;",
+                "  end;",
+                "endthread;")
 
         code = '\n'.join(code)