Update unit and integration tests

Author: smlindauer

src/sasctl/pzmm/write_json_files.py

@@ -262,6 +262,7 @@ def write_model_properties_json(
         json_path: Union[str, Path, None] = None,
         model_desc: Optional[str] = None,
         model_algorithm: Optional[str] = None,
+        model_function: Optional[str] = None,
         modeler: Optional[str] = None,
         train_table: Optional[str] = None,
         properties: Optional[List[dict]] = None,
@@ -295,7 +296,9 @@ def write_model_properties_json(
         model_desc : str, optional
             User-defined model description. The default value is an empty string.
         model_algorithm : str, optional
-            User-defined model type. The default value is an empty string.
+            User-defined model algorithm name. The default value is an empty string.
+        model_function : str, optional
+            User-defined model function name. The default value is an empty string.
         modeler : str, optional
             User-defined value for the name of the modeler. The default value is an
             empty string.
@@ -372,11 +375,11 @@ def write_model_properties_json(
         output_json = {
             "name": model_name,
             "description": model_desc if model_desc else "",
-            "function": model_function if model_function else "",
             "scoreCodeType": "python",
             "trainTable": train_table if train_table else "",
             "trainCodeType": "Python",
             "algorithm": model_algorithm if model_algorithm else "",
+            "function": model_function if model_function else "",
             "targetVariable": target_variable if target_variable else "",
             "targetEvent": target_event if target_event else "",
             "targetLevel": target_level if target_level else "",
@@ -471,7 +474,7 @@ def write_file_metadata_json(
         dict_list = [
             {"role": "inputVariables", "name": INPUT},
             {"role": "outputVariables", "name": OUTPUT},
-            {"role": "score", "name": model_prefix + "Score.py"},
+            {"role": "score", "name": f"score_{model_prefix}.py"},
         ]
         if is_h2o_model:
             dict_list.append({"role": "scoreResource", "name": model_prefix + ".mojo"})
@@ -914,8 +917,8 @@ def check_for_data(
 
     @staticmethod
     def stat_dataset_to_dataframe(
-        data: Union[DataFrame, List[list], Type["numpy.array"]] = None,
-        target_value: Union[str, int, float, None] = None,
+        data: Union[DataFrame, List[list], Type["numpy.array"]],
+        target_value: Union[str, int, float] = None,
     ) -> DataFrame:
         """
         Convert the user supplied statistical dataset from either a pandas DataFrame,

src/sasctl/pzmm/write_score_code.py

@@ -3,7 +3,7 @@
 
 import re
 from pathlib import Path
-from typing import Union, Optional, Callable, List, Tuple, Any, Generator
+from typing import Union, Optional, Callable, List, Tuple, Any, Generator, Type
 from warnings import warn
 
 import pandas as pd
@@ -15,8 +15,6 @@
 MAS_CODE_NAME = "dmcas_packagescorecode.sas"
 CAS_CODE_NAME = "dmcas_epscorecode.sas"
 
-# TODO: add converter for any type of dataset (list, dataframe, numpy array)
-
 
 class ScoreCode:
     score_code: str = ""
@@ -161,7 +159,7 @@ def write_score_code(
         else:
             raise ValueError(
                 "Either the binary_string or the model_file_name argument needs to be"
-                "specified in order to generate the score code."
+                " specified in order to generate the score code."
             )
 
         # Add the core imports to the score code with the specified model serializer
@@ -176,16 +174,16 @@ def write_score_code(
         if model_id and not binary_string:
             model_load = cls._viya35_model_load(
                 model_id,
-                pickle_type,
                 model_file_name,
+                pickle_type=pickle_type,
                 mojo_model="mojo_model" in kwargs,
                 binary_h2o_model="binary_h2o_model" in kwargs,
             )
         # As above, but for SAS Viya 4 models
         elif not binary_string:
             model_load = cls._viya4_model_load(
-                pickle_type,
                 model_file_name,
+                pickle_type=pickle_type,
                 mojo_model="mojo_model" in kwargs,
                 binary_h2o_model="binary_h2o_model" in kwargs,
             )
@@ -195,7 +193,7 @@ def write_score_code(
         model_prefix = cls._check_valid_model_prefix(model_prefix)
 
         # Define the score function using the variables found in input_data
-        cls.score_code += f"def score{model_prefix}({', '.join(input_var_list)}):\n"
+        cls.score_code += f"def score_{model_prefix}({', '.join(input_var_list)}):\n"
 
         if not score_metrics:
             score_metrics = cls._determine_score_metrics(
@@ -234,9 +232,10 @@ def write_score_code(
             )
             # Include check for numpy values and a conversion operation as needed
             cls.score_code += (
-                f"{'':4}prediction = [val.item() if type(val)."
-                f"__module__ == np.__name__ else val for val in "
-                f"prediction]\n"
+                f"\n{'':4}# Check for numpy values and convert to a CAS readable "
+                f"representation\n"
+                f"{'':4}if isinstance(prediction, np.ndarray):\n"
+                f"{'':8}prediction = prediction.tolist()[0]\n\n"
             )
             cls._predictions_to_metrics(
                 score_metrics,
@@ -252,7 +251,7 @@ def write_score_code(
             )
 
         if score_code_path:
-            py_code_path = Path(score_code_path) / (model_prefix + "_score.py")
+            py_code_path = Path(score_code_path) / f"score_{model_prefix}.py"
             with open(py_code_path, "w") as py_file:
                 py_file.write(cls.score_code)
             if model_id and score_cas:
@@ -263,7 +262,7 @@ def write_score_code(
                     # noinspection PyUnboundLocalVariable
                     sas_file.write(cas_code)
         else:
-            output_dict = {model_prefix + "_score.py": cls.score_code}
+            output_dict = {f"score_{model_prefix}.py": cls.score_code}
             if model_id and score_cas:
                 # noinspection PyUnboundLocalVariable
                 output_dict[MAS_CODE_NAME] = mas_code
@@ -1000,19 +999,19 @@ def _binary_target(
                     f"{'':8}{metrics} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics}"
+                    f"{'':4}return {metrics}"
                 )
             # One return that is the classification
             elif len(returns) == 1 and returns[0]:
-                cls.score_code += f"\n\nreturn prediction"
+                cls.score_code += f"{'':4}return prediction"
             # One return that is a probability
             elif len(returns) == 1 and not returns[0]:
                 cls.score_code += (
                     f"{'':4}if prediction > {threshold}:\n"
                     f"{'':8}{metrics} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics}"
+                    f"{'':4}return {metrics}"
                 )
             # Two returns from the prediction method
             elif len(returns) == 2 and sum(returns) == 0:
@@ -1022,7 +1021,7 @@ def _binary_target(
                     f"{'':8}{metrics} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics}"
+                    f"{'':4}return {metrics}"
                 )
             # Classification and probability returned; return classification value
             elif len(returns) > 1 and sum(returns) == 1:
@@ -1041,7 +1040,7 @@ def _binary_target(
                     "score code should output the classification and probability for "
                     "the target event to occur."
                 )
-                cls.score_code += f"\n\nreturn prediction[1][0], prediction[1][2]"
+                cls.score_code += f"{'':4}return prediction[1][0], prediction[1][2]"
             # Calculate the classification; return the classification and probability
             elif sum(returns) == 0 and len(returns) == 1:
                 warn(
@@ -1054,7 +1053,7 @@ def _binary_target(
                     f"{'':8}{metrics[0]} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics[0]} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics[0]}, prediction"
+                    f"{'':4}return {metrics[0]}, prediction"
                 )
             # Calculate the classification; return the classification and probability
             elif sum(returns) == 0 and len(returns) == 2:
@@ -1068,11 +1067,11 @@ def _binary_target(
                     f"{'':8}{metrics[0]} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics[0]} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics[0]}, prediction[0]"
+                    f"{'':4}return {metrics[0]}, prediction[0]"
                 )
             # Return classification and probability value
             elif sum(returns) == 1 and len(returns) == 2:
-                cls.score_code += f"\n\nreturn prediction[0], prediction[1]"
+                cls.score_code += f"{'':4}return prediction[0], prediction[1]"
             elif sum(returns) == 1 and len(returns) == 3:
                 warn(
                     "Due to the ambiguity of the provided metrics and prediction return"
@@ -1082,17 +1081,17 @@ def _binary_target(
                 # Determine which return is the classification value
                 class_index = [i for i, x in enumerate(returns) if x][0]
                 if class_index == 0:
-                    cls.score_code += f"\n\nreturn prediction[0], prediction[1]"
+                    cls.score_code += f"{'':4}return prediction[0], prediction[1]"
                 else:
                     cls.score_code += (
-                        f"\n\nreturn prediction[{class_index}], prediction[0]"
+                        f"{'':4}return prediction[{class_index}], prediction[0]"
                     )
             else:
                 cls._invalid_predict_config()
         elif len(metrics) == 3:
             if h2o_model:
                 cls.score_code += (
-                    f"\n\nreturn prediction[1][0], prediction[1][1], prediction[1][2]"
+                    f"{'':4}return prediction[1][0], prediction[1][1], prediction[1][2]"
                 )
             elif sum(returns) == 0 and len(returns) == 1:
                 warn(
@@ -1105,7 +1104,7 @@ def _binary_target(
                     f"{'':8}{metrics[0]} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics[0]} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics[0]}, prediction, 1 - prediction"
+                    f"{'':4}return {metrics[0]}, prediction, 1 - prediction"
                 )
             elif sum(returns) == 0 and len(returns) == 2:
                 warn(
@@ -1118,24 +1117,24 @@ def _binary_target(
                     f"{'':8}{metrics[0]} = \"{target_values[0]}\"\n"
                     f"{'':4}else:\n"
                     f"{'':8}{metrics[0]} = \"{target_values[1]}\"\n\n"
-                    f"return {metrics[0]}, prediction[0], prediction[1]"
+                    f"{'':4}return {metrics[0]}, prediction[0], prediction[1]"
                 )
             # Find which return is the classification, then return probabilities
             elif sum(returns) == 1 and len(returns) == 2:
                 # Determine which return is the classification value
                 class_index = [i for i, x in enumerate(returns) if x][0]
                 if class_index == 0:
                     cls.score_code += (
-                        f"\n\nreturn prediction[0], prediction[1], 1 - prediction[1]"
+                        f"{'':4}return prediction[0], prediction[1], 1 - prediction[1]"
                     )
                 else:
                     cls.score_code += (
-                        f"\n\nreturn prediction[1], prediction[0], 1 - prediction[0]"
+                        f"{'':4}return prediction[1], prediction[0], 1 - prediction[0]"
                     )
             # Return all values from prediction method
             elif sum(returns) == 1 and len(returns) == 3:
                 cls.score_code += (
-                    f"\n\nreturn prediction[0], prediction[1], prediction[2]"
+                    f"{'':4}return prediction[0], prediction[1], prediction[2]"
                 )
             else:
                 cls._invalid_predict_config()
@@ -1181,20 +1180,20 @@ def _nonbinary_targets(
                     f"{'':4}target_values = {target_values}\n{'':4}"
                     f"{metrics} = target_values[prediction[1][1:]."
                     f"index(max(prediction[1][1:]))]\n{'':4}"
-                    f"return {metrics}"
+                    f"{'':4}return {metrics}"
                 )
             # One return that is the classification
             elif len(returns) == 1:
                 cls.score_code += f"{'':4}{metrics} = prediction\n\nreturn {metrics}"
             elif len(returns) == len(target_values):
                 cls.score_code += (
                     f"{'':4}target_values = {target_values}\n\n"
-                    f"return target_values[prediction.index(max(prediction))]"
+                    f"{'':4}return target_values[prediction.index(max(prediction))]"
                 )
             elif len(returns) == (len(target_values) + 1):
                 # Determine which return is the classification value
                 class_index = [i for i, x in enumerate(returns) if x][0]
-                cls.score_code += f"\n\nreturn prediction[{class_index}]"
+                cls.score_code += f"{'':4}return prediction[{class_index}]"
             else:
                 cls._invalid_predict_config()
         elif len(metrics) == 2:
@@ -1203,19 +1202,19 @@ def _nonbinary_targets(
                     f"{'':4}target_values = {target_values}\n{'':4}"
                     f"{metrics} = target_values[prediction[1][1:]."
                     f"index(max(prediction[1][1:]))]\n{'':4}"
-                    f"return {metrics}, max(prediction[1][1:])"
+                    f"{'':4}return {metrics}, max(prediction[1][1:])"
                 )
             elif len(returns) == len(target_values):
                 cls.score_code += (
                     f"{'':4}target_values = {target_values}\n\n"
-                    f"return target_values[prediction.index(max(prediction))], "
+                    f"{'':4}return target_values[prediction.index(max(prediction))], "
                     f"max(prediction)"
                 )
             elif len(returns) == (len(target_values) + 1):
                 # Determine which return is the classification value
                 class_index = [i for i, x in enumerate(returns) if x][0]
                 cls.score_code += (
-                    f"\n\nreturn prediction[{class_index}], "
+                    f"{'':4}return prediction[{class_index}], "
                     f"max(prediction[:{class_index}] + prediction[{class_index + 1}:])"
                 )
             else:
@@ -1224,25 +1223,25 @@ def _nonbinary_targets(
             if h2o_model:
                 if len(metrics) == len(target_values):
                     h2o_returns = [f"prediction[1][{i+1}]" for i in range(len(metrics))]
-                    cls.score_code += f"\n\nreturn {', '.join(h2o_returns)}"
+                    cls.score_code += f"{'':4}return {', '.join(h2o_returns)}"
                 elif len(metrics) == (len(target_values) + 1):
                     h2o_returns = [f"prediction[1][{i}]" for i in range(len(metrics))]
-                    cls.score_code += f"\n\nreturn {', '.join(h2o_returns)}"
+                    cls.score_code += f"{'':4}return {', '.join(h2o_returns)}"
             elif (
                 len(metrics) == len(target_values) == len(returns) and sum(returns) == 0
             ) or (
                 len(metrics) == (len(target_values) + 1) == len(returns)
                 and sum(returns) == 1
             ):
                 proba_returns = [f"prediction[{i}]" for i in range(len(returns))]
-                cls.score_code += f"\n\nreturn {', '.join(proba_returns)}"
+                cls.score_code += f"{'':4}return {', '.join(proba_returns)}"
             elif (len(metrics) - 1) == len(returns) == len(target_values) and sum(
                 returns
             ) == 0:
                 proba_returns = [f"prediction[{i}]" for i in range(len(returns))]
                 cls.score_code += (
                     f"{'':4}target_values = {target_values}\n\n"
-                    f"return target_values[prediction.index(max(prediction))], "
+                    f"{'':4}return target_values[prediction.index(max(prediction))], "
                     f"{', '.join(proba_returns)}"
                 )
             else:
@@ -1454,15 +1453,19 @@ def _viya35_score_code_import(
         """
         files = [
             {
-                "name": f"{prefix}_score.py",
+                "name": f"score_{prefix}.py",
                 "file": cls.score_code,
                 "role": "score",
             }
         ]
         cls.upload_and_copy_score_resources(model_id, files)
-        mr.convert_python_to_ds2(model_id)
+        # The typeConversion endpoint is only valid for models with Python score code
+        model = mr.get_model(model_id)
+        model["scoreCodeType"] = "Python"
+        model = mr.update_model(model)
+        mr.convert_python_to_ds2(model)
         if score_cas:
-            model_contents = mr.get_model_contents(model_id)
+            model_contents = mr.get_model_contents(model)
             for file in model_contents:
                 if file.name == "score.sas":
                     mas_code = mr.get(
@@ -1478,9 +1481,9 @@ def _viya35_score_code_import(
                             }
                         ],
                     )
-                    cas_code = cls.convert_mas_to_cas(mas_code, model_id)
+                    cas_code = cls.convert_mas_to_cas(mas_code, model)
                     cls.upload_and_copy_score_resources(
-                        model_id,
+                        model,
                         [
                             {
                                 "name": CAS_CODE_NAME,

src/sasctl/pzmm/zip_model.py

@@ -29,7 +29,7 @@ def _filter_files(file_dir: Union[str, Path], is_viya4: Optional[bool] = False)
     file_names = []
     file_names.extend(sorted(Path(file_dir).glob("*.json")))
     if is_viya4:
-        file_names.extend(sorted(Path(file_dir).glob("*Score.py")))
+        file_names.extend(sorted(Path(file_dir).glob("score_*.py")))
     file_names.extend(sorted(Path(file_dir).glob("*.pickle")))
     # Include H2O.ai MOJO files
     file_names.extend(sorted(Path(file_dir).glob("*.mojo")))