Merge branch 'release/8.0'

Author: samronsin

HISTORY.txt

@@ -1,6 +1,11 @@
 Changelog
 ==========
 
+7.0.0 (2020-07-21)
+------------------
+
+* Initial release for DSS 7.0
+
 5.1.0 (2019-03-01)
 ------------------
 

dataikuapi/dss/admin.py

@@ -290,7 +290,7 @@ def remove_connection_credential(self,connection):
 
     def set_basic_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, user, password):
         """Sets per-user-credentials for a plugin preset that takes a user/password pair"""
-        name = json.dumps(["PLUGIN", pluginId, paramSetId, presetId, paramName])[1:-1]
+        name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1]
 
         self.settings["credentials"][name] = {
             "type": "BASIC",
@@ -300,7 +300,7 @@ def set_basic_plugin_credential(self, plugin_id, param_set_id, preset_id, param_
 
     def set_oauth2_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, refresh_token):
         """Sets per-user-credentials for a plugin preset that takes a OAuth refresh token"""
-        name = json.dumps(["PLUGIN", pluginId, paramSetId, presetId, paramName])[1:-1]
+        name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1]
 
         self.settings["credentials"][name] = {
             "type": "OAUTH_REFRESH_TOKEN",
@@ -309,7 +309,7 @@ def set_oauth2_plugin_credential(self, plugin_id, param_set_id, preset_id, param
 
     def remove_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name):
         """Removes per-user-credentials for a plugin preset"""
-        name = json.dumps(["PLUGIN", pluginId, paramSetId, presetId, paramName])[1:-1]
+        name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1]
 
         if name in self.settings["credentials"]:
             del self.settings["credentials"][name]
@@ -732,14 +732,15 @@ def set_jupyter_support(self, active):
             raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {}))))
         return resp
 
-    def update_packages(self):
+    def update_packages(self, force_rebuild_env=False):
         """
         Update the code env packages so that it matches its spec
         
         Note: this call requires an API key with admin rights
         """
         resp = self.client._perform_json(
-            "POST", "/admin/code-envs/%s/%s/packages" % (self.env_lang, self.env_name))
+            "POST", "/admin/code-envs/%s/%s/packages" % (self.env_lang, self.env_name),
+            params={"forceRebuildEnv": force_rebuild_env})
         if resp is None:
             raise Exception('Env update returned no data')
         if resp.get('messages', {}).get('error', False):
@@ -797,6 +798,7 @@ def set_definition(self, definition):
             "PUT", "/admin/globalAPIKeys/%s" % self.key,
             body = definition)
 
+
 class DSSCluster(object):
     """
     A handle to interact with a cluster on the DSS instance
@@ -873,7 +875,7 @@ def start(self):
         resp = self.client._perform_json(
             "POST", "/admin/clusters/%s/actions/start" % (self.cluster_id))
         if resp is None:
-            raise Exception('Env update returned no data')
+            raise Exception('Cluster operation returned no data')
         if resp.get('messages', {}).get('error', False):
             raise Exception('Cluster operation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {}))))
         return resp
@@ -895,6 +897,9 @@ def stop(self, terminate=True):
         return resp
 
 class DSSClusterSettings(object):
+    """
+    The settings of a cluster
+    """
     def __init__(self, client, cluster_id, settings):
         """Do not call directly, use :meth:`DSSCluster.get_settings`"""
         self.client = client
@@ -927,6 +932,9 @@ def save(self):
             "PUT", "/admin/clusters/%s" % (self.cluster_id), body=self.settings)
 
 class DSSClusterStatus(object):
+    """
+    The status of a cluster
+    """
     def __init__(self, client, cluster_id, status):
         """Do not call directly, use :meth:`DSSCluster.get_Status`"""
         self.client = client
@@ -937,4 +945,4 @@ def get_raw(self):
         """
         Gets the whole status as a raw dictionary.
         """
-        return self.status
+        return self.status

dataikuapi/dss/app.py

@@ -1,41 +1,42 @@
-import sys
+import random
 import re
-import os.path as osp
+import string
+
 from .future import DSSFuture
-from dataikuapi.utils import DataikuException
-import random, string
+
 
 def random_string(length):
     return ''.join(random.choice(string.ascii_letters) for _ in range(length))
 
+
 class DSSApp(object):
     """
-    A handle to interact with a app on the DSS instance.
-    Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_app``
+    A handle to interact with an application on the DSS instance.
+    Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_app`
     """
     def __init__(self, client, app_id):
-       self.client = client
-       self.app_id = app_id
+        self.client = client
+        self.app_id = app_id
 
     ########################################################
     # Instances
     ########################################################
 
     def create_instance(self, instance_key, instance_name, wait=True):
         """
-        Creates a new instance of this app. Each instance. must have a globally unique
+        Creates a new instance of this application. Each instance. must have a globally unique
         instance key, separate from any project key across the whole DSS instance
 
         :return: 
         """
         future_resp = self.client._perform_json(
             "POST", "/apps/%s/instances" % self.app_id, body={
-            "targetProjectKey" : instance_key,
-            "targetProjectName" : instance_name
-        })
+                "targetProjectKey": instance_key,
+                "targetProjectName": instance_name
+            })
         future = DSSFuture(self.client, future_resp.get("jobId", None), future_resp)
         if wait:
-            result = future.wait_for_result()
+            future.wait_for_result()
             return DSSAppInstance(self.client, instance_key)
         else:
             return future
@@ -46,13 +47,13 @@ def make_random_project_key(self):
 
     def create_temporary_instance(self):
         """
-        Creates a new temporary instance of this app.
+        Creates a new temporary instance of this application.
         The return value should be used as a Python context manager. Upon exit, the temporary app
         instance is deleted
         :return a :class:`TemporaryDSSAppInstance`
         """
         key = self.make_random_project_key()
-        instance = self.create_instance(key, key, True)
+        self.create_instance(key, key, True)
         return TemporaryDSSAppInstance(self.client, key)
 
     def list_instance_keys(self):
@@ -85,7 +86,7 @@ def get_manifest(self):
 class DSSAppManifest(object):
 
     def __init__(self, client, raw_data, project_key=None):
-        """The manifest for an app. Do not create this class directly"""
+        """The manifest for an application. Do not create this class directly"""
         self.client = client
         self.raw_data = raw_data
         self.project_key = project_key
@@ -110,34 +111,34 @@ def save(self):
 class DSSAppInstance(object):
 
     def __init__(self, client, project_key):
-       self.client = client
-       self.project_key = project_key
+        self.client = client
+        self.project_key = project_key
 
     def get_as_project(self):
         """
-        Get the :class:`dataikuapi.dss.project DSSProject` corresponding to this app instance
+        Get the :class:`dataikuapi.dss.project DSSProject` corresponding to this application instance
         """
         return self.client.get_project(self.project_key)
 
     def get_manifest(self):
         """
-        Get the app manifest for this instance, as a :class:`DSSAppManifest`
+        Get the application manifest for this instance, as a :class:`DSSAppManifest`
         """
         raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key)
         return DSSAppManifest(self.client, raw_data)
 
+
 class TemporaryDSSAppInstance(DSSAppInstance):
     """internal class"""
 
     def __init__(self, client, project_key):
         DSSAppInstance.__init__(self, client,project_key)
 
-
     def close(self):
         self.get_as_project().delete(drop_data=True)
 
     def __enter__(self,):
         return self
 
     def __exit__(self, type, value, traceback):
-        self.close()
+        self.close()

dataikuapi/dss/continuousactivity.py

@@ -1,6 +1,6 @@
 import time
 import sys
-from dataikuapi.utils import DataikuException
+from ..utils import DataikuException
 
 class DSSContinuousActivity(object):
     """
@@ -40,4 +40,4 @@ def get_recipe(self):
     	Return a handle on the associated recipe
     	"""
     	from .recipe import DSSRecipe
-    	return DSSRecipe(self.client, self.project_key, self.recipe_id)
+    	return DSSRecipe(self.client, self.project_key, self.recipe_id)

dataikuapi/dss/flow.py

@@ -515,8 +515,9 @@ def get_successor_computables(self, node, as_type="dict"):
         return self._convert_nodes_list(computables, as_type)
 
     def _convert_nodes_list(self, nodes, as_type):
+        actual_nodes = [node for node in nodes if node['type'] != 'RUNNABLE_IMPLICIT_RECIPE']
         if as_type == "object" or as_type == "objects":
-            return [self._get_object_from_graph_node(node) for node in nodes]
+            return [self._get_object_from_graph_node(node) for node in actual_nodes]
         else:
             return nodes
 
@@ -529,6 +530,8 @@ def _get_object_from_graph_node(self, node):
             return DSSManagedFolder(self.flow.client, self.flow.project.project_key, node["ref"])
         elif node["type"] == "COMPUTABLE_SAVED_MODEL":
             return DSSSavedModel(self.flow.client, self.flow.project.project_key, node["ref"])
+        elif node["type"] == "COMPUTABLE_STREAMING_ENDPOINT":
+            return DSSStreamingEndpoint(self.flow.client, self.flow.project.project_key, node["ref"])
         else:
             # TODO add streaming elements
             raise Exception("unsupported node type: %s" % node["type"])

dataikuapi/dss/ml.py

@@ -275,7 +275,7 @@ def disable_all_algorithms(self):
             custom_mllib["enabled"] = False
         for custom_python in self.mltask_settings["modeling"]["custom_python"]:
             custom_python["enabled"] = False
-        for plugin in self.mltask_settings["modeling"]["plugin"].values():
+        for plugin in self.mltask_settings["modeling"]["plugin_python"].values():
             plugin["enabled"] = False
 
     def get_all_possible_algorithm_names(self):
@@ -1447,7 +1447,7 @@ class DSSMLTask(object):
 
     @staticmethod
     def from_full_model_id(client, fmi, project_key=None):
-        match = re.match("^A-(\w+)-(\w+)-(\w+)-(s[0-9]+)-(pp[0-9]+(-part-(\w+)|-base)?)-(m[0-9]+)$", fmi)
+        match = re.match(r"^A-(\w+)-(\w+)-(\w+)-(s[0-9]+)-(pp[0-9]+(-part-(\w+)|-base)?)-(m[0-9]+)$", fmi)
         if match is None:
             return DataikuException("Invalid model id: {}".format(fmi))
         else:

dataikuapi/dss/plugin.py

@@ -1,4 +1,4 @@
-from dataikuapi.utils import DataikuException
+from ..utils import DataikuException
 
 
 class DSSPluginSettings(object):

dataikuapi/dss/project.py

@@ -1384,9 +1384,11 @@ def set_python_code_env(self, code_env_name):
         """
         if code_env_name is None:
             self.settings["settings"]["codeEnvs"]["python"]["useBuiltinEnv"] = True
+            self.settings["settings"]["codeEnvs"]["python"]["mode"] = "USE_BUILTIN_MODE"
         else:
             self.settings["settings"]["codeEnvs"]["python"]["useBuiltinEnv"] = False
             self.settings["settings"]["codeEnvs"]["python"]["envName"] = code_env_name
+            self.settings["settings"]["codeEnvs"]["python"]["mode"] = "EXPLICIT_ENV"
 
     def set_r_code_env(self, code_env_name):
         """Sets the default R code env used by this project
@@ -1395,9 +1397,11 @@ def set_r_code_env(self, code_env_name):
         """
         if code_env_name is None:
             self.settings["settings"]["codeEnvs"]["r"]["useBuiltinEnv"] = True
+            self.settings["settings"]["codeEnvs"]["r"]["mode"] = "USE_BUILTIN_MODE"
         else:
             self.settings["settings"]["codeEnvs"]["r"]["useBuiltinEnv"] = False
             self.settings["settings"]["codeEnvs"]["r"]["envName"] = code_env_name
+            self.settings["settings"]["codeEnvs"]["r"]["mode"] = "EXPLICIT_ENV"
 
     def set_container_exec_config(self, config_name):
         """Sets the default containerized execution config used by this project

dataikuapi/dss/savedmodel.py

@@ -1,5 +1,5 @@
-from dataikuapi.dss.ml import DSSMLTask
 from .metrics import ComputedMetrics
+from .ml import DSSMLTask
 from .ml import DSSTrainedClusteringModelDetails
 from .ml import DSSTrainedPredictionModelDetails
 

dataikuapi/dss/tools/codegen.py

@@ -14,7 +14,7 @@ def __exit__(self, b, c, d):
 
 
 def slugify(name):
-    return re.sub("[^A-Za-z0-9_]", "_", name)
+    return re.sub("r[^A-Za-z0-9_]", "_", name)
 
 class FlowCodeGenerator(object):
     def __init__(self):