Merge branch 'master' into feature/mes

Author: fterrazzoni

HISTORY.txt

@@ -1,6 +1,11 @@
 Changelog
 ==========
 
+7.0.0 (2020-07-21)
+------------------
+
+* Initial release for DSS 7.0
+
 5.1.0 (2019-03-01)
 ------------------
 

dataikuapi/dss/admin.py

@@ -290,7 +290,7 @@ def remove_connection_credential(self,connection):
 
     def set_basic_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, user, password):
         """Sets per-user-credentials for a plugin preset that takes a user/password pair"""
-        name = json.dumps(["PLUGIN", pluginId, paramSetId, presetId, paramName])[1:-1]
+        name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1]
 
         self.settings["credentials"][name] = {
             "type": "BASIC",
@@ -300,7 +300,7 @@ def set_basic_plugin_credential(self, plugin_id, param_set_id, preset_id, param_
 
     def set_oauth2_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name, refresh_token):
         """Sets per-user-credentials for a plugin preset that takes a OAuth refresh token"""
-        name = json.dumps(["PLUGIN", pluginId, paramSetId, presetId, paramName])[1:-1]
+        name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1]
 
         self.settings["credentials"][name] = {
             "type": "OAUTH_REFRESH_TOKEN",
@@ -309,7 +309,7 @@ def set_oauth2_plugin_credential(self, plugin_id, param_set_id, preset_id, param
 
     def remove_plugin_credential(self, plugin_id, param_set_id, preset_id, param_name):
         """Removes per-user-credentials for a plugin preset"""
-        name = json.dumps(["PLUGIN", pluginId, paramSetId, presetId, paramName])[1:-1]
+        name = json.dumps(["PLUGIN", plugin_id, param_set_id, preset_id, param_name])[1:-1]
 
         if name in self.settings["credentials"]:
             del self.settings["credentials"][name]
@@ -732,14 +732,15 @@ def set_jupyter_support(self, active):
             raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {}))))
         return resp
 
-    def update_packages(self):
+    def update_packages(self, force_rebuild_env=False):
         """
         Update the code env packages so that it matches its spec
         
         Note: this call requires an API key with admin rights
         """
         resp = self.client._perform_json(
-            "POST", "/admin/code-envs/%s/%s/packages" % (self.env_lang, self.env_name))
+            "POST", "/admin/code-envs/%s/%s/packages" % (self.env_lang, self.env_name),
+            params={"forceRebuildEnv": force_rebuild_env})
         if resp is None:
             raise Exception('Env update returned no data')
         if resp.get('messages', {}).get('error', False):
@@ -797,6 +798,7 @@ def set_definition(self, definition):
             "PUT", "/admin/globalAPIKeys/%s" % self.key,
             body = definition)
 
+
 class DSSCluster(object):
     """
     A handle to interact with a cluster on the DSS instance
@@ -873,7 +875,7 @@ def start(self):
         resp = self.client._perform_json(
             "POST", "/admin/clusters/%s/actions/start" % (self.cluster_id))
         if resp is None:
-            raise Exception('Env update returned no data')
+            raise Exception('Cluster operation returned no data')
         if resp.get('messages', {}).get('error', False):
             raise Exception('Cluster operation failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {}))))
         return resp
@@ -895,6 +897,9 @@ def stop(self, terminate=True):
         return resp
 
 class DSSClusterSettings(object):
+    """
+    The settings of a cluster
+    """
     def __init__(self, client, cluster_id, settings):
         """Do not call directly, use :meth:`DSSCluster.get_settings`"""
         self.client = client
@@ -927,6 +932,9 @@ def save(self):
             "PUT", "/admin/clusters/%s" % (self.cluster_id), body=self.settings)
 
 class DSSClusterStatus(object):
+    """
+    The status of a cluster
+    """
     def __init__(self, client, cluster_id, status):
         """Do not call directly, use :meth:`DSSCluster.get_Status`"""
         self.client = client
@@ -937,4 +945,4 @@ def get_raw(self):
         """
         Gets the whole status as a raw dictionary.
         """
-        return self.status
+        return self.status

dataikuapi/dss/apideployer.py

@@ -1,6 +1,7 @@
 import json
 from .future import DSSFuture
 
+
 class DSSAPIDeployer(object):
     """
     Handle to interact with the API Deployer.
@@ -10,7 +11,7 @@ class DSSAPIDeployer(object):
     def __init__(self, client):
         self.client = client
 
-    def list_deployments(self, as_objects = True):
+    def list_deployments(self, as_objects=True):
         """
         Lists deployments on the API Deployer
 
@@ -55,7 +56,16 @@ def create_deployment(self, deployment_id, service_id, infra_id, version):
         self.client._perform_json("POST", "/api-deployer/deployments", body=settings)
         return self.get_deployment(deployment_id)
 
-    def list_infras(self, as_objects = True):
+    def list_stages(self):
+        """
+        Lists infrastructure stages of the API Deployer
+
+        :rtype: a list of dict. Each dict contains a field "id" for the stage identifier and "desc" for its description.
+        :rtype: list
+        """
+        return self.client._perform_json("GET", "/api-deployer/stages")
+
+    def list_infras(self, as_objects=True):
         """
         Lists deployment infrastructures on the API Deployer
 
@@ -97,7 +107,7 @@ def get_infra(self, infra_id):
         """
         return DSSAPIDeployerInfra(self.client, infra_id)
 
-    def list_services(self, as_objects = True):
+    def list_services(self, as_objects=True):
         """
         Lists API services on the API Deployer
 
@@ -121,7 +131,7 @@ def create_service(self, service_id):
         :rtype: :class:`DSSAPIDeployerService`
         """
         settings = {
-            "serviceId" : service_id
+            "publishedServiceId" : service_id
         }
         self.client._perform_json("POST", "/api-deployer/services", body=settings)
         return self.get_service(service_id)
@@ -331,6 +341,7 @@ def save(self):
                 "PUT", "/api-deployer/deployments/%s/settings" % (self.deployment_id),
                 body = self.settings)
 
+
 class DSSAPIDeployerDeploymentStatus(object):
     """The status of an API Deployer deployment. 
 
@@ -386,7 +397,6 @@ def get_health_messages(self):
         return self.heavy_status["healthMessages"]
 
 
-
 ###############################################
 # Published Service
 ###############################################
@@ -414,16 +424,15 @@ def get_status(self):
         light = self.client._perform_json("GET", "/api-deployer/services/%s" % (self.service_id))
         return DSSAPIDeployerServiceStatus(self.client, self.service_id, light)
 
-    def import_version(self, version_id, fp):
+    def import_version(self, fp):
         """
         Imports a new version for an API service from a file-like object pointing 
         to a version package Zip file
 
-        :param string version_id: identifier of the new version
         :param string fp: A file-like object pointing to a version package Zip file
         """
         return self.client._perform_empty("POST",
-                "/api-deployer/services/%s/packages/%s" % (self.service_id, version_id), files={"file":fp})
+                "/api-deployer/services/%s/versions" % (self.service_id), files={"file":fp})
 
     def get_settings(self):
         """
@@ -440,6 +449,23 @@ def get_settings(self):
 
         return DSSAPIDeployerServiceSettings(self.client, self.service_id, settings)
 
+    def delete_version(self, version):
+        """
+        Deletes a version from this service
+        :param string version: The version to delete
+        """
+        self.client._perform_empty(
+            "DELETE", "/api-deployer/services/%s/versions/%s" % (self.service_id, version))
+
+    def delete(self):
+        """
+        Deletes this service
+
+        You may only delete a service if it has no deployments on it anymore.
+        """
+        return self.client._perform_empty(
+            "DELETE", "/api-deployer/services/%s" % (self.service_id))
+
 
 class DSSAPIDeployerServiceSettings(object):
     """The settings of an API Deployer Service. 
@@ -466,6 +492,7 @@ def save(self):
                 "PUT", "/api-deployer/services/%s/settings" % (self.service_id),
                 body = self.settings)
 
+
 class DSSAPIDeployerServiceStatus(object):
     """The status of an API Deployer Service. 
 

dataikuapi/dss/app.py

@@ -1,41 +1,42 @@
-import sys
+import random
 import re
-import os.path as osp
+import string
+
 from .future import DSSFuture
-from dataikuapi.utils import DataikuException
-import random, string
+
 
 def random_string(length):
     return ''.join(random.choice(string.ascii_letters) for _ in range(length))
 
+
 class DSSApp(object):
     """
-    A handle to interact with a app on the DSS instance.
-    Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_app``
+    A handle to interact with an application on the DSS instance.
+    Do not create this class directly, instead use :meth:`dataikuapi.DSSClient.get_app`
     """
     def __init__(self, client, app_id):
-       self.client = client
-       self.app_id = app_id
+        self.client = client
+        self.app_id = app_id
 
     ########################################################
     # Instances
     ########################################################
 
     def create_instance(self, instance_key, instance_name, wait=True):
         """
-        Creates a new instance of this app. Each instance. must have a globally unique
+        Creates a new instance of this application. Each instance. must have a globally unique
         instance key, separate from any project key across the whole DSS instance
 
         :return: 
         """
         future_resp = self.client._perform_json(
             "POST", "/apps/%s/instances" % self.app_id, body={
-            "targetProjectKey" : instance_key,
-            "targetProjectName" : instance_name
-        })
+                "targetProjectKey": instance_key,
+                "targetProjectName": instance_name
+            })
         future = DSSFuture(self.client, future_resp.get("jobId", None), future_resp)
         if wait:
-            result = future.wait_for_result()
+            future.wait_for_result()
             return DSSAppInstance(self.client, instance_key)
         else:
             return future
@@ -46,13 +47,13 @@ def make_random_project_key(self):
 
     def create_temporary_instance(self):
         """
-        Creates a new temporary instance of this app.
+        Creates a new temporary instance of this application.
         The return value should be used as a Python context manager. Upon exit, the temporary app
         instance is deleted
         :return a :class:`TemporaryDSSAppInstance`
         """
         key = self.make_random_project_key()
-        instance = self.create_instance(key, key, True)
+        self.create_instance(key, key, True)
         return TemporaryDSSAppInstance(self.client, key)
 
     def list_instance_keys(self):
@@ -85,7 +86,7 @@ def get_manifest(self):
 class DSSAppManifest(object):
 
     def __init__(self, client, raw_data, project_key=None):
-        """The manifest for an app. Do not create this class directly"""
+        """The manifest for an application. Do not create this class directly"""
         self.client = client
         self.raw_data = raw_data
         self.project_key = project_key
@@ -110,34 +111,34 @@ def save(self):
 class DSSAppInstance(object):
 
     def __init__(self, client, project_key):
-       self.client = client
-       self.project_key = project_key
+        self.client = client
+        self.project_key = project_key
 
     def get_as_project(self):
         """
-        Get the :class:`dataikuapi.dss.project DSSProject` corresponding to this app instance
+        Get the :class:`dataikuapi.dss.project DSSProject` corresponding to this application instance
         """
         return self.client.get_project(self.project_key)
 
     def get_manifest(self):
         """
-        Get the app manifest for this instance, as a :class:`DSSAppManifest`
+        Get the application manifest for this instance, as a :class:`DSSAppManifest`
         """
         raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key)
         return DSSAppManifest(self.client, raw_data)
 
+
 class TemporaryDSSAppInstance(DSSAppInstance):
     """internal class"""
 
     def __init__(self, client, project_key):
         DSSAppInstance.__init__(self, client,project_key)
 
-
     def close(self):
         self.get_as_project().delete(drop_data=True)
 
     def __enter__(self,):
         return self
 
     def __exit__(self, type, value, traceback):
-        self.close()
+        self.close()

dataikuapi/dss/continuousactivity.py

@@ -1,6 +1,6 @@
 import time
 import sys
-from dataikuapi.utils import DataikuException
+from ..utils import DataikuException
 
 class DSSContinuousActivity(object):
     """
@@ -40,4 +40,4 @@ def get_recipe(self):
     	Return a handle on the associated recipe
     	"""
     	from .recipe import DSSRecipe
-    	return DSSRecipe(self.client, self.project_key, self.recipe_id)
+    	return DSSRecipe(self.client, self.project_key, self.recipe_id)

dataikuapi/dss/flow.py

@@ -518,8 +518,9 @@ def get_successor_computables(self, node, as_type="dict"):
         return self._convert_nodes_list(computables, as_type)
 
     def _convert_nodes_list(self, nodes, as_type):
+        actual_nodes = [node for node in nodes if node['type'] != 'RUNNABLE_IMPLICIT_RECIPE']
         if as_type == "object" or as_type == "objects":
-            return [self._get_object_from_graph_node(node) for node in nodes]
+            return [self._get_object_from_graph_node(node) for node in actual_nodes]
         else:
             return nodes
 
@@ -532,6 +533,8 @@ def _get_object_from_graph_node(self, node):
             return DSSManagedFolder(self.flow.client, self.flow.project.project_key, node["ref"])
         elif node["type"] == "COMPUTABLE_SAVED_MODEL":
             return DSSSavedModel(self.flow.client, self.flow.project.project_key, node["ref"])
+        elif node["type"] == "COMPUTABLE_STREAMING_ENDPOINT":
+            return DSSStreamingEndpoint(self.flow.client, self.flow.project.project_key, node["ref"])
         else:
             # TODO add streaming elements
             raise Exception("unsupported node type: %s" % node["type"])