Merge branch 'release/10.0' into chore/dss10-clarify-get-variable-apis

Author: AFrancani

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 .idea
 *.iml
+egg-info

dataikuapi/__init__.py

@@ -1,8 +1,9 @@
 from .dssclient import DSSClient
+from .fmclient import FMClientAWS, FMClientAzure
 
 from .apinode_client import APINodeClient
 from .apinode_admin_client import APINodeAdminClient
 
 from .dss.recipe import GroupingRecipeCreator, JoinRecipeCreator, StackRecipeCreator, WindowRecipeCreator, SyncRecipeCreator, SamplingRecipeCreator, SQLQueryRecipeCreator, CodeRecipeCreator, SplitRecipeCreator, SortRecipeCreator, TopNRecipeCreator, DistinctRecipeCreator, DownloadRecipeCreator, PredictionScoringRecipeCreator, ClusteringScoringRecipeCreator
 
-from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule
+from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule

dataikuapi/dss/admin.py

@@ -710,7 +710,19 @@ def set_definition(self, env):
         """
         return self.client._perform_json(
             "PUT", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name), body=env)
-    
+
+    def get_version_for_project(self, project_key):
+        """
+        Resolve the code env version for a given project
+
+        Note: version will only be non-empty for versioned code envs actually used by the project
+
+        :returns: the code env reference, with a version field
+        """
+        return self.client._perform_json(
+            "GET", "/admin/code-envs/%s/%s/%s/version" % (self.env_lang, self.env_name, project_key))
+
+   
     ########################################################
     # Code env actions
     ########################################################
@@ -747,6 +759,52 @@ def update_packages(self, force_rebuild_env=False):
             raise Exception('Env update failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {}))))
         return resp
 
+    def update_images(self, env_version=None):
+        """
+        Rebuild the docker image of the code env
+        
+        Note: this call requires an API key with admin rights
+        """
+        resp = self.client._perform_json(
+            "POST", "/admin/code-envs/%s/%s/images" % (self.env_lang, self.env_name),
+            params={"envVersion": env_version})
+        if resp is None:
+            raise Exception('Env image build returned no data')
+        if resp.get('messages', {}).get('error', False):
+            raise Exception('Env image build failed : %s' % (json.dumps(resp.get('messages', {}).get('messages', {}))))
+        return resp
+
+    def list_usages(self, env_version=None):
+        """
+        List usages of the code env in the instance
+
+        :return: a list of objects where the code env is used
+        """
+        return self.client._perform_json(
+            "GET", "/admin/code-envs/%s/%s/usages" % (self.env_lang, self.env_name), params={"envVersion": env_version})
+
+    def list_logs(self, env_version=None):
+        """
+        List logs of the code env in the instance
+
+        :return: a list of log descriptions
+        """
+        return self.client._perform_json(
+            "GET", "/admin/code-envs/%s/%s/logs" % (self.env_lang, self.env_name), params={"envVersion": env_version})
+
+    def get_log(self, log_name):
+        """
+        Get the logs of the code env
+        
+        Args:
+            log_name: name of the log to fetch
+            
+           Returns:
+               the log, as a string
+        """
+        return self.client._perform_text(
+            "GET", "/admin/code-envs/%s/%s/logs/%s" % (self.env_lang, self.env_name, log_name))
+
 
 class DSSGlobalApiKey(object):
     """

dataikuapi/dss/apideployer.py

@@ -36,7 +36,7 @@ def get_deployment(self, deployment_id):
         """
         return DSSAPIDeployerDeployment(self.client, deployment_id)
 
-    def create_deployment(self, deployment_id, service_id, infra_id, version):
+    def create_deployment(self, deployment_id, service_id, infra_id, version, ignore_warnings=False):
         """
         Creates a deployment and returns the handle to interact with it. The returned deployment
         is not yet started and you need to call :meth:`~DSSAPIDeployerDeployment.start_update`
@@ -45,6 +45,7 @@ def create_deployment(self, deployment_id, service_id, infra_id, version):
         :param str service_id: Identifier of the API Service to target
         :param str infra_id: Identifier of the deployment infrastructure to use
         :param str version_id: Identifier of the API Service version to deploy
+        :param boolean ignore_warnings: ignore warnings concerning the governance status of the model version(s) to deploy
         :rtype: :class:`DSSAPIDeployerDeployment`
         """
         settings = {
@@ -53,7 +54,7 @@ def create_deployment(self, deployment_id, service_id, infra_id, version):
             "infraId" : infra_id,
             "version" : version
         }
-        self.client._perform_json("POST", "/api-deployer/deployments", body=settings)
+        self.client._perform_json("POST", "/api-deployer/deployments", params={"ignoreWarnings": ignore_warnings}, body=settings)
         return self.get_deployment(deployment_id)
 
     def list_stages(self):
@@ -81,19 +82,21 @@ def list_infras(self, as_objects=True):
         else:
             return l
 
-    def create_infra(self, infra_id, stage, type):
+    def create_infra(self, infra_id, stage, type, govern_check_policy="NO_CHECK"):
         """
         Creates a new infrastructure on the API Deployer and returns the handle to interact with it.
 
         :param str infra_id: Unique Identifier of the infra to create
         :param str stage: Infrastructure stage. Stages are configurable on each API Deployer
         :param str type: STATIC or KUBERNETES
+        :param str govern_check_policy: PREVENT, WARN, or NO_CHECK depending if the deployer will check wether the saved model versions deployed on this infrastructure has to be managed and approved in Dataiku Govern
         :rtype: :class:`DSSAPIDeployerInfra`
         """
         settings = {
             "id": infra_id,
             "stage": stage,
             "type": type,
+            "govern_check_policy": govern_check_policy,
         }
         self.client._perform_json("POST", "/api-deployer/infras", body=settings)
         return self.get_infra(infra_id)
@@ -308,6 +311,16 @@ def get_status(self):
 
         return DSSAPIDeployerDeploymentStatus(self.client, self.deployment_id, light, heavy)
 
+    def get_governance_status(self, version=""):
+        """
+        Returns the governance status about this deployment if applicable
+        It covers all the embedded model versions
+
+        :param str version: (Optional) The specific package version of the published service to get status from. If empty, consider all the versions used in the deployment generation mapping.
+        :rtype: dict InforMessages containing the governance status
+        """
+        return self.client._perform_json("GET", "/api-deployer/deployments/%s/governance-status" % (self.deployment_id), params={ "version": version })
+
     def get_settings(self):
         """
         Gets the settings of this deployment. If you want to modify the settings, you need to
@@ -334,15 +347,29 @@ def start_update(self):
 
         return DSSFuture(self.client, future_response.get('jobId', None), future_response)
 
-    def delete(self):
+    def delete(self, disable_first=False):
         """
-        Deletes this deployment
+        Deletes this deployment. The disable_first flag automatically disables the deployment
+        before its deletion.
+
+        :param boolean disable_first: If True, automatically disables this deployment before deleting it.
+        If False, will raise an Exception if this deployment is enabled.
 
-        You may only delete a deployment if it is disabled and has been updated after disabling it.
         """
-        return self.client._perform_empty(
-            "DELETE", "/api-deployer/deployments/%s" % (self.deployment_id))
 
+        # Check if the deployment is disabled
+        is_enabled = self.get_status().light_status["deploymentBasicInfo"].get("enabled")
+        if is_enabled and not disable_first:
+            raise Exception("Deployment {} deletion failed: deployment must be disabled first.".format(self.deployment_id))
+        if is_enabled:
+            settings = self.get_settings()
+            settings.set_enabled(enabled=False)
+            settings.save()
+        self.client._perform_empty(
+                "DELETE", "/api-deployer/deployments/%s" % (self.deployment_id))
+
+                
+            
 
 class DSSAPIDeployerDeploymentSettings(object):
     """
@@ -381,12 +408,15 @@ def set_single_version(self, version):
             "generation": version
         }
 
-    def save(self):
+    def save(self, ignore_warnings=False):
         """
         Saves back these settings to the deployment
+
+        :param boolean ignore_warnings: ignore warnings concerning the governance status of the model version(s) to deploy
         """
         self.client._perform_empty(
                 "PUT", "/api-deployer/deployments/%s/settings" % (self.deployment_id),
+                params = { "ignoreWarnings" : ignore_warnings },
                 body = self.settings)
 
 
@@ -518,7 +548,7 @@ def delete(self):
 
         You may only delete a service if it has no deployments on it anymore.
         """
-        return self.client._perform_empty(
+        self.client._perform_empty(
             "DELETE", "/api-deployer/services/%s" % (self.service_id))
 
 

dataikuapi/dss/metrics.py

@@ -4,6 +4,9 @@ class ComputedMetrics(object):
     def __init__(self, raw):
         self.raw = raw
 
+    def get_raw(self):
+        return self.raw
+
     def get_metric_by_id(self, id):
         all_ids = []
         for metric in self.raw["metrics"]:

dataikuapi/dss/ml.py

@@ -1804,6 +1804,10 @@ def get_raw(self):
         """
         return self.details
 
+    @property
+    def full_id(self):
+        return self.details["fullModelId"]
+
     def get_raw_snippet(self):
         """
         Gets the raw dictionary of trained model snippet. 
@@ -1863,7 +1867,7 @@ def get_diagnostics(self):
         :returns: list of diagnostics
         :rtype: list of type `dataikuapi.dss.ml.DSSMLDiagnostic`
         """
-        diagnostics = self.details.get("trainDiagnostics", {})
+        diagnostics = self.details.get("mlDiagnostics", {})
         return [DSSMLDiagnostic(d) for d in diagnostics.get("diagnostics", [])]
 
     def generate_documentation(self, folder_id=None, path=None):
@@ -3348,7 +3352,7 @@ def get_settings(self):
         else:
             return DSSClusteringMLTaskSettings(self.client, self.project_key, self.analysis_id, self.mltask_id, settings)
 
-    def train(self, session_name=None, session_description=None):
+    def train(self, session_name=None, session_description=None, run_queue=False):
         """
         Trains models for this ML Task
         
@@ -3366,7 +3370,7 @@ def train(self, session_name=None, session_description=None):
         :return: A list of model identifiers
         :rtype: list of strings
         """
-        train_ret = self.start_train(session_name, session_description)
+        train_ret = self.start_train(session_name, session_description, run_queue)
         self.wait_train_complete()
         return self.get_trained_models_ids(session_id = train_ret["sessionId"])
 
@@ -3395,7 +3399,7 @@ def ensemble(self, model_ids=None, method=None):
         return train_ret
 
 
-    def start_train(self, session_name=None, session_description=None):
+    def start_train(self, session_name=None, session_description=None, run_queue=False):
         """
         Starts asynchronously a new train session for this ML Task.
 
@@ -3406,7 +3410,8 @@ def start_train(self, session_name=None, session_description=None):
         """
         session_info = {
                             "sessionName" : session_name,
-                            "sessionDescription" : session_description
+                            "sessionDescription" : session_description,
+                            "runQueue": run_queue
                         }
 
         return self.client._perform_json(
@@ -3521,6 +3526,16 @@ def delete_trained_model(self, model_id):
         self.client._perform_empty(
             "DELETE", "/projects/%s/models/lab/%s/%s/models/%s" % (self.project_key, self.analysis_id, self.mltask_id, model_id))
 
+    def train_queue(self):
+        """
+        Trains this MLTask's queue
+
+        :return: A dict including the next sessionID to be trained in the queue
+        :rtype dict
+        """
+        return self.client._perform_json(
+            "POST", "/projects/%s/models/lab/%s/%s/actions/train-queue" % (self.project_key, self.analysis_id, self.mltask_id))
+
     def deploy_to_flow(self, model_id, model_name, train_dataset, test_dataset=None, redo_optimization=True):
         """
         Deploys a trained model from this ML Task to a saved model + train recipe in the Flow.
@@ -3606,3 +3621,17 @@ def guess(self, prediction_type=None, reguess_level=None):
             "PUT",
             "/projects/%s/models/lab/%s/%s/guess" % (self.project_key, self.analysis_id, self.mltask_id),
             params = obj)
+
+
+class DSSMLTaskQueues(object):
+    """
+    Iterable listing of MLTask queues
+    """
+    def __init__(self, data):
+        self.data = data
+
+    def __iter__(self):
+        return self.data["queues"].__iter__()
+
+    def get_raw(self):
+        return self.data