MLflow for DSS 10.0.3 (#206)

* Support DSSDataset or dataiku.Dataset in MLFlowVersionHandler.evaluate (#189)

* Support DSSDataset or dataiku.Dataset in MLFlowVersionHandler.evaluate

* Adjust a comment

* upload_folder function for managed folders (#186)

* upload_folder function for managed folders

* Add import from managed folder for mlflow

* Improve doc mlflow

* Improve doc mlflow

* Add possibility to load mlflow version from managed_folder object

* Improve upload_folder utils function

* Rename smartFolderId into folderRef

* Remove useless argument check

* Update documentation

Co-authored-by: Adrien Lavoillotte <[email protected]>

* Add inline comment

Co-authored-by: Adrien Lavoillotte <[email protected]>

Co-authored-by: Adrien Lavoillotte <[email protected]>

* Check server side that a MLflow saved model name is not empty when creating or updating

* Simplify and reword doc of upload_folder

* Turn DSSDataset & dataiku.Dataset in :class:`DSSDataset` and :class:`dataiku.Dataset`

Co-authored-by: Aurélien Larive <[email protected]>
Co-authored-by: Valentin Thorey <[email protected]>
Co-authored-by: Adrien Lavoillotte <[email protected]>
Co-authored-by: Ferran Candela <[email protected]>
Co-authored-by: Ferran Candela <[email protected]>

Author: 5 people

dataikuapi/dss/managedfolder.py

@@ -2,6 +2,7 @@
 from ..utils import DataikuUTF8CSVReader
 from ..utils import DataikuStreamedHttpUTF8CSVReader
 import json
+import os
 from requests import utils
 from .metrics import ComputedMetrics
 from .future import DSSFuture
@@ -99,7 +100,7 @@ def delete_file(self, path):
     def put_file(self, path, f):
         """
         Upload the file to the managed folder
-        
+
         Args:
             f: the file contents, as a stream
             path: the path of the file
@@ -108,6 +109,22 @@ def put_file(self, path, f):
                 "POST", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path)),
                 "", f)
 
+    def upload_folder(self, path, folder):
+        """
+        Upload the content of a folder at path in the managed folder.
+
+        Note: upload_folder("target", "source") will result in "target" containing the content
+        of "source", not in "target" containing "source".
+
+        :param str path: the destination path of the folder in the managed folder
+        :param str folder: path  (absolute or relative) of the source folder to upload
+        """
+        for root, _, files in os.walk(folder):
+            for file in files:
+                filename = os.path.join(root, file)
+                with open(filename, "rb") as f:
+                    self.put_file(os.path.join(path, os.path.relpath(filename, folder)), f)
+
     ########################################################
     # Managed folder actions
     ########################################################

dataikuapi/dss/project.py

@@ -725,8 +725,6 @@ def create_mlflow_pyfunc_model(self, name, prediction_type = None):
         :param string name: Human readable name for the new saved model in the flow
         :param string prediction_type: Optional (but needed for most operations). One of BINARY_CLASSIFICATION, MULTICLASS or REGRESSION
         """
-        if not name:
-            raise ValueError("name can not be empty")
         model = {
             "savedModelType" : "MLFLOW_PYFUNC",
             "predictionType" : prediction_type,

dataikuapi/dss/savedmodel.py

@@ -4,6 +4,7 @@
 from .ml import DSSMLTask
 from .ml import DSSTrainedClusteringModelDetails
 from .ml import DSSTrainedPredictionModelDetails
+from .managedfolder import DSSManagedFolder
 
 from ..utils import _make_zipfile
 
@@ -135,7 +136,6 @@ def import_mlflow_version_from_path(self, version_id, path, code_env_name="INHER
         :return a :class:MLFlowVersionHandler in order to interact with the new MLFlow model version
         """
         # TODO: Add a check that it's indeed a MLFlow model folder
-        # TODO: cleanup the archive
         import shutil
         import os
 
@@ -150,6 +150,36 @@ def import_mlflow_version_from_path(self, version_id, path, code_env_name="INHER
         finally:
             shutil.rmtree(archive_temp_dir)
 
+    def import_mlflow_version_from_managed_folder(self, version_id, managed_folder, path, code_env_name="INHERIT"):
+        """
+        Create a new version for this saved model from a path containing a MLFlow model in a managed folder.
+
+        Requires the saved model to have been created using :meth:`dataikuapi.dss.project.DSSProject.create_mlflow_pyfunc_model`.
+
+        :param str version_id: Identifier of the version to create
+        :param str managed_folder: Identifier of the managed folder or `dataikuapi.dss.managedfolder.DSSManagedFolder`
+        :param str path: Path of the MLflow folder in the managed folder
+        :param str code_env_name: Name of the code env to use for this model version. The code env must contain at least
+                                  mlflow and the package(s) corresponding to the used MLFlow-compatible frameworks.
+                                  If value is "INHERIT", the default active code env of the project will be used
+        :return a :class:MLFlowVersionHandler in order to interact with the new MLFlow model version
+        """
+        # TODO: Add a check that it's indeed a MLFlow model folder
+        folder_ref = None
+        if type(managed_folder) is DSSManagedFolder:
+            folder_ref = "{}.{}".format(managed_folder.project_key, managed_folder.id)
+        else:
+            folder_ref = managed_folder
+
+        self.client._perform_empty(
+            "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}?codeEnvName={codeEnvName}".format(
+                project_id=self.project_key, saved_model_id=self.sm_id, version_id=version_id, codeEnvName=code_env_name
+            ),
+            params={"folderRef": folder_ref, "path": path},
+            files={"file": (None, None)}  # required for backend-mandated multipart request
+        )
+        return self.get_mlflow_version_handler(version_id)
+
     def get_mlflow_version_handler(self, version_id):
         """
         Returns a :class:MLFlowVersionHandler to interact with a MLFlow model version
@@ -315,9 +345,10 @@ def evaluate(self, dataset_ref):
 
         :meth:`set_core_metadata` must be called before you can evaluate a dataset
 
-        :param str dataset_ref: Name of the evaluation dataset to use (either a dataset name or "PROJECT.datasetName")
+        :param str dataset_ref: Evaluation dataset to use (either a dataset name, "PROJECT.datasetName", :class:`DSSDataset` instance or :class:`dataiku.Dataset` instance)
         """
-        # TODO Add support for handling a DSSDataset or dataiku.Dataset as dataset_ref
+        if hasattr(dataset_ref, 'name'):
+            dataset_ref = dataset_ref.name
         req = {
             "datasetRef" : dataset_ref
         }
@@ -348,4 +379,4 @@ def prediction_metrics_settings(self):
     def save(self):
         """Saves the settings of this saved model"""
         self.saved_model.client._perform_empty("PUT", "/projects/%s/savedmodels/%s" % (self.saved_model.project_key, self.saved_model.sm_id),
-                    body=self.settings)
+                    body=self.settings)