Feature/remote url (#117)

* add requests to `requirements.txt`

* add a valid `.json` file to be used in

* add download function

* integrate download from url

* add download from url test cases

* fix exception raise during test

* add `Retry` mechanism

* remove unused imports + removing trailing whitespaces

* update docstring for url

* refactor error messages and add them to `pymilo_param.py`

Author: AHReccese

dev-requirements.txt

@@ -1,6 +1,7 @@
 numpy==2.0.1
 scikit-learn==1.5.1
 scipy>=0.19.1
+requests>=2.0.0
 setuptools>=40.8.0
 vulture>=1.0
 bandit>=1.5.1

pymilo/exceptions/deserialize_exception.py

@@ -42,7 +42,7 @@ def __init__(self, meta_data):
         error_type = meta_data['error_type']
         error_type_to_message = {
             DeserializationErrorTypes.CORRUPTED_JSON_FILE:
-            'the given json file is not a valid .json file.',
+            'the given file is not a valid .json file.',
             DeserializationErrorTypes.INVALID_MODEL:
             'the given model is not supported or is not a valid model.',
             DeserializationErrorTypes.VALID_MODEL_INVALID_INTERNAL_STRUCTURE:

pymilo/pymilo_obj.py

@@ -1,15 +1,14 @@
 # -*- coding: utf-8 -*-
 """PyMilo modules."""
-from .pymilo_func import get_sklearn_data, get_sklearn_version, to_sklearn_model
-from .utils.util import get_sklearn_type
-from .pymilo_param import PYMILO_VERSION, PYMILO_VERSION_DOES_NOT_EXIST, UNEQUAL_PYMILO_VERSIONS, UNEQUAL_SKLEARN_VERSIONS
 import json
-
-from .exceptions.deserialize_exception import PymiloDeserializationException, DeserializationErrorTypes
-from .exceptions.serialize_exception import PymiloSerializationException, SerializationErrorTypes
-from traceback import format_exc
-from warnings import warn
 from copy import deepcopy
+from warnings import warn
+from traceback import format_exc
+from .utils.util import get_sklearn_type, download_model
+from .pymilo_func import get_sklearn_data, get_sklearn_version, to_sklearn_model
+from .exceptions.serialize_exception import PymiloSerializationException, SerializationErrorTypes
+from .exceptions.deserialize_exception import PymiloDeserializationException, DeserializationErrorTypes
+from .pymilo_param import PYMILO_VERSION, UNEQUAL_PYMILO_VERSIONS, UNEQUAL_SKLEARN_VERSIONS, INVALID_IMPORT_INIT_PARAMS
 
 
 class Export:
@@ -84,25 +83,29 @@ class Import:
     >>> imported_sklearn_model.predict(x_test)
     """
 
-    def __init__(self, file_adr, json_dump=None):
+    def __init__(self, file_adr=None, json_dump=None, url=None):
         """
         Initialize the Pymilo Import instance.
 
         :param file_adr: the file path where the serialized model's JSON file is located.
-        :type file_adr: string
+        :type file_adr: str or None
         :param json_dump: the json dump of the associated model, it can be None(reading from the file_adr)
         :type json_dump: str or None
+        :param url: url to exported JSON file
+        :type: str or None
         :return: an instance of the Pymilo Import class
         """
         serialized_model_obj = None
+        if url is not None:
+            serialized_model_obj = download_model(url)
+        elif json_dump is not None and isinstance(json_dump, str):
+            serialized_model_obj = json.loads(json_dump)
+        elif file_adr is not None:
+            with open(file_adr, 'r') as fp:
+                serialized_model_obj = json.load(fp)
+        else:
+            raise Exception(INVALID_IMPORT_INIT_PARAMS)
         try:
-            if json_dump and isinstance(json_dump, str):
-                serialized_model_obj = json.loads(json_dump)
-            else:
-                with open(file_adr, 'r') as fp:
-                    serialized_model_obj = json.load(fp)
-            if "pymilo_version" not in serialized_model_obj:
-                raise Exception(PYMILO_VERSION_DOES_NOT_EXIST)
             if not serialized_model_obj["pymilo_version"] == PYMILO_VERSION:
                 warn(UNEQUAL_PYMILO_VERSIONS, category=Warning)
             if not serialized_model_obj["sklearn_version"] == get_sklearn_version():
@@ -114,17 +117,19 @@ def __init__(self, file_adr, json_dump=None):
             json_content = None
             if json_dump and isinstance(json_dump, str):
                 json_content = json_dump
-            else:
+            elif file_adr is not None:
                 with open(file_adr) as f:
                     json_content = f.readlines()
+            else:
+                json_content = serialized_model_obj
             raise PymiloDeserializationException(
                 {
                     'json_file': json_content,
                     'error_type': DeserializationErrorTypes.CORRUPTED_JSON_FILE,
                     'error': {
                         'Exception': repr(e),
                         'Traceback': format_exc()},
-                    'object': serialized_model_obj})
+                    'object': ""})
 
     def to_model(self):
         """

pymilo/pymilo_param.py

@@ -87,6 +87,10 @@
 PYMILO_VERSION_DOES_NOT_EXIST = "Corrupted JSON file, `pymilo_version` doesn't exist in this file."
 UNEQUAL_PYMILO_VERSIONS = "warning: Installed PyMilo version differs from the PyMilo version used to create the JSON file."
 UNEQUAL_SKLEARN_VERSIONS = "warning: Installed Scikit version differs from the Scikit version used to create the JSON file and it may prevent PyMilo from transporting seamlessly."
+INVALID_IMPORT_INIT_PARAMS = "Invalid input parameters, you should either pass a valid file_adr or a json_dump or a url to initiate Import class."
+DOWNLOAD_MODEL_FAILED = "Failed to download the JSON file, Server didn't respond."
+INVALID_DOWNLOADED_MODEL = "The downloaded content is not a valid JSON file."
+
 
 SKLEARN_LINEAR_MODEL_TABLE = {
     "DummyRegressor": dummy.DummyRegressor,

pymilo/utils/util.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 """utility module."""
-from inspect import signature
+import requests
 import importlib
+from inspect import signature
+from ..pymilo_param import DOWNLOAD_MODEL_FAILED, INVALID_DOWNLOADED_MODEL
 
 
 def get_sklearn_type(model):
@@ -135,3 +137,31 @@ def prefix_list(list1, list2):
     if len(list1) < len(list2):
         return False
     return all(list1[j] == list2[j] for j in range(len(list2)))
+
+
+def download_model(url):
+    """
+    Download the model from the given url.
+
+    :param url: url to exported JSON file
+    :type url: str
+
+    :return: obj
+    """
+    s = requests.Session()
+    retries = requests.adapters.Retry(
+        total=5,
+        backoff_factor=0.1,
+        status_forcelist=[500, 502, 503, 504]
+    )
+    s.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
+    s.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
+    try:
+        response = s.get(url)
+    except Exception:
+        raise Exception(DOWNLOAD_MODEL_FAILED)
+    try:
+        if response.status_code == 200:
+            return response.json()
+    except ValueError:
+        raise Exception(INVALID_DOWNLOADED_MODEL)

requirements.txt

@@ -1,3 +1,4 @@
 numpy>=1.9.0
 scikit-learn>=0.22.2
 scipy>=0.19.1
+requests>=2.0.0

tests/test_exceptions/import_exceptions.py

@@ -1,9 +1,9 @@
 # CORRUPTED_JSON_FILE = 1 -> tested.
 # INVALID_MODEL = 2 -> tested.
 # VALID_MODEL_INVALID_INTERNAL_STRUCTURE = 3 -> tested.
+import os
 from pymilo.pymilo_obj import Import
 
-import os
 
 def invalid_json(print_output = True):
     json_files = ["corrupted", "unknown-model"]
@@ -16,4 +16,24 @@ def invalid_json(print_output = True):
       except Exception as e:
         if print_output: print("An Exception occured\n", e)
         return True
- 
+
+def invalid_url():
+  try:
+    url = "https://invalid_url"
+    Import(url=url)
+    return False
+  except Exception:
+    return True
+
+def valid_url_invalid_file():
+  try:
+    url = "https://filesamples.com/samples/code/json/sample1.json"
+    Import(url=url)
+    return False
+  except Exception:
+    return True
+
+# def valid_url_valid_file():
+#   with pytest.raises(Exception):
+#     url = "https://raw.githubusercontent.com/openscilab/pymilo/main/tests/test_exceptions/valid_jsons/linear_regression.json"
+#     Import(url=url)

tests/test_exceptions/test_exceptions.py

@@ -3,10 +3,15 @@
 from export_exceptions import valid_model_invalid_structure_neural_network
 from export_exceptions import valid_model_irrelevant_chain
 
-from import_exceptions import invalid_json
+from import_exceptions import invalid_json, invalid_url, valid_url_invalid_file#, valid_url_valid_file
 
 EXCEPTION_TESTS = {
-    'IMPORT': [invalid_json],
+    'IMPORT': [
+        invalid_json,
+        invalid_url,
+        valid_url_invalid_file,
+        #valid_url_valid_file,
+        ],
     'EXPORT': [
         invalid_model,
         valid_model_invalid_structure_linear_model,

tests/test_exceptions/valid_jsons/linear_regression.json

@@ -0,0 +1,55 @@
+{
+    "data": {
+        "fit_intercept": true,
+        "copy_X": true,
+        "n_jobs": null,
+        "positive": false,
+        "n_features_in_": 10,
+        "coef_": {
+            "pymiloed-ndarray-list": [
+                0.30609424754267966,
+                -237.63557011300716,
+                510.53804765114097,
+                327.7298779909887,
+                -814.1119263534517,
+                492.7995945034062,
+                102.84123996793083,
+                184.6034960903708,
+                743.5093875957093,
+                76.09664636971895
+            ],
+            "pymiloed-ndarray-dtype": "float64",
+            "pymiloed-ndarray-shape": [
+                10
+            ],
+            "pymiloed-data-structure": "numpy.ndarray"
+        },
+        "rank_": 10,
+        "singular_": {
+            "pymiloed-ndarray-list": [
+                1.9578051002417796,
+                1.1797491126040702,
+                1.0755406405377144,
+                0.9579192686906345,
+                0.7980638292867588,
+                0.7594342409324799,
+                0.7216957209064547,
+                0.6459380350140406,
+                0.27271507089040337,
+                0.0915832239699
+            ],
+            "pymiloed-ndarray-dtype": "float64",
+            "pymiloed-ndarray-shape": [
+                10
+            ],
+            "pymiloed-data-structure": "numpy.ndarray"
+        },
+        "intercept_": {
+            "value": 152.76429169049118,
+            "np-type": "numpy.float64"
+        }
+    },
+    "sklearn_version": "1.3.0",
+    "pymilo_version": "0.9",
+    "model_type": "LinearRegression"
+}