add the download check for the taskflow (#758)

* add the download check for tht taskflow

* update the taskflow download message

* update the user generate for the paddlenlp

Author: wawltor

paddlenlp/taskflow/sentiment_analysis.py

@@ -104,19 +104,19 @@ def _construct_model(self, model):
             network = self.kwargs['network']
         if network == "bow":
             model = BoWModel(vocab_size, num_classes, padding_idx=pad_token_id)
-            model_full_name = download_file(self.model, "senta_bow.pdparams",
-                                            URLS['senta_bow'][0],
-                                            URLS['senta_bow'][1])
+            model_full_name = download_file(
+                self.model, "senta_bow.pdparams", URLS['senta_bow'][0],
+                URLS['senta_bow'][1], "sentiment_analysis")
         elif network == "lstm":
             model = LSTMModel(
                 vocab_size,
                 num_classes,
                 direction='forward',
                 padding_idx=pad_token_id,
                 pooling_type='max')
-            model_full_name = download_file(self.model, "senta_lstm.pdparams",
-                                            URLS['senta_lstm'][0],
-                                            URLS['senta_lstm'][1])
+            model_full_name = download_file(
+                self.model, "senta_lstm.pdparams", URLS['senta_lstm'][0],
+                URLS['senta_lstm'][1], "sentiment_analysis")
         else:
             raise ValueError(
                 "Unknown network: {}, it must be one of bow, lstm.".format(

paddlenlp/taskflow/text2knowledge.py

@@ -147,9 +147,9 @@ class WordTagTask(Task):
 
     def __init__(self, model, task, **kwargs):
         super().__init__(model=model, task=task, **kwargs)
-        term_schema_path = download_file(self.model, "termtree_type.csv",
-                                         URLS['termtree_type'][0],
-                                         URLS['termtree_type'][1])
+        term_schema_path = download_file(
+            self.model, "termtree_type.csv", URLS['termtree_type'][0],
+            URLS['termtree_type'][1], "text2knowledge")
         term_data_path = download_file(self.model, "TermTree.V1.0",
                                        URLS['TermTree.V1.0'][0],
                                        URLS['TermTree.V1.0'][1])

paddlenlp/taskflow/utils.py

@@ -15,7 +15,7 @@
 
 import os
 from paddle.dataset.common import md5file
-from ..utils.downloader import get_path_from_url
+from ..utils.downloader import get_path_from_url, DownloaderCheck
 from ..utils.env import MODEL_HOME
 
 DOC_FORMAT = r"""
@@ -24,7 +24,7 @@
                """
 
 
-def download_file(save_dir, filename, url, md5=None):
+def download_file(save_dir, filename, url, md5=None, task=None):
     """
     Download the file from the url to specified directory. 
     Check md5 value when the file is exists, if the md5 value is the same as the existed file, just use 
@@ -36,6 +36,7 @@ def download_file(save_dir, filename, url, md5=None):
         url(string): The url downling the file.
         md5(string, optional): The md5 value that checking the version downloaded. 
     """
+    DownloaderCheck(task).start()
     default_root = os.path.join(MODEL_HOME, save_dir)
     fullname = os.path.join(default_root, filename)
     if os.path.exists(fullname):

paddlenlp/utils/downloader.py

@@ -20,12 +20,16 @@
 import sys
 import os.path as osp
 import shutil
+import json
 import requests
 import hashlib
 import tarfile
 import zipfile
 import time
+import uuid
+import threading
 from collections import OrderedDict
+from .env import DOWNLOAD_SERVER, SUCCESS_STATUS, FAILED_STATUS
 
 try:
     from tqdm import tqdm
@@ -236,6 +240,15 @@ def _md5check(fullname, md5sum=None):
     return True
 
 
+def _md5(text):
+    """
+    Calculate the md5 value of the input text.
+    """
+
+    md5code = hashlib.md5(text.encode())
+    return md5code.hexdigest()
+
+
 def _decompress(fname):
     """
     Decompress for zip and tar file
@@ -336,3 +349,64 @@ def _is_a_single_dir(file_list):
         if file_name != new_file_list[i].split(os.sep)[0]:
             return False
     return True
+
+
+class DownloaderCheck(threading.Thread):
+    """
+    Check the resource applicability  when downloading the models.
+    """
+
+    def __init__(self, task, command="taskflow", addition=None):
+        threading.Thread.__init__(self)
+        self.command = command
+        self.task = task
+        self.addition = addition
+        self.hash_flag = _md5(str(uuid.uuid1())[9:18]) + "-" + str(
+            int(time.time()))
+
+    def uri_path(self, server_url, api):
+        srv = server_url
+        if server_url.endswith('/'):
+            srv = server_url[:-1]
+        if api.startswith('/'):
+            srv += api
+        else:
+            api = '/' + api
+            srv += api
+        return srv
+
+    def request_check(self, task, command, addition):
+        if task is None:
+            return SUCCESS_STATUS
+        payload = {'word': self.task}
+        api_url = self.uri_path(DOWNLOAD_SERVER, 'search')
+        cache_path = os.path.join("～")
+        if os.path.exists(cache_path):
+            extra = {
+                "command": self.command,
+                "mtime": os.stat(cache_path).st_mtime,
+                "hub_name": self.hash_flag
+            }
+        else:
+            extra = {
+                "command": self.command,
+                "mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+                "hub_name": self.hash_flag
+            }
+        if addition is not None:
+            extra.update({"addition": addition})
+        try:
+            import paddle
+            payload['hub_version'] = " "
+            payload['paddle_version'] = paddle.__version__.split('-')[0]
+            payload['extra'] = json.dumps(extra)
+            r = requests.get(api_url, payload, timeout=1).json()
+            if r.get("update_cache", 0) == 1:
+                return SUCCESS_STATUS
+            else:
+                return FAILED_STATUS
+        except Exception as err:
+            return FAILED_STATUS
+
+    def run(self):
+        self.request_check(self.task, self.command, self.addition)

paddlenlp/utils/env.py

@@ -51,3 +51,6 @@ def _get_sub_home(directory, parent_home=_get_ppnlp_home()):
 PPNLP_HOME = _get_ppnlp_home()
 MODEL_HOME = _get_sub_home('models')
 DATA_HOME = _get_sub_home('datasets')
+DOWNLOAD_SERVER = "http://paddlepaddle.org.cn/paddlehub"
+FAILED_STATUS = -1
+SUCCESS_STATUS = 0