Fix taskflow tests and the FileLock import bug (#3898)

* fix taskflow tests

* add copyright

* pre-commit

* use md5 for lock-file name

* fix imports

* fix gpu test

* reemove print statement

Co-authored-by: wj-Mcat <[email protected]>

Author: sijunhe

paddlenlp/utils/downloader.py

@@ -11,21 +11,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import hashlib
+import json
 import os
-import sys
 import os.path as osp
-from typing import Optional
 import shutil
-import json
-import requests
-import hashlib
+import sys
 import tarfile
-import zipfile
+import threading
 import time
 import uuid
-import threading
+import zipfile
 from collections import OrderedDict
-from .env import DOWNLOAD_SERVER, SUCCESS_STATUS, FAILED_STATUS
+from typing import Optional
+
+import requests
+
+from .env import DOWNLOAD_SERVER, FAILED_STATUS, LOCK_FILE_HOME, SUCCESS_STATUS
+from .file_lock import FileLock
 
 try:
     from tqdm import tqdm
@@ -117,7 +120,7 @@ def get_weights_path_from_url(url, md5sum=None):
     Args:
         url (str): download url
         md5sum (str): md5 sum of download package
-    
+
     Returns:
         str: a local path to save downloaded weights.
     Examples:
@@ -147,7 +150,7 @@ def get_path_from_url(url, root_dir, md5sum=None, check_exist=True):
         root_dir (str): root dir for downloading, it should be
                         WEIGHTS_HOME or DATASET_HOME
         md5sum (str): md5 sum of download package
-    
+
     Returns:
         str: a local path to save downloaded models & weights & datasets.
     """
@@ -192,11 +195,13 @@ def get_path_from_url_with_filelock(url: str,
     os.makedirs(root_dir, exist_ok=True)
 
     # create lock file, which is empty, under the `LOCK_FILE_HOME` directory.
-    lock_file_path = os.path.join(LOCK_FILE_HOME,
-                                  f"{str(hash(url + root_dir))}")
+    lock_file_name = hashlib.md5((url + root_dir).encode("utf-8")).hexdigest()
+    lock_file_path = os.path.join(LOCK_FILE_HOME, lock_file_name)
+
     with FileLock(lock_file_path):
         # import get_path_from_url from paddle framework
-        from paddle.utils.download import get_path_from_url as _get_path_from_url
+        from paddle.utils.download import \
+            get_path_from_url as _get_path_from_url
         result = _get_path_from_url(url=url,
                                     root_dir=root_dir,
                                     md5sum=md5sum,
@@ -435,6 +440,7 @@ def request_check(self, task, command, addition):
             extra.update({"addition": addition})
         try:
             import paddle
+
             import paddlenlp
             payload['hub_version'] = " "
             payload['ppnlp_version'] = paddlenlp.__version__

tests/taskflow/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/taskflow/test_text_classification.py

@@ -52,7 +52,7 @@ def tearDown(self):
         self.temp_dir.cleanup()
 
     @parameterized.expand([(1, ), (2, )])
-    def test_text_classification_taskf(self, batch_size):
+    def test_text_classification_task(self, batch_size):
         # input_text is a tuple to simulate the args passed from Taskflow to TextClassificationTask
         input_text = (["百度", "深度学习框架", "飞桨", "PaddleNLP"], )
         id2label = {
@@ -64,7 +64,8 @@ def test_text_classification_taskf(self, batch_size):
             task="text_classification",
             task_path=self.dygraph_model_path,
             id2label=id2label,
-            batch_size=batch_size)
+            batch_size=batch_size,
+            device_id=0)
 
         dygraph_results = dygraph_taskflow(input_text)
         self.assertEqual(len(dygraph_results), len(input_text[0]))
@@ -75,7 +76,8 @@ def test_text_classification_taskf(self, batch_size):
             is_static_model=True,
             task_path=self.static_model_path,
             id2label=id2label,
-            batch_size=batch_size)
+            batch_size=batch_size,
+            device_id=0)
 
         static_results = static_taskflow(input_text)
         self.assertEqual(len(static_results), len(input_text[0]))

tests/transformers/test_modeling_utils.py

@@ -55,14 +55,15 @@ def test_multiprocess_downloading(self):
         num_process_in_pool, num_jobs = 10, 20
         small_model_path = "https://paddlenlp.bj.bcebos.com/models/community/__internal_testing__/bert/model_state.pdparams"
 
-        from paddlenlp.transformers.model_utils import get_path_from_url
+        from paddlenlp.transformers.model_utils import get_path_from_url_with_filelock
         with TemporaryDirectory() as tempdir:
 
             with Pool(num_process_in_pool) as pool:
-                pool.starmap(get_path_from_url, [(small_model_path, tempdir)
-                                                 for _ in range(num_jobs)])
+                pool.starmap(get_path_from_url_with_filelock,
+                             [(small_model_path, tempdir)
+                              for _ in range(num_jobs)])
 
-    # @slow
+    @slow
     def test_model_from_pretrained_with_multiprocessing(self):
         """
         this test can not init tooooo many models which will occupy CPU/GPU memorys.