fix import download_utils & support ci set network proxy (#2477)

Author: fjjF77

paddleformers/transformers/__init__.py

@@ -18,6 +18,8 @@
 from typing import TYPE_CHECKING
 from ..utils.lazy_import import _LazyModule
 
+from .download_utils import *
+
 # from .auto.modeling import AutoModelForCausalLM
 import_structure = {
     "kto_criterion": [

paddleformers/utils/download/download.py

@@ -28,7 +28,12 @@
     RepositoryNotFoundError,
     RevisionNotFoundError,
 )
-from paddle import __version__
+
+try:
+    from paddle import __version__
+except ImportError:
+    __version__ = ""
+
 from requests import HTTPError
 
 from ..log import logger

tests/dataset/test_zero_padding.py

@@ -74,9 +74,7 @@ def preprocess_fn(
         return_attention_mask=True,
     ):
         inputs = example["sentence"][:2]
-        model_inputs = self.tokenizer(
-            inputs, max_length=max_src_length, truncation=True, return_attention_mask=False, return_position_ids=False
-        )
+        model_inputs = self.tokenizer(inputs, max_length=max_src_length, truncation=True, return_attention_mask=False)
         labels_input_ids = model_inputs["input_ids"] + [self.tokenizer.eos_token_id]
         model_inputs["labels"] = [-100] * len(model_inputs["input_ids"]) + labels_input_ids
         model_inputs["input_ids"] = model_inputs["input_ids"] + labels_input_ids

tests/testing_utils.py

@@ -30,6 +30,7 @@
 import yaml
 
 from paddleformers.trainer.argparser import strtobool
+from paddleformers.utils.download import DownloadSource
 from paddleformers.utils.import_utils import is_package_available, is_paddle_available
 
 __all__ = ["get_vocab_list", "stable_softmax", "cross_entropy"]
@@ -539,3 +540,53 @@ def init_dist_env(self, config: dict = {}):
 
         fleet.init(is_collective=True, strategy=strategy)
         fleet.get_hybrid_communicate_group()
+
+
+def set_proxy(download_hub: DownloadSource = None):
+    """
+    set network proxy for downloading model from aistudio/huggingface/modelscope
+    """
+
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            if download_hub is None:
+                return func(*args, **kwargs)
+            elif download_hub == DownloadSource.HUGGINGFACE:
+                command = "source $work_dir/../../../proxy_hf && env"
+            elif download_hub == DownloadSource.AISTUDIO:
+                command = "source $work_dir/../../../proxy_aistudio && env"
+            elif download_hub == DownloadSource.MODELSCOPE:
+                command = "source $work_dir/../../../proxy_aistudio && env"  # proxy_aistudio also suit for modelscope
+
+            proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
+            out, _ = proc.communicate()
+
+            proxy_env = {}
+            for line in out.decode().splitlines():
+                if "=" not in line:
+                    continue
+                key, _, value = line.partition("=")
+                proxy_env[key] = value
+
+            ori_env = {}
+            proxy_vars = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"]
+            if download_hub == DownloadSource.AISTUDIO:
+                proxy_vars.extend(["STUDIO_GIT_HOST", "STUDIO_CDN_HOST"])
+
+            for key in proxy_vars:
+                if key in proxy_env:
+                    ori_env[key] = os.environ.get(key, "")
+                    os.environ[key] = proxy_env[key]
+
+            try:
+                return func(*args, **kwargs)
+            finally:
+                for key, old_value in ori_env.items():
+                    if old_value is None:
+                        os.environ.pop(key, None)
+                    else:
+                        os.environ[key] = old_value
+
+        return wrapper
+
+    return decorator

tests/transformers/auto/test_confiugration.py

@@ -23,7 +23,9 @@
 from paddleformers.transformers import AutoConfig
 from paddleformers.transformers.auto.configuration import CONFIG_MAPPING
 from paddleformers.transformers.bert.configuration import BertConfig
+from paddleformers.utils.download import DownloadSource
 from paddleformers.utils.env import CONFIG_NAME
+from tests.testing_utils import set_proxy
 
 from ...utils.test_module.custom_configuration import CustomConfig
 
@@ -66,17 +68,20 @@ def test_community_model_class(self):
             self.assertEqual(auto_config.hidden_size, number)
 
     @unittest.skip("skipping due to connection error!")
+    # @set_proxy(DownloadSource.HUGGINGFACE)
     def test_from_hf_hub(self):
-        config = AutoConfig.from_pretrained("facebook/opt-66b", download_hub="huggingface")
-        self.assertEqual(config.hidden_size, 9216)
+        config = AutoConfig.from_pretrained("dfargveazd/tiny-random-llama-paddle", download_hub="huggingface")
+        self.assertEqual(config.hidden_size, 192)
 
-    @unittest.skip("skipping due to connection error!")
+    # @unittest.skip("skipping due to connection error!")
+    @set_proxy(DownloadSource.AISTUDIO)
     def test_from_aistudio(self):
         config = AutoConfig.from_pretrained("test_paddleformers/tiny-random-llama", download_hub="aistudio")
-        self.assertEqual(config.hidden_size, 768)
+        self.assertEqual(config.hidden_size, 192)
 
-    @unittest.skip("skipping due to connection error!")
-    def test_from_mdoelscope(self):
+    # @unittest.skip("skipping due to connection error!")
+    @set_proxy(DownloadSource.MODELSCOPE)
+    def test_from_modelscope(self):
         config = AutoConfig.from_pretrained("sqlhuman/tiny-random-llama", download_hub="modelscope")
         self.assertEqual(config.hidden_size, 768)
 

tests/transformers/auto/test_modeling.py

@@ -33,7 +33,9 @@
 )
 from paddleformers.transformers.auto.configuration import CONFIG_MAPPING
 from paddleformers.transformers.auto.modeling import MODEL_MAPPING
+from paddleformers.utils.download import DownloadSource
 from paddleformers.utils.env import CONFIG_NAME, PADDLE_WEIGHTS_NAME
+from tests.testing_utils import set_proxy
 
 from ...utils.test_module.custom_configuration import CustomConfig
 from ...utils.test_module.custom_model import CustomModel
@@ -74,16 +76,19 @@ def test_model_from_pretrained_cache_dir(self):
             self.assertFalse(os.path.exists(os.path.join(tempdir, model_name, model_name)))
 
     @unittest.skip("skipping due to connection error!")
+    # @set_proxy(DownloadSource.HUGGINGFACE)
     def test_from_hf_hub(self):
-        model = AutoModel.from_pretrained("dfargveazd/tiny-random-llama", download_hub="huggingface")
+        model = AutoModel.from_pretrained("dfargveazd/tiny-random-llama-paddle", download_hub="huggingface")
         self.assertIsInstance(model, LlamaModel)
 
-    @unittest.skip("skipping due to connection error!")
+    # @unittest.skip("skipping due to connection error!")
+    @set_proxy(DownloadSource.AISTUDIO)
     def test_from_aistudio(self):
         model = AutoModel.from_pretrained("test_paddleformers/tiny-random-llama", download_hub="aistudio")
         self.assertIsInstance(model, LlamaModel)
 
-    @unittest.skip("skipping due to connection error!")
+    # @unittest.skip("skipping due to connection error!")
+    @set_proxy(DownloadSource.MODELSCOPE)
     def test_from_modelscope(self):
         model = AutoModel.from_pretrained("sqlhuman/tiny-random-llama", download_hub="modelscope")
         self.assertIsInstance(model, LlamaModel)

tests/transformers/test_configuration_utils.py

@@ -153,6 +153,7 @@ def test_from_pretrained_cache_dir(self):
             self.assertFalse(os.path.exists(os.path.join(tempdir, model_id, model_id)))
 
     @unittest.skip("skipping due to connection error!")
+    # @set_proxy(DownloadSource.HUGGINGFACE)
     def test_load_from_hf(self):
         """test load config from hf"""
         config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-BertModel", download_hub="huggingface")