Skip to content

Commit 495645b

Browse files
fjjF77fjjF77
authored
support ci which download models from hugging face (#2490)
1 parent 48387c8 commit 495645b

File tree

12 files changed

+75
-923
lines changed

12 files changed

+75
-923
lines changed

‎.github/workflows/unittest-cpu.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ jobs:
3939
-e work_dir \
4040
-e no_proxy \
4141
-e python_version \
42+
-e HF_PROXY_PATH=$work_dir/../../../proxy_huggingface \
43+
-e AISTUDIO_PROXY_PATH=$work_dir/../../../proxy_aistudio \
4244
-w /workspace ${docker_image}
4345
4446
- name: Download Code

‎.github/workflows/unittest-gpu.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ jobs:
7474
-e paddle_whl \
7575
-e FLAGS_dynamic_static_unified_comm \
7676
-e python_version \
77+
-e HF_PROXY_PATH=$work_dir/../../../proxy_huggingface \
78+
-e AISTUDIO_PROXY_PATH=$work_dir/../../../proxy_aistudio \
7779
-w /workspace --runtime=nvidia $IMAGE_NAME
7880
fi
7981

‎paddleformers/transformers/download_utils.py‎

Lines changed: 31 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -711,57 +711,58 @@
711711
}
712712
)
713713

714+
# ernie
714715
register_model_group(
715716
models={
716717
"ERNIE-4.5-300B-A47B-Base": {
717-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-300B-A47B-Base-Paddle",
718-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-300B-A47B-Base-Paddle",
719-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-300B-A47B-Base-Paddle",
718+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-300B-A47B-Base-PT",
719+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-300B-A47B-Base-PT",
720+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-300B-A47B-Base-PT",
720721
},
721722
"ERNIE-4.5-300B-A47B": {
722-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-300B-A47B-Paddle",
723-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-300B-A47B-Paddle",
724-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-300B-A47B-Paddle",
723+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-300B-A47B-PT",
724+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-300B-A47B-PT",
725+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-300B-A47B-PT",
725726
},
726727
"ERNIE-4.5-21B-A3B-Base": {
727-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-21B-A3B-Base-Paddle",
728-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-21B-A3B-Base-Paddle",
729-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-21B-A3B-Base-Paddle",
728+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-21B-A3B-Base-PT",
729+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-21B-A3B-Base-PT",
730+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-21B-A3B-Base-PT",
730731
},
731732
"ERNIE-4.5-21B-A3B": {
732-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-21B-A3B-Paddle",
733-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-21B-A3B-Paddle",
734-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-21B-A3B-Paddle",
733+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-21B-A3B-PT",
734+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-21B-A3B-PT",
735+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-21B-A3B-PT",
735736
},
736737
"ERNIE-4.5-0.3B-Base": {
737-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-0.3B-Base-Paddle",
738-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-0.3B-Base-Paddle",
739-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-0.3B-Base-Paddle",
738+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-0.3B-Base-PT",
739+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-0.3B-Base-PT",
740+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-0.3B-Base-PT",
740741
},
741742
"ERNIE-4.5-0.3B": {
742-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-0.3B-Paddle",
743-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-0.3B-Paddle",
744-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-0.3B-Paddle",
743+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-0.3B-PT",
744+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-0.3B-PT",
745+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-0.3B-PT",
745746
},
746747
"ERNIE-4.5-VL-424B-A47B-Base": {
747-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-424B-A47B-Base-Paddle",
748-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-Paddle",
749-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-Paddle",
748+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-424B-A47B-Base-PT",
749+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-PT",
750+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-PT",
750751
},
751752
"ERNIE-4.5-VL-424B": {
752-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-424B-Paddle",
753-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-424B-Paddle",
754-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-424B-Paddle",
753+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-424B-PT",
754+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-424B-PT",
755+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-424B-PT",
755756
},
756757
"ERNIE-4.5-VL-28B-A3B-Base": {
757-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-28B-A3B-Base-Paddle",
758-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-Paddle",
759-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-Paddle",
758+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-28B-A3B-Base-PT",
759+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-PT",
760+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-PT",
760761
},
761762
"ERNIE-4.5-VL-28B-A3B": {
762-
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-28B-A3B-Paddle",
763-
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle",
764-
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle",
763+
DownloadSource.HUGGINGFACE: "baidu/ERNIE-4.5-VL-28B-A3B-PT",
764+
DownloadSource.AISTUDIO: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-PT",
765+
DownloadSource.MODELSCOPE: "PaddlePaddle/ERNIE-4.5-VL-28B-A3B-PT",
765766
},
766767
}
767768
)

‎tests/testing_utils.py‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -552,11 +552,28 @@ def wrapper(*args, **kwargs):
552552
if download_hub is None:
553553
return func(*args, **kwargs)
554554
elif download_hub == DownloadSource.HUGGINGFACE:
555-
command = "source $work_dir/../../../proxy_hf && env"
555+
if "HF_PROXY_PATH" not in os.environ:
556+
print(
557+
"`HF_PROXY_PATH` environment variable does not defined before using `set_proxy`, please define it first"
558+
)
559+
proxy_path = os.path.abspath(os.environ["HF_PROXY_PATH"])
556560
elif download_hub == DownloadSource.AISTUDIO:
557-
command = "source $work_dir/../../../proxy_aistudio && env"
561+
if "AISTUDIO_PROXY_PATH" not in os.environ:
562+
print(
563+
"`AISTUDIO_PROXY_PATH` environment variable does not defined before using `set_proxy`, please define it first"
564+
)
565+
proxy_path = os.path.abspath(os.environ["AISTUDIO_PROXY_PATH"])
558566
elif download_hub == DownloadSource.MODELSCOPE:
559-
command = "source $work_dir/../../../proxy_aistudio && env" # proxy_aistudio also suit for modelscope
567+
if "AISTUDIO_PROXY_PATH" not in os.environ:
568+
print(
569+
"`AISTUDIO_PROXY_PATH` environment variable does not defined before using `set_proxy`, please define it first"
570+
)
571+
proxy_path = os.path.abspath(
572+
os.environ["AISTUDIO_PROXY_PATH"]
573+
) # proxy_aistudio also suit for modelscope
574+
575+
print(f"set proxy for {download_hub}, proxy path: {proxy_path}")
576+
command = f"source {proxy_path} && env"
560577

561578
proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
562579
out, _ = proc.communicate()
@@ -569,10 +586,9 @@ def wrapper(*args, **kwargs):
569586
proxy_env[key] = value
570587

571588
ori_env = {}
572-
proxy_vars = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"]
589+
proxy_vars = ["http_proxy", "https_proxy", "no_proxy"]
573590
if download_hub == DownloadSource.AISTUDIO:
574591
proxy_vars.extend(["STUDIO_GIT_HOST", "STUDIO_CDN_HOST"])
575-
576592
for key in proxy_vars:
577593
if key in proxy_env:
578594
ori_env[key] = os.environ.get(key, "")

‎tests/transformers/auto/test_configuration.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,19 +67,16 @@ def test_community_model_class(self):
6767
auto_config = AutoConfig.from_pretrained(tempdir)
6868
self.assertEqual(auto_config.hidden_size, number)
6969

70-
@unittest.skip("skipping due to connection error!")
71-
# @set_proxy(DownloadSource.HUGGINGFACE)
70+
@set_proxy(DownloadSource.HUGGINGFACE)
7271
def test_from_hf_hub(self):
73-
config = AutoConfig.from_pretrained("dfargveazd/tiny-random-llama-paddle", download_hub="huggingface")
72+
config = AutoConfig.from_pretrained("dfargveazd/tiny-random-llama-paddle-safe", download_hub="huggingface")
7473
self.assertEqual(config.hidden_size, 16)
7574

76-
# @unittest.skip("skipping due to connection error!")
7775
@set_proxy(DownloadSource.AISTUDIO)
7876
def test_from_aistudio(self):
7977
config = AutoConfig.from_pretrained("test_paddleformers/tiny-random-llama", download_hub="aistudio")
8078
self.assertEqual(config.hidden_size, 16)
8179

82-
# @unittest.skip("skipping due to connection error!")
8380
@set_proxy(DownloadSource.MODELSCOPE)
8481
def test_from_modelscope(self):
8582
config = AutoConfig.from_pretrained("sqlhuman/tiny-random-llama", download_hub="modelscope")

‎tests/transformers/auto/test_modeling.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,12 @@ def test_model_from_pretrained_cache_dir(self):
7575
# check against double appending model_name in cache_dir
7676
self.assertFalse(os.path.exists(os.path.join(tempdir, model_name, model_name)))
7777

78-
@unittest.skip("skipping due to connection error!")
79-
# @set_proxy(DownloadSource.HUGGINGFACE)
78+
# @unittest.skip("skipping due to connection error!")
79+
@set_proxy(DownloadSource.HUGGINGFACE)
8080
def test_from_hf_hub(self):
81-
model = AutoModel.from_pretrained("dfargveazd/tiny-random-llama-paddle", download_hub="huggingface")
81+
model = AutoModel.from_pretrained(
82+
"dfargveazd/tiny-random-llama-paddle-safe", download_hub="huggingface", convert_from_hf=False
83+
)
8284
self.assertIsInstance(model, LlamaModel)
8385

8486
# @unittest.skip("skipping due to connection error!")

‎tests/transformers/load_subfolder/test_config.py‎

Lines changed: 0 additions & 47 deletions
This file was deleted.

‎tests/transformers/load_subfolder/test_image_processor.py‎

Lines changed: 0 additions & 69 deletions
This file was deleted.

0 commit comments

Comments
 (0)