使用昇腾npu训练拓展模型,出现报错 #4901
Unanswered
xiaoyaoyouh
asked this question in
Q&A
Replies: 2 comments
-
|
相关配置: |
Beta Was this translation helpful? Give feedback.
0 replies
-
|
https://quay.io/repository/ascend/verl?tab=tags&tag=latest |
Beta Was this translation helpful? Give feedback.
0 replies
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Uh oh!
There was an error while loading. Please reload this page.
-
Traceback (most recent call last):
File "/media/t1/gjq123/verl/verl/trainer/main_ppo.py", line 45, in main
run_ppo(config)
File "/media/t1/gjq123/verl/verl/trainer/main_ppo.py", line 99, in run_ppo
ray.get(runner.run.remote(config))
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(*args, **kwargs)
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/site-packages/ray/_private/worker.py", line 2967, in get
values, debugger_breakpoint = worker.get_objects(
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/site-packages/ray/_private/worker.py", line 1015, in get_objects
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(ImportError): ray::TaskRunner.run() (pid=1273709, ip=192.168.0.140, actor_id=73e484a0a654932a78182b6301000000, repr=<main_ppo.TaskRunner object at 0xffcf91af5b40>)
File "/media/t1/gjq123/verl/verl/trainer/main_ppo.py", line 359, in run
trainer.init_workers()
File "/media/t1/gjq123/verl/verl/trainer/ppo/ray_trainer.py", line 958, in init_workers
self.actor_rollout_wg.init_model()
File "/media/t1/gjq123/verl/verl/single_controller/ray/base.py", line 54, in call
output = ray.get(output)
ray.exceptions.RayTaskError(ImportError): ray::WorkerDict.actor_rollout_init_model() (pid=1275501, ip=192.168.0.140, actor_id=e2ebf9b29349f81297021be401000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0xffcfb5bc0d90>)
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/concurrent/futures/_base.py", line 458, in result
return self.__get_result()
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
raise self._exception
File "/media/t1/gjq123/verl/verl/single_controller/ray/base.py", line 844, in func
return getattr(self.worker_dict[key], name)(*args, **kwargs)
File "/media/t1/gjq123/verl/verl/single_controller/base/decorator.py", line 462, in inner
return func(*args, **kwargs)
File "/media/t1/gjq123/verl/verl/utils/transferqueue_utils.py", line 314, in dummy_inner
output = func(*args, **kwargs)
File "/media/t1/gjq123/verl/verl/workers/fsdp_workers.py", line 847, in init_model
self._build_rollout(trust_remote_code=self.config.model.get("trust_remote_code", False))
File "/media/t1/gjq123/verl/verl/workers/fsdp_workers.py", line 650, in _build_rollout
self.rollout = get_rollout_class(rollout_config.name, rollout_config.mode)(
File "/media/t1/gjq123/verl/verl/workers/rollout/base.py", line 100, in get_rollout_class
rollout_module = importlib.import_module(module_name)
File "/root/miniconda3/envs/MindSpeed/lib/python3.10/importlib/init.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "", line 1050, in _gcd_import
File "", line 1027, in _find_and_load
File "", line 1006, in _find_and_load_unlocked
File "", line 688, in _load_unlocked
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "/media/t1/gjq123/verl/verl/workers/rollout/vllm_rollout/init.py", line 17, in
from .vllm_rollout import vLLMAsyncRollout # noqa: F401
File "/media/t1/gjq123/verl/verl/workers/rollout/vllm_rollout/vllm_rollout.py", line 57, in
from verl.third_party.vllm import VLLM_SLEEP_LEVEL, get_version
File "/media/t1/gjq123/verl/verl/third_party/vllm/init.py", line 44, in
from vllm import LLM
ImportError: cannot import name 'LLM' from 'vllm' (unknown location)
Beta Was this translation helpful? Give feedback.
All reactions