Could not find CUDA driver entry point for cuCtxGetCurrent #235
Description
Run singleview generate CMD:
PYTHONPATH="cosmos-transfer1" python scripts/generate_video_single_view.py
--caption_path outputs/captions
--input_path outputs
--video_save_folder outputs/single_view
--checkpoint_dir checkpoints/
--is_av_sample
--controlnet_specs assets/sample_av_hdmap_spec.json
Output:
RuntimeError:
Traceback (most recent call last):
File "Cosmos-Drive-Dreams-main/scripts/generate_video_single_view.py", line 369, in
demo(cfg, pipeline, current_control_inputs, video_save_name, prompt,)
File "Cosmos-Drive-Dreams-main/scripts/generate_video_single_view.py", line 220, in demo
generated_output = pipeline.generate(
^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/inference/world_generation_pipeline.py", line 731, in generate
videos = self._run_model_with_offload(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/inference/world_generation_pipeline.py", line 437, in _run_model_with_offload
samples = self._run_model(prompt_embeddings=prompt_embeddings, negative_prompt_embeddings=negative_prompt_embeddings, video_paths=video_paths, control_inputs_list=control_inputs_list,)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/inference/world_generation_pipeline.py", line 588, in _run_model
latents = generate_world_from_control(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/inference/inference_utils.py", line 733, in generate_world_from_control
sample = model.generate_samples_from_batch(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/model/model_ctrl.py", line 672, in generate_samples_from_batch
samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=sigma_max)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/diffusion/modules/res_sampler.py", line 151, in forward
return self._forward_impl(float64_x0_fn, x_sigma_max, sampler_cfg).to(in_dtype)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/diffusion/modules/res_sampler.py", line 180, in _forward_impl
denoised_output = differential_equation_solver(denoiser_fn, sigmas_L, sampler_cfg.solver, callback_fns=callback_fns)(noisy_input_B_StateShape)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/diffusion/modules/res_sampler.py", line 278, in sample_fn
x_at_eps, _ = fori_loop(0, num_step, step_fn, [input_xT_B_StateShape, None])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/diffusion/modules/res_sampler.py", line 205, in fori_loop
val = body_fun(i, val)
^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/diffusion/modules/res_sampler.py", line 263, in step_fn
x0_pred_B_StateShape = x0_fn(input_x_B_StateShape, sigma_cur_0 * ones_B)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/diffusion/modules/res_sampler.py", line 133, in float64_x0_fn
return x0_fn(x_B_StateShape.to(in_dtype), t_B.to(in_dtype)).to(torch.float64)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/model/model_ctrl.py", line 599, in x0_fn
cond_x0 = self.denoise(
^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/model/model_t2w.py", line 171, in denoise
net_output = self.net(
^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/networks/general_dit_ctrl_enc.py", line 299, in forward
x = block(
^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/module/blocks.py", line 567, in forward
x = block(
^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/module/blocks.py", line 473, in forward
x = x + gate_1_1_1_B_D * self.block(
^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/module/blocks.py", line 334, in forward
x_THW_B_D = self.attn(
^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/module/attention.py", line 461, in forward
return self.cal_attn(q, k, v, mask)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Cosmos-Drive-Dreams-main/cosmos-transfer1/cosmos_transfer1/diffusion/module/attention.py", line 435, in cal_attn
out = self.attn_op(q, k, v, core_attention_bias_type="no_bias", core_attention_bias=None) # [B, Mq, H, V]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/transformer_engine/pytorch/attention/dot_product_attention/dot_product_attention.py", line 1367, in forward
) = dpa_utils.get_attention_backend(attention_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/cosmos-transfer1/lib/python3.12/site-packages/transformer_engine/pytorch/attention/dot_product_attention/utils.py", line 984, in get_attention_backend
fused_attention_backend = tex.get_fused_attn_backend(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: 3rd_library0112/TransformerEngine_release_v2.10/transformer_engine/common/util/cuda_driver.cpp:42 in function get_symbol: Assertion failed: driver_result == cudaDriverEntryPointSuccess. Could not find CUDA driver entry point for cuCtxGetCurrent
environment setup:
conda python 3.12.12
cuda 12.4 torch 2.6.0+cu124
megatron-core 0.15.2
transformer_engine 2.10.0