|
21 | 21 |
|
22 | 22 | import pytest |
23 | 23 |
|
24 | | -# --------------------------------------------------------------------------- |
25 | | -# vLLM 0.14+ spawns its EngineCore via fork() by default. If any prior code |
26 | | -# (e.g. HF AutoProcessor) has created threads, the forked child inherits |
27 | | -# their lock state and deadlocks. Forcing spawn avoids this entirely. |
28 | | -# TOKENIZERS_PARALLELISM=false prevents HF fast-tokenizer threads from being |
29 | | -# created in the first place (belt-and-suspenders). |
30 | | -# Both must be set before any import that might start threads. |
31 | | -# --------------------------------------------------------------------------- |
32 | | -os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") |
33 | | -os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") |
34 | | -_custom_hf = os.environ.get("CUSTOM_HF_DATASET", "") |
35 | | -if _custom_hf: |
36 | | - os.environ.setdefault("HF_HOME", _custom_hf) |
37 | | - |
38 | 24 | # --------------------------------------------------------------------------- |
39 | 25 | # Override the session-level autouse Ray cluster fixture from the root conftest. |
40 | 26 | # Integration tests call stages directly — no Ray pipeline needed. |
@@ -87,6 +73,16 @@ def enhancement_stage(): |
87 | 73 |
|
88 | 74 | setup() loads vLLM's LLM() with Qwen2.5-14B-Instruct via HF auto-download. |
89 | 75 | """ |
| 76 | + # Set env vars here (not module-level) so they don't affect other GPU tests |
| 77 | + # collected in the same pytest session (e.g. tests/core/test_serve.py). |
| 78 | + # vLLM 0.14+ spawns its EngineCore via fork() by default; forcing spawn |
| 79 | + # avoids deadlocks when HF AutoProcessor has already created threads. |
| 80 | + os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") |
| 81 | + os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") |
| 82 | + _custom_hf = os.environ.get("CUSTOM_HF_DATASET", "") |
| 83 | + if _custom_hf: |
| 84 | + os.environ.setdefault("HF_HOME", _custom_hf) |
| 85 | + |
90 | 86 | from nemo_curator.stages.video.caption.caption_enhancement import CaptionEnhancementStage |
91 | 87 |
|
92 | 88 | model_dir = os.environ.get("CURATOR_TEST_MODEL_DIR", "") |
|
0 commit comments