Resolve race condition in data tests by using dynamic temporary directories (#1044)

Author: JingofXin

tests/data_tests/Operators/test_agentic_rag_ops.py

@@ -1,6 +1,7 @@
 import json
 import os
 import shutil
+import tempfile
 from lazyllm import config
 from lazyllm.tools.data import agenticrag
 
@@ -48,7 +49,7 @@ def share(self, prompt=None, format=None, stream=None, history=None):
 
 class TestAgenticRAGOperators:
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_basic_ops.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import tempfile
 
 from lazyllm import config, LOG
 from lazyllm.tools.data import refine, chunker, filter
@@ -27,7 +28,7 @@ def start(self):
 class TestRefineOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()
@@ -99,7 +100,7 @@ def test_refine_chained_operations(self):
 class TestChunkerOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()
@@ -131,7 +132,7 @@ def test_token_chunker(self):
 class TestFilterOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_codegen_ops.py

@@ -1,6 +1,7 @@
 import os
 import shutil
 import json
+import tempfile
 from lazyllm import config
 from lazyllm.tools.data.operators import codegen_ops
 
@@ -32,7 +33,7 @@ def __call__(self, *args, **kwargs):
 class TestCodeGenOps:
 
     def setup_method(self):
-        self.root_dir = './test_codegen_ops'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_demo_ops.py

@@ -4,6 +4,7 @@
 import pytest
 import random
 import json
+import tempfile
 from lazyllm import config, LOG
 from lazyllm.tools.data import data_register, demo1, demo2, llm_json_base
 
@@ -30,7 +31,7 @@ def start(self):
 class TestDataDemoOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()
@@ -168,7 +169,7 @@ def dummy_llm_op(data, input_key='text', output_key='llm_output'):
 class TestDataLLMDemoOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_embedding_synthesis_ops.py

@@ -1,6 +1,7 @@
 import json
 import os
 import shutil
+import tempfile
 import numpy as np
 from lazyllm import config
 from lazyllm.tools.data import embedding
@@ -64,7 +65,7 @@ def __call__(self, texts):
 
 class TestEmbeddingSynthesisOperators:
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_generate_cot_ops.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import tempfile
 
 from lazyllm import config
 from lazyllm.tools.data import genCot
@@ -27,7 +28,7 @@ def start(self):
 class TestGenCotOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_knowledge_cleaning_ops.py

@@ -47,7 +47,7 @@ def share(self, prompt=None, format=None, stream=None, history=None):
 
 class TestKnowledgeCleaningOperators:
     def setup_method(self):
-        self.root_dir = './test_kbc_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_math_qa_ops.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import tempfile
 
 from lazyllm import config
 from lazyllm.tools.data import MathQA
@@ -27,7 +28,7 @@ def start(self):
 class TestMathQAOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_preference_ops.py

@@ -1,8 +1,9 @@
 import os
 import shutil
+import tempfile
+
 from lazyllm import config
 from lazyllm.tools.data.operators import preference_ops
-import pytest  # noqa: F401
 
 class TestPreferenceOperators:
     class MockModel:
@@ -19,7 +20,7 @@ def __call__(self, x, **kwargs):
             return self.return_val
 
     def setup_method(self):
-        self.root_dir = './test_preference_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()

tests/data_tests/Operators/test_pretrain_ops.py

@@ -31,7 +31,7 @@ def start(self):
 class TestPretrainOperators:
 
     def setup_method(self):
-        self.root_dir = './test_data_op'
+        self.root_dir = tempfile.mkdtemp()
         self.keep_dir = config['data_process_path']
         os.environ['LAZYLLM_DATA_PROCESS_PATH'] = self.root_dir
         config.refresh()