[evaluation] use thread lock for nltk data download to avoid race condition (#37487)

* use threadlock for nltk data download

* update

Author: ninghu

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py

@@ -13,7 +13,11 @@
 
 from typing import List
 
+import threading
 import numpy as np
+import nltk
+
+_nltk_data_download_lock = threading.Lock()
 
 
 def get_harm_severity_level(harm_score: int) -> str:
@@ -38,21 +42,24 @@ def get_harm_severity_level(harm_score: int) -> str:
     return np.nan
 
 
-def nltk_tokenize(text: str) -> List[str]:
-    """Tokenize the input text using the NLTK tokenizer."""
+def ensure_nltk_data_downloaded():
+    """Download NLTK data packages if not already downloaded."""
+    with _nltk_data_download_lock:
+        try:
+            from nltk.tokenize.nist import NISTTokenizer
+        except LookupError:
+            nltk.download("perluniprops")
+            nltk.download("punkt")
+            nltk.download("punkt_tab")
 
-    import nltk
 
-    try:
-        from nltk.tokenize.nist import NISTTokenizer
-    except LookupError:
-        nltk.download("perluniprops")
-        nltk.download("punkt")
-        nltk.download("punkt_tab")
-        from nltk.tokenize.nist import NISTTokenizer
+def nltk_tokenize(text: str) -> List[str]:
+    """Tokenize the input text using the NLTK tokenizer."""
+    ensure_nltk_data_downloaded()
 
     if not text.isascii():
         # Use NISTTokenizer for international tokenization
+        from nltk.tokenize.nist import NISTTokenizer
         tokens = NISTTokenizer().international_tokenize(text)
     else:
         # By default, use NLTK word tokenizer

sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py

@@ -33,10 +33,6 @@
 @pytest.mark.usefixtures("recording_injection", "recorded_test")
 @pytest.mark.localtest
 class TestBuiltInEvaluators:
-    @pytest.mark.skipif(
-        condition=platform.python_implementation() == "PyPy",
-        reason="Temporary skip to merge 37201, will re-enable in subsequent pr",
-    )
     def test_math_evaluator_bleu_score(self):
         eval_fn = BleuScoreEvaluator()
         score = eval_fn(