[evaluation] ci,tests,fix: Improve reliability of nltk data download in CI (Azure#38059)

* refactor: Move meteor nltk data init to ensure_nltk_data_downloaded

* tests: Add a fixture that ensures that nltk data is downloaded

  Allows us to fail fast if something goes wrong

* style: Run isort

Author: kdestin

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py

@@ -48,13 +48,19 @@ def get_harm_severity_level(harm_score: int) -> Union[str, float]:
 
 def ensure_nltk_data_downloaded():
     """Download NLTK data packages if not already downloaded."""
+    nltk_data = [
+        ("wordnet", "corpora/wordnet.zip"),
+        ("perluniprops", "misc/perluniprops.zip"),
+        ("punkt", "tokenizers/punkt.zip"),
+        ("punkt_tab", "tokenizers/punkt_tab.zip"),
+    ]
+
     with _nltk_data_download_lock:
-        try:
-            from nltk.tokenize.nist import NISTTokenizer  # pylint: disable=unused-import
-        except LookupError:
-            nltk.download("perluniprops")
-            nltk.download("punkt")
-            nltk.download("punkt_tab")
+        for _id, resource_name in nltk_data:
+            try:
+                nltk.find(resource_name)
+            except LookupError:
+                nltk.download(_id)
 
 
 def nltk_tokenize(text: str) -> List[str]:

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_meteor/_meteor.py

@@ -1,11 +1,10 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-import nltk
 from nltk.translate.meteor_score import meteor_score
 from promptflow._utils.async_utils import async_run_allowing_running_loop
 
-from azure.ai.evaluation._common.utils import nltk_tokenize
+from azure.ai.evaluation._common.utils import nltk_tokenize, ensure_nltk_data_downloaded
 
 
 class _AsyncMeteorScoreEvaluator:
@@ -14,10 +13,7 @@ def __init__(self, alpha: float = 0.9, beta: float = 3.0, gamma: float = 0.5):
         self._beta = beta
         self._gamma = gamma
 
-        try:
-            nltk.find("corpora/wordnet.zip")
-        except LookupError:
-            nltk.download("wordnet")
+        ensure_nltk_data_downloaded()
 
     async def __call__(self, *, ground_truth: str, response: str, **kwargs):
         reference_tokens = nltk_tokenize(ground_truth)

sdk/evaluation/azure-ai-evaluation/tests/conftest.py

@@ -9,22 +9,27 @@
 from unittest.mock import patch
 
 import pytest
-from azure.core.credentials import TokenCredential
+from ci_tools.variables import in_ci
 from devtools_testutils import add_body_key_sanitizer, add_general_regex_sanitizer, add_header_regex_sanitizer, is_live
 from devtools_testutils.config import PROXY_URL
 from devtools_testutils.fake_credentials import FakeTokenCredential
 from devtools_testutils.helpers import get_recording_id
 from devtools_testutils.proxy_testcase import transform_request
+from filelock import FileLock
 from promptflow.client import PFClient
-from azure.ai.evaluation import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from promptflow.executor._line_execution_process_pool import _process_wrapper
 from promptflow.executor._process_manager import create_spawned_fork_process_manager
 from pytest_mock import MockerFixture
 
+from azure.ai.evaluation import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.ai.evaluation._common.utils import ensure_nltk_data_downloaded
+from azure.core.credentials import TokenCredential
+
 # Import of optional packages
 AZURE_INSTALLED = True
 try:
     import jwt
+
     from azure.ai.ml._ml_client import MLClient
 except ImportError:
     AZURE_INSTALLED = False
@@ -42,6 +47,21 @@ class SanitizedValues(str, Enum):
     USER_OBJECT_ID = "00000000-0000-0000-0000-000000000000"
 
 
+@pytest.fixture(scope="session", autouse=True)
+def ensure_nltk_data() -> None:
+    """Ensures that nltk data has been downloaded."""
+
+    def try_download_nltk():
+        for _ in range(3):
+            ensure_nltk_data_downloaded()
+
+    if in_ci():
+        with FileLock(Path.home() / "azure_ai_evaluation_nltk_data.txt"):
+            try_download_nltk()
+    else:
+        try_download_nltk()
+
+
 @pytest.fixture(scope="session", autouse=True)
 def add_sanitizers(
     test_proxy,

sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py

@@ -10,7 +10,6 @@
 from ci_tools.variables import in_ci
 
 from azure.ai.evaluation import (
-    evaluate,
     ContentSafetyEvaluator,
     F1ScoreEvaluator,
     FluencyEvaluator,

sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_metrics_upload.py

@@ -8,10 +8,10 @@
 from devtools_testutils import is_live
 from promptflow.tracing import _start_trace
 
+from azure.ai.evaluation import F1ScoreEvaluator
 from azure.ai.evaluation._evaluate import _utils as ev_utils
 from azure.ai.evaluation._evaluate._eval_run import EvalRun
 from azure.ai.evaluation._evaluate._evaluate import evaluate
-from azure.ai.evaluation import F1ScoreEvaluator
 
 
 @pytest.fixture

sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_sim_and_eval.py

@@ -1,23 +1,19 @@
+import asyncio
 import json
 import os
 import pathlib
 import time
-from typing import Dict, List, Any
-import asyncio
+from typing import Any, Dict, List
+
 import pandas as pd
 import pytest
 import requests
 from ci_tools.variables import in_ci
 from devtools_testutils import is_live
-from azure.identity import DefaultAzureCredential
-
-from azure.ai.evaluation import (
-    evaluate,
-    ProtectedMaterialEvaluator,
-    ViolenceEvaluator,
-)
 
+from azure.ai.evaluation import ProtectedMaterialEvaluator, ViolenceEvaluator, evaluate
 from azure.ai.evaluation.simulator import AdversarialScenario, AdversarialSimulator
+from azure.identity import DefaultAzureCredential
 
 
 @pytest.fixture

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_batch_run_context.py

@@ -5,8 +5,8 @@
 from promptflow.client import PFClient
 
 from azure.ai.evaluation._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT
+from azure.ai.evaluation._evaluate._batch_run import CodeClient, EvalRunContext, ProxyClient
 from azure.ai.evaluation._user_agent import USER_AGENT
-from azure.ai.evaluation._evaluate._batch_run import EvalRunContext, CodeClient, ProxyClient
 
 
 @pytest.fixture

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_built_in_evaluator.py

@@ -2,8 +2,8 @@
 
 import pytest
 
+from azure.ai.evaluation import FluencyEvaluator, RetrievalEvaluator, SimilarityEvaluator
 from azure.ai.evaluation._exceptions import EvaluationException
-from azure.ai.evaluation import FluencyEvaluator, SimilarityEvaluator, RetrievalEvaluator
 
 
 async def quality_async_mock():

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_content_safety_defect_rate.py

@@ -4,8 +4,8 @@
 import pandas as pd
 import pytest
 
-from azure.ai.evaluation._evaluate._evaluate import _aggregate_metrics
 from azure.ai.evaluation import ContentSafetyEvaluator
+from azure.ai.evaluation._evaluate._evaluate import _aggregate_metrics
 
 
 def _get_file(name):

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_content_safety_rai_script.py

@@ -6,9 +6,6 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
-from azure.core.exceptions import HttpResponseError
-from azure.core.rest import AsyncHttpResponse, HttpRequest
-from azure.identity import DefaultAzureCredential
 
 from azure.ai.evaluation._common.constants import EvaluationMetrics, HarmSeverityLevel, RAIService
 from azure.ai.evaluation._common.rai_service import (
@@ -21,6 +18,9 @@
     parse_response,
     submit_request,
 )
+from azure.core.exceptions import HttpResponseError
+from azure.core.rest import AsyncHttpResponse, HttpRequest
+from azure.identity import DefaultAzureCredential
 
 
 @pytest.fixture