add suffix to integration tests (#474)

* add suffix to integration tests

* dont suffix bundle

Author: saiatmakuri

integration_tests/rest_api_utils.py

@@ -15,6 +15,14 @@
 print(f"Integration tests using gateway {BASE_PATH=}")
 DEFAULT_NETWORK_TIMEOUT_SEC = 10
 
+# add suffix to avoid name collisions
+SERVICE_IDENTIFIER = os.environ.get("SERVICE_IDENTIFIER", "")
+
+
+def format_name(name: str) -> str:
+    return f"{name}-{SERVICE_IDENTIFIER}" if SERVICE_IDENTIFIER else name
+
+
 # Use the scale-launch-integration-tests id
 USER_ID_0 = os.getenv("TEST_USER_ID", "fakeuser")
 
@@ -97,7 +105,7 @@ def my_model(**keyword_args):
 
 CREATE_ASYNC_MODEL_ENDPOINT_REQUEST_SIMPLE = {
     "bundle_name": "model_bundle_simple",
-    "name": "model-endpoint-simple-async",
+    "name": format_name("model-endpoint-simple-async"),
     "endpoint_type": "async",
     "cpus": "0.5",
     "memory": "500Mi",
@@ -110,12 +118,12 @@ def my_model(**keyword_args):
 }
 
 CREATE_SYNC_MODEL_ENDPOINT_REQUEST_SIMPLE = CREATE_ASYNC_MODEL_ENDPOINT_REQUEST_SIMPLE.copy()
-CREATE_SYNC_MODEL_ENDPOINT_REQUEST_SIMPLE["name"] = "model-endpoint-simple-sync"
+CREATE_SYNC_MODEL_ENDPOINT_REQUEST_SIMPLE["name"] = format_name("model-endpoint-simple-sync")
 CREATE_SYNC_MODEL_ENDPOINT_REQUEST_SIMPLE["endpoint_type"] = "sync"
 
 CREATE_ASYNC_MODEL_ENDPOINT_REQUEST_RUNNABLE_IMAGE = {
     "bundle_name": "model_bundle_runnable_image",
-    "name": "model-endpoint-runnable-image-async",
+    "name": format_name("model-endpoint-runnable-async"),
     "post_inference_hooks": [],
     "endpoint_type": "async",
     "cpus": "1",
@@ -132,9 +140,9 @@ def my_model(**keyword_args):
 CREATE_SYNC_STREAMING_MODEL_ENDPOINT_REQUEST_RUNNABLE_IMAGE = (
     CREATE_ASYNC_MODEL_ENDPOINT_REQUEST_RUNNABLE_IMAGE.copy()
 )
-CREATE_SYNC_STREAMING_MODEL_ENDPOINT_REQUEST_RUNNABLE_IMAGE[
-    "name"
-] = "model-endpoint-runnable-image-sync-streaming"
+CREATE_SYNC_STREAMING_MODEL_ENDPOINT_REQUEST_RUNNABLE_IMAGE["name"] = format_name(
+    "model-endpoint-runnable-sync-streaming"
+)
 CREATE_SYNC_STREAMING_MODEL_ENDPOINT_REQUEST_RUNNABLE_IMAGE["endpoint_type"] = "streaming"
 
 UPDATE_MODEL_ENDPOINT_REQUEST_SIMPLE = {
@@ -175,7 +183,7 @@ def my_model(**keyword_args):
 }
 
 CREATE_DOCKER_IMAGE_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = {
-    "name": "di_batch_job_bundle_1",
+    "name": format_name("di_batch_job_bundle_1"),
     "image_repository": "model-engine",
     "image_tag": "2c1951dfff7159d7d29dd13b4f888e8355f8d51e",
     "command": ["jq", ".", "/launch_mount_location/file"],
@@ -188,14 +196,14 @@ def my_model(**keyword_args):
 }
 
 CREATE_DOCKER_IMAGE_BATCH_JOB_REQUEST: Dict[str, Any] = {
-    "docker_image_batch_job_bundle_name": "di_batch_job_bundle_1",
+    "docker_image_batch_job_bundle_name": format_name("di_batch_job_bundle_1"),
     "job_config": {"data": {"to": "mount"}},
     "labels": {"team": "infra", "product": "testing"},
     "resource_requests": {"cpus": 0.15, "memory": "15Mi"},
 }
 
 CREATE_FINE_TUNE_DI_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = {
-    "name": "fine_tune_di_batch_job_bundle_1",
+    "name": format_name("fine_tune_di_batch_job_bundle_1"),
     "image_repository": "model-engine",
     "image_tag": "2c1951dfff7159d7d29dd13b4f888e8355f8d51e",
     "command": ["cat", "/launch_mount_location/file"],
@@ -700,9 +708,16 @@ def ensure_n_ready_endpoints_short(n: int, user_id: str):
     assert len(ready_endpoints) >= n
 
 
-def delete_all_endpoints(user_id):
+def delete_all_endpoints(user_id: str, delete_suffix_only: bool):
     endpoints = list_model_endpoints(user_id)
     for i, endpoint in enumerate(endpoints):
+        if (
+            delete_suffix_only
+            and SERVICE_IDENTIFIER
+            and not endpoint["name"].endswith(SERVICE_IDENTIFIER)
+        ):
+            continue
+
         response = delete_model_endpoint(endpoint["name"], user_id)
         assert response["deleted"]
         print(f"[{i + 1}/{len(endpoints)}] Deleted {endpoint=}")
@@ -745,7 +760,9 @@ def ensure_all_async_tasks_success(task_ids: List[str], user_id: str, return_pic
         ensure_inference_task_response_is_correct(response, return_pickled)
 
 
-def delete_existing_endpoints(users: Sequence[str] = DEFAULT_USERS) -> None:
+def delete_existing_endpoints(
+    users: Sequence[str] = DEFAULT_USERS, delete_suffix_only: bool = True
+) -> None:
     if len(users) == 0:
         raise ValueError("Must supply at least one user!")
 
@@ -778,8 +795,9 @@ def delete_existing_endpoints(users: Sequence[str] = DEFAULT_USERS) -> None:
     print(f"[{len({users})}] Deleting all user endpoints...")
     try:
         for i, u in enumerate(users):
-            print(f"[{i + 1}/{len(users)}] Deleting all endpoints for user with ID {u}")
-            delete_all_endpoints(u)
+            suffix_msg = f" with suffix {SERVICE_IDENTIFIER}" if delete_suffix_only else ""
+            print(f"[{i + 1}/{len(users)}] Deleting all endpoints{suffix_msg} for user with ID {u}")
+            delete_all_endpoints(u, delete_suffix_only)
     except Exception:  # noqa
         try:
             j: str = json.dumps(all_endpoint_info, indent=2)
@@ -788,5 +806,4 @@ def delete_existing_endpoints(users: Sequence[str] = DEFAULT_USERS) -> None:
         barrier: str = "-" * 80
         print(f"ERROR! Deletion failed. All endpoint information:\n{barrier}\n{j}\n{barrier}")
         raise
-
     time.sleep(15)

integration_tests/test_docs.py

@@ -2,7 +2,6 @@
 # flake8: noqa: W605
 import importlib.util
 import os
-import random
 import re
 from pathlib import Path
 from textwrap import dedent
@@ -13,6 +12,7 @@
 ROOT_DIR = Path(__file__).parent.parent
 
 TEST_SKIP_MAGIC_STRING = "# test='skip'"
+SERVICE_IDENTIFIER = os.environ.get("SERVICE_IDENTIFIER", "")
 
 
 @pytest.fixture
@@ -50,18 +50,12 @@ def env():
     setenv.clear()
 
 
-@pytest.fixture()
-def seed() -> int:
-    """Returns a random seed between 0 and 999, inclusive."""
-    return random.randint(0, 999)
-
-
 @pytest.fixture()
 def integration_test_user_id() -> str:
-    return "62bc820451dbea002b1c5421"
+    return os.getenv("TEST_USER_ID", "fakeuser")
 
 
-def modify_source(source: str, seed: int) -> str:
+def modify_source(source: str) -> str:
     # Adds some custom logic to update code from docs to comply with some requirements.
     source = re.sub(r"('team'|\"team\"): ('\w+'|\"\w+\")", r"'team': 'infra'", source)
     source = re.sub(
@@ -72,15 +66,21 @@ def modify_source(source: str, seed: int) -> str:
 
     # Add suffix to avoid name collisions
     source = re.sub(
-        r"('endpoint_name'|\"endpoint_name\"): ('(\w+)'|\"(\w+)\")",
-        f"'endpoint_name': '\g<3>\g<4>-{seed}'",
+        r"('endpoint_name'|\"endpoint_name\"): ('([\w-]+)'|\"([\w-]+)\")",
+        rf"'endpoint_name': '\g<3>\g<4>-{SERVICE_IDENTIFIER}'",
         source,
     )
     source = re.sub(
-        r"endpoint_name=('(\w+)'|\"(\w+)\")",
-        f"endpoint_name='\g<2>\g<3>-{seed}'",
+        r"endpoint_name=('([\w-]+)'|\"([\w-]+)\")",
+        rf"endpoint_name='\g<2>\g<3>-{SERVICE_IDENTIFIER}'",
         source,
     )
+    source = re.sub(
+        r"get_model_endpoint\(\"([\w-]+)\"\)",
+        rf'get_model_endpoint("\g<1>-{SERVICE_IDENTIFIER}")',
+        source,
+    )
+
     source = re.sub(r'"repository": "..."', '"repository": "launch_rearch"', source)
     source = re.sub(
         r'"tag": "..."', '"tag": "11d9d42047cc9a0c6435b19e5e91bc7e0ad31efc-cpu"', source
@@ -126,15 +126,15 @@ def modify_source(source: str, seed: int) -> str:
 
 @pytest.fixture
 def import_execute(request, tmp_work_path: Path):
-    def _import_execute(module_name: str, source: str, seed: int, rewrite_assertions: bool = False):
+    def _import_execute(module_name: str, source: str, rewrite_assertions: bool = False):
         if rewrite_assertions:
             loader = AssertionRewritingHook(config=request.config)
             loader.mark_rewrite(module_name)
         else:
             loader = None
 
         module_path = tmp_work_path / f"{module_name}.py"
-        modified_source = modify_source(source, seed)
+        modified_source = modify_source(source)
         module_path.write_text(modified_source)
         spec = importlib.util.spec_from_file_location("__main__", str(module_path), loader=loader)
         module = importlib.util.module_from_spec(spec)
@@ -196,7 +196,6 @@ def test_docs_examples(
     source_code,
     import_execute,
     env,
-    seed,
     integration_test_user_id,
 ):
     if source_code == "__skip__":
@@ -205,6 +204,6 @@ def test_docs_examples(
     env("LAUNCH_API_KEY", os.getenv("LAUNCH_TEST_API_KEY", integration_test_user_id))
 
     try:
-        import_execute(module_name, source_code, seed, True)
+        import_execute(module_name, source_code, True)
     except Exception:
         raise

model-engine/requirements-test.txt

@@ -1,16 +1,17 @@
+coverage==5.5
+diff-cover==7.7.0
+frozendict==2.3.4
+func-timeout==4.3.5
 multiprocess==0.70.14
+moto==3.1.12
+mypy==1.3.0
+pylint<3.0.0
 pytest==7.2.0
 pytest-asyncio==0.20.1
 pytest-cov==2.10.0
-diff-cover==7.7.0
-moto==3.1.12
-coverage==5.5
-mypy==1.3.0
 pytest-mypy==0.9.1
 pytest-mypy-plugins==1.10.1
-pytest-asyncio==0.20.1
 pytest-pylint==0.18.0
-pylint<3.0.0
 types-cachetools==5.3.0.5
 types-croniter==1.4.0.0
 types-PyYAML==6.0.7
@@ -23,5 +24,3 @@ types-toml==0.10.8
 types-ujson==5.5.0
 types-urllib3==1.26.14
 types-waitress==2.1.4
-frozendict==2.3.4
-func-timeout==4.3.5