Merge pull request #68 from aws/release

Sagemaker Hyperpod Recipes Release v2.0.2

Author: reneeyge

.github/scripts/trigger_lambda.py

@@ -111,9 +111,8 @@ def handle_lambda_response(response, recipe_metadata):
     parser.add_argument("--lambda-name", required=True, help="Lambda function name")
     args = parser.parse_args()
 
-    # Configure the inbuilt retry behavior
-    config = Config(retries=dict(max_attempts=3, mode="standard"), read_timeout=960)  # 16 minutes
-
+    # Disable retries completely
+    config = Config(retries={"max_attempts": 1, "mode": "standard"}, read_timeout=960)  # 16 minutes
     lambda_client = boto3.client("lambda", region_name=region, config=config)
     recipe_metadata = args.metadata
 

.gitignore

@@ -31,9 +31,13 @@ coverage_html_report/
 mypg/
 .idea/
 recipes_collection/recipes/fine-tuning/sft/
+.kiro/
 
 # Test artifacts
 launch_json_test_report.json
 launch_json_test.log
 templatized*
 failed*
+throughput_results*
+
+debug_serverless/

hyperpod_recipes/__init__.py

@@ -11,9 +11,19 @@ def list_recipes() -> List[Recipe]:
     if not os.path.exists(RECIPES_DIR):
         raise FileNotFoundError(f"Recipes directory not found: {RECIPES_DIR}")
 
+    # Skip __pycache__ and hydra_config directories
+    # hydra_config contains Hydra composition components, not standalone recipes
+    skip_dirs = {"__pycache__", "hydra_config"}
+
     recipes = []
-    for root, _, files in os.walk(RECIPES_DIR):
+    for root, dirs, files in os.walk(RECIPES_DIR):
+        # Prune directories we don't want to traverse
+        dirs[:] = [d for d in dirs if d not in skip_dirs]
+
         for f in files:
+            # Only include .yaml files
+            if not f.endswith(".yaml"):
+                continue
             abs_path = os.path.join(root, f)
             recipes.append(Recipe(abs_path))
     return recipes

launcher/recipe_templatization/evaluation/evaluation_regional_parameters.json

@@ -3,76 +3,76 @@
     "open_source_deterministic_eval": {
       "container_image": {
         "prod": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       },
       "gamma": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       }
     },
       "smtj_container_image": {
         "prod": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       },
       "gamma": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       }
     }
     },
     "open_source_llmaj_eval": {
       "container_image": {
         "prod": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       },
     "gamma": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       }
     },
       "smtj_container_image": {
         "prod": {
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       },
       "gamma":{
-        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04",
-        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:pytorch2.6.0-sagemaker-evaluation0.10.0-gpu-py312-cu126-ubuntu22.04"
+        "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest",
+        "us-west-2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-west-1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest",
+        "eu-central-1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-southeast-2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest",
+        "ap-northeast-1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest"
       }
     }
     }

launcher/recipe_templatization/llmft/llmft_recipe_template_processor.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 import json
-import os
 from collections import OrderedDict
+from pathlib import Path
 from typing import Optional
 
+from hydra import compose, initialize_config_dir
+from hydra.core.global_hydra import GlobalHydra
 from omegaconf import OmegaConf
 
 from ..base_recipe_template_processor import (
@@ -107,7 +109,7 @@ def get_recipe_metadata(self, recipe_file_path: str) -> OrderedDict:
         }],
         """
         metadata = OrderedDict()
-        recipe_cfg = OmegaConf.load(os.path.join("./recipes_collection/recipes", recipe_file_path + ".yaml"))
+        recipe_cfg = self._load_recipe_config(recipe_file_path)
         recipe_metadata_helpers = self.matched_template_group["recipe_metadata_helpers"]
 
         # Get Name
@@ -207,6 +209,21 @@ def get_recipe_metadata(self, recipe_file_path: str) -> OrderedDict:
 
         return metadata
 
+    def _load_recipe_config(self, recipe_file_path: str):
+        recipes_collection_dir = Path("./recipes_collection").absolute()
+        hydra_config_searchpath = recipes_collection_dir / "recipes" / "fine-tuning"
+
+        GlobalHydra.instance().clear()
+
+        with initialize_config_dir(version_base=None, config_dir=str(recipes_collection_dir)):
+            searchpath_override = f"hydra.searchpath=[file://{hydra_config_searchpath}]"
+            recipe_override = f"recipes={recipe_file_path}"
+            cfg = compose(config_name="config", overrides=[searchpath_override, recipe_override])
+
+            if "recipes" in cfg and cfg.recipes is not None:
+                return cfg.recipes
+            return cfg
+
     def _extract_peft_type_from_config(self, recipe_cfg) -> Optional[str]:
         """Extract peft type name from recipe configuration."""
         training_config = recipe_cfg.get("training_config") or {}