Increasd reserved resources amounts

Author: Sean Archer

.gitignore

@@ -23,6 +23,11 @@ doc/_build/
 /sagemaker-hyperpod/.coverage
 /sagemaker-hyperpod/.coverage.*
 
+/hyperpod-cluster-stack-template/build
+/hyperpod-pytorch-job-template/build
+/hyperpod-custom-inference-template/build
+/hyperpod-jumpstart-inference-template/build
+
 # Ignore all contents of result and results directories
 /result/
 /results/
@@ -31,5 +36,3 @@ doc/_build/
 
 .venv*
 venv
-
-/hyperpod-cluster-stack-template/build

src/sagemaker/hyperpod/training/quota_allocation_util.py

@@ -137,9 +137,6 @@
     "ml.i3en.24xlarge": {"cpu": 96, "gpu": 0, "trainium": 0, "memory": 768}
 }
 
-MAX_MEMORY_PROPORTION = 0.85
-MAX_CPU_PROPORTION = 0.92
-
 def _has_compute_resource_quota_allocation_resources(memory_in_gib: Optional[float], vcpu: Optional[float], accelerators: Optional[int]) -> bool:
     return (
         (memory_in_gib is not None) or
@@ -269,7 +266,7 @@ def _resolve_default_cpu_values(instance_type: str, requests_values: dict) -> No
             f"Maximum available CPU for {instance_type} is {total_available_cpu}."
         )
 
-    max_allocatable_cpu = int(total_available_cpu * MAX_CPU_PROPORTION)
+    max_allocatable_cpu = int(total_available_cpu - _calculate_cpu_reservation(total_available_cpu))
     cpu_request = min(cpu_request, max_allocatable_cpu)
     requests_values["cpu"] = str(cpu_request)
 
@@ -297,9 +294,11 @@ def _resolve_default_memory_values(instance_type: str, requests_values: dict, li
             f"Maximum available memory for {instance_type} is {total_available_memory}Gi."
         )
 
-    max_allocatable_memory = int(total_available_memory * MAX_MEMORY_PROPORTION)
+    max_allocatable_memory = int(total_available_memory - _calculate_memory_reservation(total_available_memory))
+
     if not user_set_limit:
         memory_limit = min(memory_limit, max_allocatable_memory)
+
     memory_request = min(memory_request, max_allocatable_memory)
     limits_values["memory"] = str(memory_limit) + "Gi"
     requests_values["memory"] = str(memory_request) + "Gi"
@@ -387,32 +386,32 @@ def _calculate_memory_reservation(memory_gb):
     reserved_memory = static_memory_overhead
     remaining = memory_gb
 
-    # First 4 GB (25%)
+    # First 4 GB (30%)
     first_4gb = min(4, remaining)
-    reserved_memory += first_4gb * 0.25
+    reserved_memory += first_4gb * 0.3
     remaining -= first_4gb
 
-    # Next 4 GB (20%)
+    # Next 4 GB (25%)
     if remaining > 0:
         next_4gb = min(4, remaining)
-        reserved_memory += next_4gb * 0.20
+        reserved_memory += next_4gb * 0.25
         remaining -= next_4gb
 
-    # Next 8 GB (10%)
+    # Next 8 GB (20%)
     if remaining > 0:
         next_8gb = min(8, remaining)
-        reserved_memory += next_8gb * 0.10
+        reserved_memory += next_8gb * 0.2
         remaining -= next_8gb
 
-    # Next 112 GB (6%)
+    # Next 112 GB (17%)
     if remaining > 0:
         next_112gb = min(112, remaining)
-        reserved_memory += next_112gb * 0.06
+        reserved_memory += next_112gb * 0.17
         remaining -= next_112gb
 
-    # Remaining memory (2%)
+    # Remaining memory (7%)
     if remaining > 0:
-        reserved_memory += remaining * 0.02
+        reserved_memory += remaining * 0.07
 
     return reserved_memory
 
@@ -424,21 +423,21 @@ def _calculate_cpu_reservation(cpu_count):
 
     reserved_cpu = static_cpu_overhead
 
-    # First core (6%)
+    # First core (30%)
     if cpu_count >= 1:
-        reserved_cpu += 0.06
+        reserved_cpu += 0.3
 
-    # Second core (1%)
+    # Second core (15%)
     if cpu_count >= 2:
-        reserved_cpu += 0.01
+        reserved_cpu += 0.15
 
-    # Cores 3-4 (0.5% each)
+    # Cores 3-4 (10% each)
     for _ in range(min(2, max(0, cpu_count - 2))):
-        reserved_cpu += 0.005
+        reserved_cpu += 0.1
 
-    # Remaining cores (0.25% each)
+    # Remaining cores (6% each)
     if cpu_count > 4:
-        reserved_cpu += (cpu_count - 4) * 0.0025
+        reserved_cpu += (cpu_count - 4) * 0.06
 
     return reserved_cpu
 

test/integration_tests/training/cli/test_gpu_quota_allocation.py

@@ -149,8 +149,8 @@ def test_create_job_with_only_accelerators_parameter(self, test_job_name):
         ]
         result = execute_command(describe_cmd)
         assert result.returncode == 0
-        assert "      Limits:   {'memory': '108Gi', 'nvidia.com/gpu': '1'}" in result.stdout
-        assert "      Requests: {'cpu': '29', 'memory': '108Gi', 'nvidia.com/gpu': '1'}" in result.stdout
+        assert "      Limits:   {'memory': '104Gi', 'nvidia.com/gpu': '1'}" in result.stdout
+        assert "      Requests: {'cpu': '29', 'memory': '104Gi', 'nvidia.com/gpu': '1'}" in result.stdout
 
         delete_cmd = [
             "hyp", "delete", "hyp-pytorch-job",

test/unit_tests/cli/test_quota_allocation_util.py

@@ -30,9 +30,6 @@
     INSTANCE_RESOURCES
 )
 
-MAX_MEMORY_PROPORTION = 0.85
-MAX_CPU_PROPORTION = 0.92
-
 def float_equals(a, b, tolerance=0.0001):
     return abs(a - b) <= tolerance
 
@@ -107,7 +104,7 @@ def test_get_resources_from_compute_quotas_memory_only(self):
     def test_get_resources_from_compute_quotas_gpu_instance_with_accelerators_ratio_1(self):
         result = _get_resources_from_compute_quotas("ml.g5.xlarge", None, None, 1)
         # ml.g5.xlarge has 1 GPU, 4 CPUs, 16GiB memory
-        assert result == {"cpu": "3.82", "memory": "12.9Gi", "nvidia.com/gpu": 1}
+        assert result == {"cpu": "3.25", "memory": "11.7Gi", "nvidia.com/gpu": 1}
 
     def test_get_resources_from_compute_quotas_gpu_instance_with_accelerators_ratio_half(self):
         result = _get_resources_from_compute_quotas("ml.g6e.48xlarge", None, None, 4)
@@ -137,7 +134,7 @@ def test_get_resources_from_compute_quotas_vcpu_only(self):
     def test_get_resources_from_compute_quotas_accelerators_and_cpu_only(self):
         result = _get_resources_from_compute_quotas("ml.g5.xlarge", 2.0, None, 1)
         # ml.g5.xlarge has 1 gpu, 4 CPUs and 16GB memory, and memory calculated as accelerator ratio
-        assert result == {'cpu': '2.0', 'memory': '12.9Gi', 'nvidia.com/gpu': 1}
+        assert result == {'cpu': '2.0', 'memory': '11.7Gi', 'nvidia.com/gpu': 1}
 
     # Tests for _get_resources_from_instance method
     @pytest.mark.parametrize(
@@ -312,8 +309,8 @@ def test_resolve_default_memory_values_set_to_allocatable(self):
         requests = {"memory": "16Gi"}
         limits = {}
         _resolve_default_memory_values("ml.g5.xlarge", requests, limits)
-        assert requests["memory"] == "13Gi"
-        assert limits["memory"] == "13Gi"
+        assert requests["memory"] == "11Gi"
+        assert limits["memory"] == "11Gi"
 
     # Tests for _validate_accelerators_inputs
     def test_validate_accelerators_inputs_valid_equal_values(self):
@@ -419,17 +416,17 @@ def test_request_modification(self):
     def test_memory_reservation_small_instance(self):
         memory_gb = 4
         reserved = _calculate_memory_reservation(memory_gb)
-        assert float_equals(reserved, 1.5)
+        assert float_equals(reserved, 1.7)
 
     def test_memory_reservation_medium_instance(self):
         memory_gb = 16
         reserved = _calculate_memory_reservation(memory_gb)
-        assert (float_equals(reserved, 3.1))
+        assert (float_equals(reserved, 4.3))
 
     def test_memory_reservation_large_instance(self):
         memory_gb = 2048
         reserved = _calculate_memory_reservation(memory_gb)
-        assert (float_equals(reserved, 48.22))
+        assert (float_equals(reserved, 157.74))
 
     def test_memory_reservation_zero(self):
         memory_gb = 0
@@ -440,23 +437,23 @@ def test_cpu_reservation_single_core(self):
         """Test CPU reservation for single core"""
         cpu_count = 1
         reserved = _calculate_cpu_reservation(cpu_count)
-        assert (float_equals(reserved, 0.16))
+        assert (float_equals(reserved, 0.4))
 
     def test_cpu_reservation_dual_core(self):
         cpu_count = 2
         reserved = _calculate_cpu_reservation(cpu_count)
-        assert (float_equals(reserved, 0.17))
+        assert (float_equals(reserved, 0.55))
 
     def test_cpu_reservation_quad_core(self):
         cpu_count = 4
         reserved = _calculate_cpu_reservation(cpu_count)
-        assert (float_equals(reserved, 0.18))
+        assert (float_equals(reserved, 0.75))
 
     def test_cpu_reservation_many_cores(self):
         """Test CPU reservation for 96 cores"""
         cpu_count = 96
         reserved = _calculate_cpu_reservation(cpu_count)
-        assert (float_equals(reserved, 0.41))
+        assert (float_equals(reserved, 6.27))
 
     def test_cpu_reservation_zero(self):
         """Test CPU reservation with 0 cores"""