Merge branch 'main' of https://github.com/haardm/sagemaker-hyperpod-cli

Author: haardm

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "src/hyperpod_cli/sagemaker_hyperpod_recipes"]
 	path = src/hyperpod_cli/sagemaker_hyperpod_recipes
 	url = https://github.com/aws/sagemaker-hyperpod-recipes.git
-	branch = main
+	branch = 1.3.1

helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/templates/health-monitoring-agent.yaml

@@ -72,6 +72,8 @@ spec:
                   - key: node.kubernetes.io/instance-type
                     operator: In
                     values:
+                      - ml.p5en.48xlarge
+                      - ml.p5e.48xlarge
                       - ml.p5.48xlarge
                       - ml.p4d.24xlarge
                       - ml.p4de.24xlarge

helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml

@@ -1,2 +1,2 @@
 namespace: "aws-hyperpod"
-hmaimage: "905418368575.dkr.ecr.us-west-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.277.0_1.0.27.0"
+hmaimage: "905418368575.dkr.ecr.us-west-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0"

helm_chart/HyperPodHelmChart/charts/job-auto-restart/templates/job-auto-restart-rbac.yaml

@@ -46,6 +46,8 @@ rules:
       - get
       - list
       - watch
+      - create
+      - delete
       - patch
       - update
       - describe
@@ -76,4 +78,4 @@ roleRef:
 subjects:
 - kind: User
   name: hyperpod-service-linked-role
-  namespace: {{ .Values.namespace }}
+  namespace: {{ .Values.namespace }}

helm_chart/readme.md

@@ -93,39 +93,34 @@ Notes:
   ```
   helm install dependencies helm_chart/HyperPodHelmChart --namespace kube-system --set computeQuotaTarget.targetId=<target_id>
   ```
-### Step Five (only required for changing the health monitoring agent installation on your cluster- version upgrade):
-* This command is required to change the version of the Health Monitoring Agent running on your Hyperpod cluster.
+### Step Four (whenever you want to upgrade the installation of helm charts):
+* This command is required to upgrade the helm chart installation on your cluster, which will also help consume the latest releases of service components like Health Monitoring Agent.
 ```
-helm upgrade dependencies helm_chart/HyperPodHelmChart/charts/health-monitoring-agent --namespace kube-system -f helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml
+helm upgrade dependencies helm_chart/HyperPodHelmChart --namespace kube-system
 ```
 
 * To install the sub-chart separately that only contains roles and role bindings
   ```
   helm install dependencies helm_chart/HyperPodHelmChart/charts/team-role-and-bindings --set computeQuotaTarget.targetId=<target_id>
   ```
-### Step Five (only required for changing the health monitoring agent installation on your cluster- version upgrade):
-* This command is required to change the version of the Health Monitoring Agent running on your Hyperpod cluster.
-```
-helm upgrade dependencies helm_chart/HyperPodHelmChart/charts/health-monitoring-agent --namespace kube-system -f helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml
-```
 
 ## 6. Notes
 - Training job auto resume is expected to work with Kubeflow training operator release v1.7.0, v1.8.0, v1.8.1 https://github.com/kubeflow/training-operator/releases
 - If you intend to use the Health Monitoring Agent container image from another region, please see below list to find relevant region's URI.
   ```
-  IAD 767398015722.dkr.ecr.us-east-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  PDX 905418368575.dkr.ecr.us-west-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  CMH 851725546812.dkr.ecr.us-east-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  SFO 011528288828.dkr.ecr.us-west-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  FRA 211125453373.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  ARN 654654141839.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  DUB 533267293120.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  LHR 011528288831.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  NRT 533267052152.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  BOM 011528288864.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  SIN 905418428165.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  SYD 851725636348.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
-  GRU 025066253954.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.318.0_1.0.35.0
+  IAD 767398015722.dkr.ecr.us-east-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  PDX 905418368575.dkr.ecr.us-west-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  CMH 851725546812.dkr.ecr.us-east-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  SFO 011528288828.dkr.ecr.us-west-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  FRA 211125453373.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  ARN 654654141839.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  DUB 533267293120.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  LHR 011528288831.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  NRT 533267052152.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  BOM 011528288864.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  SIN 905418428165.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  SYD 851725636348.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
+  GRU 025066253954.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-health-monitoring-agent:1.0.408.0_1.0.105.0
   ```
 
 ## 7. Troubleshooting

src/hyperpod_cli/commands/job.py

@@ -19,6 +19,7 @@
 import sys
 import subprocess
 from typing import Any, Dict, Optional, List
+from tabulate import tabulate
 
 import click
 import yaml
@@ -40,6 +41,7 @@
     SAGEMAKER_QUOTA_ALLOCATION_LABEL,
     JobPatchType,
     SAGEMAKER_TRAINING_LAUNCHER_DIR,
+    OutputFormat,
     PullPolicy,
     RestartPolicy,
     PersistentVolumeClaim,
@@ -170,6 +172,13 @@ def get_job(
     required=False,
     help="Optional. A label selector to filter the listed jobs. The selector supports the '=', '==', and '!=' operators (e.g., `-l key1=value1,key2=value2`).",
 )
+@click.option(
+    "--output",
+    type=click.Choice([c.value for c in OutputFormat]),
+    required=False,
+    default=OutputFormat.JSON.value,
+    help="Optional. The output format. Available values are `TABLE` and `JSON`. The default value is `JSON`.",
+)
 @click.option(
     "--debug",
     is_flag=True,
@@ -179,6 +188,7 @@ def list_jobs(
     namespace: Optional[str],
     all_namespaces: Optional[bool],
     selector: Optional[str],
+    output: Optional[str],
     debug: bool,
 ):
     if debug:
@@ -189,7 +199,7 @@ def list_jobs(
     try:
         logger.debug("Listing training jobs")
         result = list_training_job_service.list_training_jobs(
-            namespace, all_namespaces, selector
+            namespace, all_namespaces, selector, output
         )
         click.echo(result)
     except Exception as e:

src/hyperpod_cli/service/list_training_jobs.py

@@ -23,9 +23,10 @@
     V1ResourceAttributes
 )
 
-from hyperpod_cli.constants.command_constants import KUEUE_WORKLOAD_PRIORITY_CLASS_LABEL_KEY
+from hyperpod_cli.constants.command_constants import KUEUE_WORKLOAD_PRIORITY_CLASS_LABEL_KEY, OutputFormat
 from hyperpod_cli.constants.pytorch_constants import PYTORCH_CUSTOM_OBJECT_GROUP, PYTORCH_CUSTOM_OBJECT_PLURAL
 from hyperpod_cli.service.discover_namespaces import DiscoverNamespaces
+from tabulate import tabulate
 
 
 class ListTrainingJobs:
@@ -37,14 +38,14 @@ def list_training_jobs(
         namespace: Optional[str],
         all_namespaces: Optional[bool],
         selector: Optional[str],
+        output: Optional[str],
     ) -> str:
         """
         List training job provided by the user in the specified namespace.
         If namespace is not provided job are listed the default namespace in user context
         If all_namespace is true we will list training job from all namespaces that user has access
         Selector when specified will filter list of job addisional based on labels filter provided
         """
-
         k8s_client = KubernetesClient()
 
         jobs: List = []
@@ -80,10 +81,12 @@ def list_training_jobs(
         except ApiException as e:
             raise RuntimeError(f"Unexpected API error: {e.reason} ({e.status})")
 
-        return self._generate_list_training_job_output(jobs)
+        return self._generate_list_training_job_output(jobs, output)
 
-    def _generate_list_training_job_output(self, jobs: List):
+    def _generate_list_training_job_output(self, jobs: List, output: Optional[str]):
         output_jobs = {"jobs": []}
+        priority_header_required = False
+
         for job in jobs:
             if job.get("metadata"):
                 name = job.get("metadata").get("name")
@@ -104,10 +107,34 @@ def _generate_list_training_job_output(self, jobs: List):
 
                 if priority is not None:
                     job_summary["priority"] = priority
+                    priority_header_required = True
 
                 output_jobs["jobs"].append(job_summary)
 
-        return json.dumps(output_jobs, indent=1, sort_keys=False)
+        if output == OutputFormat.TABLE.value:
+            return self._generate_table(output_jobs, priority_header_required)
+        return json.dumps(output_jobs, indent=4, sort_keys=False)
+
+    def _generate_table(self, output_jobs, priority_header_required):
+        headers = [
+                "Name",
+                "Namespace",
+                "CreationTime",
+                "State"
+            ]
+
+        if priority_header_required:
+            headers.append("Priority")
+
+        jobs = []
+        if "jobs" in output_jobs and isinstance(output_jobs["jobs"], list):
+            for job in output_jobs["jobs"]:
+                job_values = list(job.values())
+                if priority_header_required and len(job_values) == 4:
+                    job_values.append("NA")
+                jobs.append(job_values)
+
+        return tabulate(jobs, headers=headers, tablefmt="presto")
 
     def _get_job_status(self, status: List) -> Optional[str]:
         current_status = None

test/unit_tests/service/test_list_training_jobs_service.py

@@ -23,6 +23,7 @@
 )
 
 from kubernetes.client.rest import ApiException
+from tabulate import tabulate
 
 SAMPLE_OUTPUT = {
     "items": [
@@ -152,7 +153,7 @@ def test_list_training_jobs_with_namespace(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.list_training_jobs.return_value = SAMPLE_OUTPUT
         result = self.mock_list_training_jobs.list_training_jobs(
-            "namespace", None, None
+            "namespace", None, None, None
         )
         self.assertIn("test-name", result)
         self.assertIn("test-name1", result)
@@ -166,7 +167,7 @@ def test_list_training_jobs_without_namespace(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.get_current_context_namespace.return_value = "namespace"
         self.mock_k8s_client.list_training_jobs.return_value = SAMPLE_OUTPUT
-        result = self.mock_list_training_jobs.list_training_jobs(None, None, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, None, None, None)
         self.assertIn("test-name", result)
         self.assertIn("test-name1", result)
         self.assertIn("Running", result)
@@ -182,7 +183,7 @@ def test_list_training_jobs_with_namespace_auto_discover(
         mock_discover_accessible_namespace.return_value = "discovered-namespace"
         self.mock_k8s_client.get_current_context_namespace.return_value = None
         self.mock_k8s_client.list_training_jobs.return_value = SAMPLE_OUTPUT
-        result = self.mock_list_training_jobs.list_training_jobs(None, None, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, None, None, None)
         mock_discover_accessible_namespace.assert_called_once_with(
             V1ResourceAttributes(
                 verb="list",
@@ -209,7 +210,7 @@ def test_list_training_jobs_without_namespace_no_jobs(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.get_current_context_namespace.return_value = "namespace"
         self.mock_k8s_client.list_training_jobs.return_value = {"items": []}
-        result = self.mock_list_training_jobs.list_training_jobs(None, None, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, None, None, None)
         self.assertNotIn("test-name", result)
         self.assertNotIn("test-name1", result)
 
@@ -221,7 +222,7 @@ def test_list_training_jobs_all_namespace(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.list_namespaces.return_value = ["namespace"]
         self.mock_k8s_client.list_training_jobs.return_value = SAMPLE_OUTPUT
-        result = self.mock_list_training_jobs.list_training_jobs(None, True, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, True, None, None)
         self.assertIn("test-name", result)
         self.assertIn("test-name1", result)
 
@@ -236,7 +237,7 @@ def test_list_training_jobs_all_namespace_api_exception(
             status="Failed", reason="unexpected"
         )
         with self.assertRaises(RuntimeError):
-            self.mock_list_training_jobs.list_training_jobs(None, True, None)
+            self.mock_list_training_jobs.list_training_jobs(None, True, None, None)
 
     @mock.patch("hyperpod_cli.clients.kubernetes_client.KubernetesClient.__new__")
     def test_list_training_jobs_all_namespace_no_jobs(
@@ -246,7 +247,7 @@ def test_list_training_jobs_all_namespace_no_jobs(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.list_namespaces.return_value = ["namespace"]
         self.mock_k8s_client.list_training_jobs.return_value = {"items": []}
-        result = self.mock_list_training_jobs.list_training_jobs(None, True, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, True, None, None)
         self.assertNotIn("test-name", result)
         self.assertNotIn("test-name1", result)
 
@@ -258,7 +259,7 @@ def test_list_training_jobs_all_namespace_missing_metadata(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.list_namespaces.return_value = ["namespace"]
         self.mock_k8s_client.list_training_jobs.return_value = INVALID_OUTPUT
-        result = self.mock_list_training_jobs.list_training_jobs(None, True, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, True, None, None)
         self.assertNotIn("name", result)
 
     @mock.patch("hyperpod_cli.clients.kubernetes_client.KubernetesClient.__new__")
@@ -269,5 +270,94 @@ def test_list_training_jobs_all_namespace_missing_status(
         mock_kubernetes_client.return_value = self.mock_k8s_client
         self.mock_k8s_client.list_namespaces.return_value = ["namespace"]
         self.mock_k8s_client.list_training_jobs.return_value = OUTPUT_WITHOUT_STATUS
-        result = self.mock_list_training_jobs.list_training_jobs(None, True, None)
+        result = self.mock_list_training_jobs.list_training_jobs(None, True, None, None)
         self.assertNotIn("State: null", result)
+
+    def test_generate_table_with_no_priority_header_and_values(self):
+        list_training_jobs = ListTrainingJobs()
+        output_jobs = {
+            "jobs": [
+                {
+                    "Name": "job1",
+                    "Namespace": "namespace1",
+                    "CreationTime": "2023-01-01T00:00:00Z",
+                    "State": "Running"
+                }
+            ]
+        }
+        priority_header_required = False
+
+        result = list_training_jobs._generate_table(output_jobs, priority_header_required)
+
+        expected_headers = ["Name", "Namespace", "CreationTime", "State"]
+        expected_jobs = [["job1", "namespace1", "2023-01-01T00:00:00Z", "Running"]]
+        expected_result = tabulate(expected_jobs, headers=expected_headers, tablefmt="presto")
+
+        assert result == expected_result
+
+    def test_generate_table_with_priority_header_and_priority_values(self):
+        list_training_jobs = ListTrainingJobs()
+        output_jobs = {
+            "jobs": [
+                {
+                    "Name": "job1",
+                    "Namespace": "namespace1",
+                    "CreationTime": "2023-01-01T00:00:00Z",
+                    "State": "Running",
+                    "priority": "high"
+                },
+                {
+                    "Name": "job2",
+                    "Namespace": "namespace2",
+                    "CreationTime": "2023-01-02T00:00:00Z",
+                    "State": "Completed",
+                    "priority": "low"
+                }
+            ]
+        }
+        priority_header_required = True
+
+        result = list_training_jobs._generate_table(output_jobs, priority_header_required)
+
+        expected_headers = ["Name", "Namespace", "CreationTime", "State", "Priority"]
+        expected_jobs = [
+            ["job1", "namespace1", "2023-01-01T00:00:00Z", "Running", "high"],
+            ["job2", "namespace2", "2023-01-02T00:00:00Z", "Completed", "low"]
+        ]
+        expected_result = tabulate(expected_jobs, headers=expected_headers, tablefmt="presto")
+
+        assert result == expected_result
+
+    def test_generate_table_with_priority_header_but_no_priority_value(self):
+        list_training_jobs = ListTrainingJobs()
+        output_jobs = {
+            "jobs": [
+                {
+                    "Name": "job1",
+                    "Namespace": "namespace1",
+                    "CreationTime": "2023-01-01T00:00:00Z",
+                    "State": "Running"
+                }
+            ]
+        }
+        priority_header_required = True
+
+        result = list_training_jobs._generate_table(output_jobs, priority_header_required)
+
+        expected_headers = ["Name", "Namespace", "CreationTime", "State", "Priority"]
+        expected_jobs = [["job1", "namespace1", "2023-01-01T00:00:00Z", "Running", "NA"]]
+        expected_result = tabulate(expected_jobs, headers=expected_headers, tablefmt="presto")
+
+        assert result == expected_result
+
+    def test_generate_table_empty_jobs(self):
+        list_training_jobs = ListTrainingJobs()
+        output_jobs = {"jobs": []}
+        priority_header_required = False
+
+        result = list_training_jobs._generate_table(output_jobs, priority_header_required)
+
+        expected_headers = ["Name", "Namespace", "CreationTime", "State"]
+        expected_result = tabulate([], headers=expected_headers, tablefmt="presto")
+
+        assert result == expected_result