From 0f79dfb0008d0a5afdc10ff7becbd6288914dd22 Mon Sep 17 00:00:00 2001 From: swastik959 Date: Sun, 22 Oct 2023 21:57:23 +0530 Subject: [PATCH 1/5] modified the projects and zones function Signed-off-by: swastik959 --- src/_nebari/provider/cloud/google_cloud.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index 746bcbc7c5..cc774eddb6 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -3,7 +3,7 @@ import os import subprocess from typing import Dict, List - +from google.cloud import resourcemanager , compute from _nebari import constants from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from nebari import schema @@ -22,22 +22,22 @@ def check_credentials(): def projects() -> Dict[str, str]: """Return a dict of available projects.""" check_credentials() - output = subprocess.check_output( - ["gcloud", "projects", "list", "--format=json(name,projectId)"] - ) - data = json.loads(output) - return {_["name"]: _["projectId"] for _ in data} + client = resourcemanager.Client() + projects = client.list_projects() + project_dict = {project.name: project.project_id for project in projects} + + return project_dict + @functools.lru_cache() def regions(project: str) -> Dict[str, str]: """Return a dict of available regions.""" - check_credentials() - output = subprocess.check_output( - ["gcloud", "compute", "regions", "list", "--project", project, "--format=json"] - ) - data = json.loads(output.decode("utf-8")) - return {_["description"]: _["name"] for _ in data} + client = compute.Client() + regions = client.list_regions(project=project) + region_dict = {region.description: region.name for region in regions} + + return region_dict @functools.lru_cache() From 305157429af103ea65174674aeaf1cef1871319f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 22 Oct 2023 16:30:44 +0000 Subject: [PATCH 2/5] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/provider/cloud/google_cloud.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index cc774eddb6..f51eceb007 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -3,7 +3,9 @@ import os import subprocess from typing import Dict, List -from google.cloud import resourcemanager , compute + +from google.cloud import compute, resourcemanager + from _nebari import constants from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from nebari import schema @@ -29,7 +31,6 @@ def projects() -> Dict[str, str]: return project_dict - @functools.lru_cache() def regions(project: str) -> Dict[str, str]: """Return a dict of available regions.""" From 5d593b107c3d57871dbc690d9db1e6dea6619e73 Mon Sep 17 00:00:00 2001 From: swastik959 Date: Fri, 3 Nov 2023 11:43:48 +0530 Subject: [PATCH 3/5] modified whole code Signed-off-by: swastik959 --- pyproject.toml | 6 + src/_nebari/provider/cloud/google_cloud.py | 182 ++++++--------------- 2 files changed, 57 insertions(+), 131 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 089b262a52..87e4290808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,12 @@ dependencies = [ "rich==13.5.1", "ruamel.yaml==0.17.32", "typer==0.9.0", + "google-cloud-resourcemanager==1.10.1", + "google-cloud-compute==4.15.0", + "google-cloud-storage==2.13.0", + "google-cloud-container==2.32.0", + "google-cloud-iam-credentials== 1.4.1", + ] [project.optional-dependencies] diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index f51eceb007..3ef1258d7a 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -1,11 +1,8 @@ import functools import json import os -import subprocess from typing import Dict, List - -from google.cloud import compute, resourcemanager - +from google.cloud import resourcemanager , compute_v1 , container_v1 , storage , iam_credentials_v1 from _nebari import constants from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from nebari import schema @@ -34,8 +31,9 @@ def projects() -> Dict[str, str]: @functools.lru_cache() def regions(project: str) -> Dict[str, str]: """Return a dict of available regions.""" - client = compute.Client() - regions = client.list_regions(project=project) + client = compute_v1.RegionClient() + request = compute_v1.ListRegionsRequest(project="project_value",) + regions = client.list(request=request) region_dict = {region.description: region.name for region in regions} return region_dict @@ -45,107 +43,57 @@ def regions(project: str) -> Dict[str, str]: def zones(project: str, region: str) -> Dict[str, str]: """Return a dict of available zones.""" check_credentials() - output = subprocess.check_output( - ["gcloud", "compute", "zones", "list", "--project", project, "--format=json"] - ) - data = json.loads(output.decode("utf-8")) - return {_["description"]: _["name"] for _ in data if _["name"].startswith(region)} + client = compute_v1.ZonesClient() + request = compute_v1.ListZonesRequest(project="project_value",) + zones = client.list(request=request) + zone_dict = {zone.description: zone.name for zone in zones if zone.name.startswith(region)} + return zone_dict @functools.lru_cache() def kubernetes_versions(region: str) -> List[str]: """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest.""" check_credentials() - output = subprocess.check_output( - [ - "gcloud", - "container", - "get-server-config", - "--region", - region, - "--format=json", - ] - ) - data = json.loads(output.decode("utf-8")) - supported_kubernetes_versions = sorted([_ for _ in data["validMasterVersions"]]) - return filter_by_highest_supported_k8s_version(supported_kubernetes_versions) + client = container_v1.ClusterManagerClient() + request = container_v1.GetServerConfigRequest() + response = client.get_server_config(request=request) + supported_kubernetes_versions = sorted(response.valid_master_versions) + filtered_versions = filter_by_highest_supported_k8s_version(supported_kubernetes_versions) + return filtered_versions + @functools.lru_cache() -def instances(project: str) -> Dict[str, str]: - """Return a dict of available instances.""" +def instances(project: str , zone: str) -> Dict[str, str]: + """Return a dict of available instances of a particular zone.""" check_credentials() - output = subprocess.check_output( - [ - "gcloud", - "compute", - "machine-types", - "list", - "--project", - project, - "--format=json", - ] - ) - data = json.loads(output.decode("utf-8")) - return {_["description"]: _["name"] for _ in data} + client = compute_v1.InstancesClient() + request = compute_v1.ListInstancesRequest(project="project",zone="zone",) + instances = client.list(request=request) + instance_dict = {instances.description: instances.name for instance in instances} + return instance_dict -def cluster_exists(cluster_name: str, project_id: str, region: str) -> bool: +def cluster_exists(cluster_name: str, project_id: str, zone: str) -> bool: """Check if a GKE cluster exists.""" - try: - subprocess.check_output( - [ - "gcloud", - "container", - "clusters", - "describe", - cluster_name, - "--project", - project_id, - "--region", - region, - ] - ) - return True - except subprocess.CalledProcessError: - return False + client = container_v1.ClusterManagerClient() + request = container_v1.GetClusterRequest() + response = client.get_cluster(request=request, project_id= project_id, zone=zone) + return response is not None def bucket_exists(bucket_name: str, project_id: str) -> bool: """Check if a storage bucket exists.""" - try: - print(f"Checking if bucket {bucket_name} exists in project {project_id}.") - subprocess.check_output( - [ - "gsutil", - "ls", - f"gs://{bucket_name}/", - "-p", - project_id, - ] - ) - return True - except subprocess.CalledProcessError: - return False + client = storage.Client(project=project_id) + bucket = client.get_bucket(bucket_name) + return bucket is not None def service_account_exists(service_account_name: str, project_id: str) -> bool: """Check if a service account exists.""" - try: - subprocess.check_output( - [ - "gcloud", - "iam", - "service-accounts", - "describe", - service_account_name, - "--project", - project_id, - ] - ) - return True - except subprocess.CalledProcessError: - return False + client = iam_credentials_v1.IAMCredentialsClient() + service_acc = client.service_account_path(project_id , service_account_name) + return service_acc is not None def delete_cluster(cluster_name: str, project_id: str, region: str): @@ -158,25 +106,15 @@ def delete_cluster(cluster_name: str, project_id: str, region: str): ) return + client = container_v1.ClusterManagerClient() + request = client.DeleteClusterRequest() try: - subprocess.check_call( - [ - "gcloud", - "container", - "clusters", - "delete", - cluster_name, - "--project", - project_id, - "--region", - region, - "--quiet", - ] - ) - print(f"Successfully deleted cluster {cluster_name}.") - except subprocess.CalledProcessError as e: - print(f"Failed to delete cluster {cluster_name}. Error: {e}") - + operation = client.delete_cluster(request=request) + except google.api_core.exceptions.GoogleAPICallError as e: + if e.status_code == 200: + print("Cluster deleted successfully!") + else: + print("error deleting cluster!") def delete_storage_bucket(bucket_name: str, project_id: str): """Delete a storage bucket if it exists.""" @@ -188,23 +126,14 @@ def delete_storage_bucket(bucket_name: str, project_id: str): ) return + client = storage.Client(project=project_id) + bucket = client.get_bucket(bucket_name) try: - subprocess.check_call( - [ - "gsutil", - "-m", - "rm", - "-r", - f"gs://{bucket_name}", - "-p", - project_id, - ] - ) + bucket.delete() print(f"Successfully deleted bucket {bucket_name}.") - except subprocess.CalledProcessError as e: + except storage.exceptions.BucketNotFoundError as e: print(f"Failed to delete bucket {bucket_name}. Error: {e}") - def delete_service_account(service_account_name: str, project_id: str): """Delete a service account if it exists.""" check_credentials() @@ -214,23 +143,14 @@ def delete_service_account(service_account_name: str, project_id: str): f"Service account {service_account_name} does not exist in project {project_id}. Exiting gracefully." ) return - + client = iam_credentials_v1.IAMCredentialsClient() + service_acc = client.service_account_path(project_id , service_account_name) try: - subprocess.check_call( - [ - "gcloud", - "iam", - "service-accounts", - "delete", - service_account_name, - "--quiet", - "--project", - project_id, - ] - ) + client.delete_service_account(service_account_name) print(f"Successfully deleted service account {service_account_name}.") - except subprocess.CalledProcessError as e: + except iam_credentials_v1.exceptions.IamServiceAccountNotFoundError as e: print(f"Failed to delete service account {service_account_name}. Error: {e}") + def gcp_cleanup(config: schema.Main): From 53ca190f1ea025bc79d64f938a95a09d4a10786a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 06:14:26 +0000 Subject: [PATCH 4/5] [pre-commit.ci] Apply automatic pre-commit fixes --- pyproject.toml | 2 +- src/_nebari/provider/cloud/google_cloud.py | 55 +++++++++++++++------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87e4290808..5c20b6e610 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ dependencies = [ "google-cloud-storage==2.13.0", "google-cloud-container==2.32.0", "google-cloud-iam-credentials== 1.4.1", - + ] [project.optional-dependencies] diff --git a/src/_nebari/provider/cloud/google_cloud.py b/src/_nebari/provider/cloud/google_cloud.py index 3ef1258d7a..69ea961582 100644 --- a/src/_nebari/provider/cloud/google_cloud.py +++ b/src/_nebari/provider/cloud/google_cloud.py @@ -1,8 +1,15 @@ import functools -import json import os from typing import Dict, List -from google.cloud import resourcemanager , compute_v1 , container_v1 , storage , iam_credentials_v1 + +from google.cloud import ( + compute_v1, + container_v1, + iam_credentials_v1, + resourcemanager, + storage, +) + from _nebari import constants from _nebari.provider.cloud.commons import filter_by_highest_supported_k8s_version from nebari import schema @@ -32,7 +39,9 @@ def projects() -> Dict[str, str]: def regions(project: str) -> Dict[str, str]: """Return a dict of available regions.""" client = compute_v1.RegionClient() - request = compute_v1.ListRegionsRequest(project="project_value",) + request = compute_v1.ListRegionsRequest( + project="project_value", + ) regions = client.list(request=request) region_dict = {region.description: region.name for region in regions} @@ -44,9 +53,13 @@ def zones(project: str, region: str) -> Dict[str, str]: """Return a dict of available zones.""" check_credentials() client = compute_v1.ZonesClient() - request = compute_v1.ListZonesRequest(project="project_value",) + request = compute_v1.ListZonesRequest( + project="project_value", + ) zones = client.list(request=request) - zone_dict = {zone.description: zone.name for zone in zones if zone.name.startswith(region)} + zone_dict = { + zone.description: zone.name for zone in zones if zone.name.startswith(region) + } return zone_dict @@ -58,17 +71,21 @@ def kubernetes_versions(region: str) -> List[str]: request = container_v1.GetServerConfigRequest() response = client.get_server_config(request=request) supported_kubernetes_versions = sorted(response.valid_master_versions) - filtered_versions = filter_by_highest_supported_k8s_version(supported_kubernetes_versions) + filtered_versions = filter_by_highest_supported_k8s_version( + supported_kubernetes_versions + ) return filtered_versions - @functools.lru_cache() -def instances(project: str , zone: str) -> Dict[str, str]: +def instances(project: str, zone: str) -> Dict[str, str]: """Return a dict of available instances of a particular zone.""" check_credentials() client = compute_v1.InstancesClient() - request = compute_v1.ListInstancesRequest(project="project",zone="zone",) + request = compute_v1.ListInstancesRequest( + project="project", + zone="zone", + ) instances = client.list(request=request) instance_dict = {instances.description: instances.name for instance in instances} return instance_dict @@ -78,10 +95,11 @@ def cluster_exists(cluster_name: str, project_id: str, zone: str) -> bool: """Check if a GKE cluster exists.""" client = container_v1.ClusterManagerClient() request = container_v1.GetClusterRequest() - response = client.get_cluster(request=request, project_id= project_id, zone=zone) + response = client.get_cluster(request=request, project_id=project_id, zone=zone) return response is not None + def bucket_exists(bucket_name: str, project_id: str) -> bool: """Check if a storage bucket exists.""" client = storage.Client(project=project_id) @@ -92,7 +110,7 @@ def bucket_exists(bucket_name: str, project_id: str) -> bool: def service_account_exists(service_account_name: str, project_id: str) -> bool: """Check if a service account exists.""" client = iam_credentials_v1.IAMCredentialsClient() - service_acc = client.service_account_path(project_id , service_account_name) + service_acc = client.service_account_path(project_id, service_account_name) return service_acc is not None @@ -109,12 +127,13 @@ def delete_cluster(cluster_name: str, project_id: str, region: str): client = container_v1.ClusterManagerClient() request = client.DeleteClusterRequest() try: - operation = client.delete_cluster(request=request) + client.delete_cluster(request=request) except google.api_core.exceptions.GoogleAPICallError as e: - if e.status_code == 200: - print("Cluster deleted successfully!") - else: - print("error deleting cluster!") + if e.status_code == 200: + print("Cluster deleted successfully!") + else: + print("error deleting cluster!") + def delete_storage_bucket(bucket_name: str, project_id: str): """Delete a storage bucket if it exists.""" @@ -134,6 +153,7 @@ def delete_storage_bucket(bucket_name: str, project_id: str): except storage.exceptions.BucketNotFoundError as e: print(f"Failed to delete bucket {bucket_name}. Error: {e}") + def delete_service_account(service_account_name: str, project_id: str): """Delete a service account if it exists.""" check_credentials() @@ -144,13 +164,12 @@ def delete_service_account(service_account_name: str, project_id: str): ) return client = iam_credentials_v1.IAMCredentialsClient() - service_acc = client.service_account_path(project_id , service_account_name) + client.service_account_path(project_id, service_account_name) try: client.delete_service_account(service_account_name) print(f"Successfully deleted service account {service_account_name}.") except iam_credentials_v1.exceptions.IamServiceAccountNotFoundError as e: print(f"Failed to delete service account {service_account_name}. Error: {e}") - def gcp_cleanup(config: schema.Main): From 549540ba29f3be742a02d3055efaf0e63f316daf Mon Sep 17 00:00:00 2001 From: swastik959 Date: Sun, 12 Nov 2023 18:17:03 +0530 Subject: [PATCH 5/5] added ability to disable color prompts Signed-off-by: swastik959 --- .../kubernetes/services/jupyterhub/files/skel/.bashrc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc index a427972228..15c02775c5 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/skel/.bashrc @@ -149,4 +149,9 @@ RESET=$(tput sgr0) NAME=`whoami | cut -d'@' -f1` -PS1="\[$CYAN\]$NAME:\w\[$RESET\]\$(parse_git_branch) \n\[$GREEN\]\A \$ \[$RESET\]" +# PS1 (Prompt String) +if [ "$ENABLE_COLORS" = true ]; then + PS1="\[$CYAN\]$NAME:\w\[$RESET\]\$(parse_git_branch) \n\[$GREEN\]\A \$ \[$RESET\]" +else + PS1="$NAME:\w\$(parse_git_branch) \n\A \$ " +fi