From f0ca18bf05307802cc56c67ea77ca0d7909fa93b Mon Sep 17 00:00:00 2001 From: Yghore Date: Wed, 16 Apr 2025 13:11:44 +0200 Subject: [PATCH 1/2] feat(gcp universe domain) add universe domain config and use for discovery or other static endpoint url for googleapis.com --- plugins/gcp/fix_plugin_gcp/config.py | 1 + plugins/gcp/fix_plugin_gcp/gcp_client.py | 9 +++-- .../resources/cloudfunctions.py | 8 +++-- .../gcp/fix_plugin_gcp/resources/compute.py | 34 ++++++++++--------- .../gcp/fix_plugin_gcp/resources/sqladmin.py | 14 ++++---- .../gcp/fix_plugin_gcp/resources/storage.py | 6 ++-- plugins/gcp/fix_plugin_gcp/utils.py | 13 ++++++- 7 files changed, 55 insertions(+), 30 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/config.py b/plugins/gcp/fix_plugin_gcp/config.py index e7373ccc7..12f5531c2 100644 --- a/plugins/gcp/fix_plugin_gcp/config.py +++ b/plugins/gcp/fix_plugin_gcp/config.py @@ -7,6 +7,7 @@ class GcpConfig: kind: ClassVar[str] = "gcp" service_account: List[str] = field(factory=list, metadata={"description": "GCP service account file(s)"}) project: List[str] = field(factory=list, metadata={"description": "GCP project(s)"}) + universeDomain: str = field(metadata={"description": "Universe domain for API"}, default="googleapis.com") collect: List[str] = field( factory=list, metadata={"description": "GCP services to collect (default: all)"}, diff --git a/plugins/gcp/fix_plugin_gcp/gcp_client.py b/plugins/gcp/fix_plugin_gcp/gcp_client.py index 7e76ece0a..1647c50b9 100644 --- a/plugins/gcp/fix_plugin_gcp/gcp_client.py +++ b/plugins/gcp/fix_plugin_gcp/gcp_client.py @@ -11,6 +11,8 @@ from fixlib.json import value_in_path from fixlib.types import Json +from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain + InternalZoneProp = "_zone" ZoneProp = "zone" RegionProp = "region" @@ -134,7 +136,8 @@ def set_labels(self, api_spec: GcpApiSpec, body: Dict[str, Any], **kwargs: Any) def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs: Any) -> Json: client = _discovery_function( - api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache() + api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache(), + discoveryServiceUrl=getUniverseApiDomain(service="discovery") ) executor = client for accessor in api_spec.accessors: @@ -150,7 +153,9 @@ def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs def list(self, api_spec: GcpApiSpec, **kwargs: Any) -> List[Json]: # todo add caching client = _discovery_function( - api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache() + api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache(), + discoveryServiceUrl=getUniverseApiDomain(service="discovery") + ) executor = client for accessor in api_spec.accessors: diff --git a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py index 857c2d357..f12d90e2c 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py +++ b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py @@ -9,6 +9,8 @@ from fixlib.baseresources import BaseServerlessFunction, MetricName from fixlib.json_bender import Bender, S, Bend, ForallBend +from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain + @define(eq=False, slots=False) class GcpRepoSource: @@ -307,7 +309,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudfunctions.googleapis.com/function/execution_count", + query_name=f"{getUniverseApiDomain(service="cloudfunctions")}/function/execution_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Invocations, @@ -327,7 +329,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudfunctions.googleapis.com/function/execution_count", + query_name=f"{getUniverseApiDomain(service="cloudfunctions")}/function/execution_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Errors, @@ -347,7 +349,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudfunctions.googleapis.com/function/execution_times", + query_name=f"{getUniverseApiDomain(service="cloudfunctions")}/function/execution_times", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Duration, diff --git a/plugins/gcp/fix_plugin_gcp/resources/compute.py b/plugins/gcp/fix_plugin_gcp/resources/compute.py index 93945b7a5..e3b94f416 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/compute.py +++ b/plugins/gcp/fix_plugin_gcp/resources/compute.py @@ -37,6 +37,8 @@ from fixlib.json_bender import Bender, S, Bend, ForallBend, MapDict, F, MapEnum, AsInt from fixlib.types import Json +from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain + log = logging.getLogger("fix.plugins.gcp") @@ -1257,7 +1259,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="compute.googleapis.com/instance/disk/average_io_queue_depth", + query_name=f"{getUniverseApiDomain(service="compute")}/instance/disk/average_io_queue_depth", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.VolumeQueueLength, @@ -1284,8 +1286,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_ops_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_ops_count", MetricName.DiskWrite), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_ops_count", MetricName.DiskWrite), ] ] ) @@ -1304,8 +1306,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_bytes_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_bytes_count", MetricName.DiskWrite), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_bytes_count", MetricName.DiskRead), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_bytes_count", MetricName.DiskWrite), ] ] ) @@ -1812,10 +1814,10 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"loadbalancing.googleapis.com/https/{lb_type}/request_count", MetricName.RequestCount), - (f"loadbalancing.googleapis.com/https/{lb_type}/request_bytes_count", MetricName.RequestBytesCount), + (f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/request_count", MetricName.RequestCount), + (f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/request_bytes_count", MetricName.RequestBytesCount), ( - f"loadbalancing.googleapis.com/https/{lb_type}/response_bytes_count", + f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/response_bytes_count", MetricName.ResponseBytesCount, ), ] @@ -1824,7 +1826,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"loadbalancing.googleapis.com/https/{lb_type}/backend_latencies", + query_name=f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/backend_latencies", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Latency, @@ -3744,7 +3746,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="compute.googleapis.com/instance/cpu/utilization", + query_name=f"{getUniverseApiDomain(service="compute")}/instance/cpu/utilization", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.CpuUtilization, @@ -3770,8 +3772,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/network/received_bytes_count", MetricName.NetworkIn), - ("compute.googleapis.com/instance/network/sent_bytes_count", MetricName.NetworkOut), + (f"{getUniverseApiDomain(service="compute")}/instance/network/received_bytes_count", MetricName.NetworkIn), + (f"{getUniverseApiDomain(service="compute")}/instance/network/sent_bytes_count", MetricName.NetworkOut), ] ] ) @@ -3790,8 +3792,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_ops_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_ops_count", MetricName.DiskWrite), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_ops_count", MetricName.DiskWrite), ] ] ) @@ -3810,8 +3812,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_bytes_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_bytes_count", MetricName.DiskWrite), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_bytes_count", MetricName.DiskRead), + (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_bytes_count", MetricName.DiskWrite), ] ] ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py index 2402768f9..5a578e5c3 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py +++ b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py @@ -12,6 +12,8 @@ from fixlib.json_bender import F, Bender, S, Bend, ForallBend, K, MapEnum, AsInt from fixlib.types import Json +from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain + log = logging.getLogger("fix.plugins.gcp") service_name = "sqladmin" @@ -773,7 +775,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudsql.googleapis.com/database/cpu/utilization", + query_name=f"{getUniverseApiDomain(service="cloudsql")}/database/cpu/utilization", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.CpuUtilization, @@ -805,9 +807,9 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("cloudsql.googleapis.com/database/network/connections", MetricName.DatabaseConnections), - ("cloudsql.googleapis.com/database/network/sent_bytes_count", MetricName.NetworkBytesSent), - ("cloudsql.googleapis.com/database/network/received_bytes_count", MetricName.NetworkBytesReceived), + (f"{getUniverseApiDomain(service="cloudsql")}/database/network/connections", MetricName.DatabaseConnections), + (f"{getUniverseApiDomain(service="cloudsql")}/database/network/sent_bytes_count", MetricName.NetworkBytesSent), + (f"{getUniverseApiDomain(service="cloudsql")}/database/network/received_bytes_count", MetricName.NetworkBytesReceived), ] ] ) @@ -828,8 +830,8 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("cloudsql.googleapis.com/database/disk/read_ops_count", MetricName.DiskRead), - ("cloudsql.googleapis.com/database/disk/write_ops_count", MetricName.DiskWrite), + (f"{getUniverseApiDomain(service="cloudsql")}/database/disk/read_ops_count", MetricName.DiskRead), + (f"{getUniverseApiDomain(service="cloudsql")}/database/disk/write_ops_count", MetricName.DiskWrite), ] ] ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/storage.py b/plugins/gcp/fix_plugin_gcp/resources/storage.py index ec4fbae66..7b8625e2e 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/storage.py +++ b/plugins/gcp/fix_plugin_gcp/resources/storage.py @@ -16,6 +16,8 @@ from fixlib.graph import Graph from fixlib.json_bender import Bender, S, Bend, ForallBend, AsBool +from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain + service_name = "storage" @@ -432,7 +434,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="storage.googleapis.com/storage/total_bytes", + query_name=f"{getUniverseApiDomain(service="storage")}/storage/total_bytes", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.BucketSizeBytes, @@ -450,7 +452,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="storage.googleapis.com/storage/object_count", + query_name=f"{getUniverseApiDomain(service="storage")}/storage/object_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.NumberOfObjects, diff --git a/plugins/gcp/fix_plugin_gcp/utils.py b/plugins/gcp/fix_plugin_gcp/utils.py index 9f9c6063f..0c02784a5 100644 --- a/plugins/gcp/fix_plugin_gcp/utils.py +++ b/plugins/gcp/fix_plugin_gcp/utils.py @@ -98,8 +98,19 @@ def load_credentials(path: str): wait_exponential_max=300000, retry_on_exception=retry_on_error, ) + +def getUniverseApiDomain(service = "endpoint"): + universeDomain = Config.gcp.universeDomain; + match service: + case "endpoint": + return universeDomain + case "discovery": + return f"{{api}}.{universeDomain}/$discovery/rest?version={{apiVersion}}" + case _: + return f"{service}.{universeDomain}" + def gcp_client(service: str, version: str, credentials: str): - client = discovery.build(service, version, credentials=credentials, cache=MemoryCache()) + client = discovery.build(service, version, credentials=credentials, cache=MemoryCache(), discoveryServiceUrl=getUniverseApiDomain(service="discovery")) return client From af3e3ef14d3fee8da2869e0d7ceb141ba46d5ebd Mon Sep 17 00:00:00 2001 From: Yghore Date: Wed, 16 Apr 2025 14:06:58 +0200 Subject: [PATCH 2/2] fix: black formatting, python convention --- plugins/gcp/fix_plugin_gcp/config.py | 2 +- plugins/gcp/fix_plugin_gcp/gcp_client.py | 17 +++-- .../resources/cloudfunctions.py | 8 +-- .../gcp/fix_plugin_gcp/resources/compute.py | 64 ++++++++++++++----- .../gcp/fix_plugin_gcp/resources/sqladmin.py | 29 +++++++-- .../gcp/fix_plugin_gcp/resources/storage.py | 6 +- plugins/gcp/fix_plugin_gcp/utils.py | 29 +++++---- 7 files changed, 105 insertions(+), 50 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/config.py b/plugins/gcp/fix_plugin_gcp/config.py index 12f5531c2..30b2dc007 100644 --- a/plugins/gcp/fix_plugin_gcp/config.py +++ b/plugins/gcp/fix_plugin_gcp/config.py @@ -7,7 +7,7 @@ class GcpConfig: kind: ClassVar[str] = "gcp" service_account: List[str] = field(factory=list, metadata={"description": "GCP service account file(s)"}) project: List[str] = field(factory=list, metadata={"description": "GCP project(s)"}) - universeDomain: str = field(metadata={"description": "Universe domain for API"}, default="googleapis.com") + universe_domain: str = field(default="googleapis.com", metadata={"description": "Universe domain for API"}) collect: List[str] = field( factory=list, metadata={"description": "GCP services to collect (default: all)"}, diff --git a/plugins/gcp/fix_plugin_gcp/gcp_client.py b/plugins/gcp/fix_plugin_gcp/gcp_client.py index 1647c50b9..70cc1ec46 100644 --- a/plugins/gcp/fix_plugin_gcp/gcp_client.py +++ b/plugins/gcp/fix_plugin_gcp/gcp_client.py @@ -11,7 +11,7 @@ from fixlib.json import value_in_path from fixlib.types import Json -from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain +from .utils import get_universe_domain_api InternalZoneProp = "_zone" ZoneProp = "zone" @@ -136,8 +136,11 @@ def set_labels(self, api_spec: GcpApiSpec, body: Dict[str, Any], **kwargs: Any) def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs: Any) -> Json: client = _discovery_function( - api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache(), - discoveryServiceUrl=getUniverseApiDomain(service="discovery") + api_spec.service, + api_spec.version, + credentials=self.credentials, + cache=MemoryCache(), + discoveryServiceUrl=get_universe_domain_api(service="discovery"), ) executor = client for accessor in api_spec.accessors: @@ -153,9 +156,11 @@ def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs def list(self, api_spec: GcpApiSpec, **kwargs: Any) -> List[Json]: # todo add caching client = _discovery_function( - api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache(), - discoveryServiceUrl=getUniverseApiDomain(service="discovery") - + api_spec.service, + api_spec.version, + credentials=self.credentials, + cache=MemoryCache(), + discoveryServiceUrl=get_universe_domain_api(service="discovery"), ) executor = client for accessor in api_spec.accessors: diff --git a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py index f12d90e2c..ca7aaffdc 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py +++ b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py @@ -9,7 +9,7 @@ from fixlib.baseresources import BaseServerlessFunction, MetricName from fixlib.json_bender import Bender, S, Bend, ForallBend -from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain +from fix_plugin_gcp.utils import get_universe_domain_api @define(eq=False, slots=False) @@ -309,7 +309,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="cloudfunctions")}/function/execution_count", + query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Invocations, @@ -329,7 +329,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="cloudfunctions")}/function/execution_count", + query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Errors, @@ -349,7 +349,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="cloudfunctions")}/function/execution_times", + query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_times", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Duration, diff --git a/plugins/gcp/fix_plugin_gcp/resources/compute.py b/plugins/gcp/fix_plugin_gcp/resources/compute.py index e3b94f416..eac4f9f87 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/compute.py +++ b/plugins/gcp/fix_plugin_gcp/resources/compute.py @@ -37,7 +37,7 @@ from fixlib.json_bender import Bender, S, Bend, ForallBend, MapDict, F, MapEnum, AsInt from fixlib.types import Json -from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain +from fix_plugin_gcp.utils import get_universe_domain_api log = logging.getLogger("fix.plugins.gcp") @@ -1259,7 +1259,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="compute")}/instance/disk/average_io_queue_depth", + query_name=f"{get_universe_domain_api(service="compute")}/instance/disk/average_io_queue_depth", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.VolumeQueueLength, @@ -1286,8 +1286,11 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), - (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_ops_count", MetricName.DiskWrite), + (f"{get_universe_domain_api(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_ops_count", + MetricName.DiskWrite, + ), ] ] ) @@ -1306,8 +1309,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_bytes_count", MetricName.DiskRead), - (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_bytes_count", MetricName.DiskWrite), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/read_bytes_count", + MetricName.DiskRead, + ), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_bytes_count", + MetricName.DiskWrite, + ), ] ] ) @@ -1814,10 +1823,16 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/request_count", MetricName.RequestCount), - (f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/request_bytes_count", MetricName.RequestBytesCount), ( - f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/response_bytes_count", + f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/request_count", + MetricName.RequestCount, + ), + ( + f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/request_bytes_count", + MetricName.RequestBytesCount, + ), + ( + f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/response_bytes_count", MetricName.ResponseBytesCount, ), ] @@ -1826,7 +1841,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="loadbalancing")}/https/{lb_type}/backend_latencies", + query_name=f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/backend_latencies", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Latency, @@ -3746,7 +3761,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="compute")}/instance/cpu/utilization", + query_name=f"{get_universe_domain_api(service="compute")}/instance/cpu/utilization", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.CpuUtilization, @@ -3772,8 +3787,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="compute")}/instance/network/received_bytes_count", MetricName.NetworkIn), - (f"{getUniverseApiDomain(service="compute")}/instance/network/sent_bytes_count", MetricName.NetworkOut), + ( + f"{get_universe_domain_api(service="compute")}/instance/network/received_bytes_count", + MetricName.NetworkIn, + ), + ( + f"{get_universe_domain_api(service="compute")}/instance/network/sent_bytes_count", + MetricName.NetworkOut, + ), ] ] ) @@ -3792,8 +3813,11 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), - (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_ops_count", MetricName.DiskWrite), + (f"{get_universe_domain_api(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_ops_count", + MetricName.DiskWrite, + ), ] ] ) @@ -3812,8 +3836,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="compute")}/instance/disk/read_bytes_count", MetricName.DiskRead), - (f"{getUniverseApiDomain(service="compute")}/instance/disk/write_bytes_count", MetricName.DiskWrite), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/read_bytes_count", + MetricName.DiskRead, + ), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_bytes_count", + MetricName.DiskWrite, + ), ] ] ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py index 5a578e5c3..16aa40036 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py +++ b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py @@ -12,7 +12,7 @@ from fixlib.json_bender import F, Bender, S, Bend, ForallBend, K, MapEnum, AsInt from fixlib.types import Json -from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain +from fix_plugin_gcp.utils import get_universe_domain_api log = logging.getLogger("fix.plugins.gcp") service_name = "sqladmin" @@ -775,7 +775,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="cloudsql")}/database/cpu/utilization", + query_name=f"{get_universe_domain_api(service="cloudsql")}/database/cpu/utilization", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.CpuUtilization, @@ -807,9 +807,18 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="cloudsql")}/database/network/connections", MetricName.DatabaseConnections), - (f"{getUniverseApiDomain(service="cloudsql")}/database/network/sent_bytes_count", MetricName.NetworkBytesSent), - (f"{getUniverseApiDomain(service="cloudsql")}/database/network/received_bytes_count", MetricName.NetworkBytesReceived), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/network/connections", + MetricName.DatabaseConnections, + ), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/network/sent_bytes_count", + MetricName.NetworkBytesSent, + ), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/network/received_bytes_count", + MetricName.NetworkBytesReceived, + ), ] ] ) @@ -830,8 +839,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"{getUniverseApiDomain(service="cloudsql")}/database/disk/read_ops_count", MetricName.DiskRead), - (f"{getUniverseApiDomain(service="cloudsql")}/database/disk/write_ops_count", MetricName.DiskWrite), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/disk/read_ops_count", + MetricName.DiskRead, + ), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/disk/write_ops_count", + MetricName.DiskWrite, + ), ] ] ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/storage.py b/plugins/gcp/fix_plugin_gcp/resources/storage.py index 7b8625e2e..61a420bd7 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/storage.py +++ b/plugins/gcp/fix_plugin_gcp/resources/storage.py @@ -16,7 +16,7 @@ from fixlib.graph import Graph from fixlib.json_bender import Bender, S, Bend, ForallBend, AsBool -from plugins.gcp.fix_plugin_gcp.utils import getUniverseApiDomain +from fix_plugin_gcp.utils import get_universe_domain_api service_name = "storage" @@ -434,7 +434,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="storage")}/storage/total_bytes", + query_name=f"{get_universe_domain_api(service="storage")}/storage/total_bytes", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.BucketSizeBytes, @@ -452,7 +452,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"{getUniverseApiDomain(service="storage")}/storage/object_count", + query_name=f"{get_universe_domain_api(service="storage")}/storage/object_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.NumberOfObjects, diff --git a/plugins/gcp/fix_plugin_gcp/utils.py b/plugins/gcp/fix_plugin_gcp/utils.py index 0c02784a5..356f2b6eb 100644 --- a/plugins/gcp/fix_plugin_gcp/utils.py +++ b/plugins/gcp/fix_plugin_gcp/utils.py @@ -92,25 +92,30 @@ def load_credentials(path: str): raise ValueError(f"No credentials file found at {file}") +def get_universe_domain_api(service: str = "endpoint"): + universe_domain = Config.gcp.universe_domain + if service == "endpoint": + return universe_domain + elif service == "discovery": + return f"{{api}}.{universe_domain}/$discovery/rest?version={{apiVersion}}" + else: + return f"{service}.{universe_domain}" + + @retry( stop_max_attempt_number=10, wait_exponential_multiplier=3000, wait_exponential_max=300000, retry_on_exception=retry_on_error, ) - -def getUniverseApiDomain(service = "endpoint"): - universeDomain = Config.gcp.universeDomain; - match service: - case "endpoint": - return universeDomain - case "discovery": - return f"{{api}}.{universeDomain}/$discovery/rest?version={{apiVersion}}" - case _: - return f"{service}.{universeDomain}" - def gcp_client(service: str, version: str, credentials: str): - client = discovery.build(service, version, credentials=credentials, cache=MemoryCache(), discoveryServiceUrl=getUniverseApiDomain(service="discovery")) + client = discovery.build( + service, + version, + credentials=credentials, + cache=MemoryCache(), + discoveryServiceUrl=get_universe_domain_api(service="discovery"), + ) return client