diff --git a/plugins/gcp/fix_plugin_gcp/config.py b/plugins/gcp/fix_plugin_gcp/config.py index e7373ccc7..30b2dc007 100644 --- a/plugins/gcp/fix_plugin_gcp/config.py +++ b/plugins/gcp/fix_plugin_gcp/config.py @@ -7,6 +7,7 @@ class GcpConfig: kind: ClassVar[str] = "gcp" service_account: List[str] = field(factory=list, metadata={"description": "GCP service account file(s)"}) project: List[str] = field(factory=list, metadata={"description": "GCP project(s)"}) + universe_domain: str = field(default="googleapis.com", metadata={"description": "Universe domain for API"}) collect: List[str] = field( factory=list, metadata={"description": "GCP services to collect (default: all)"}, diff --git a/plugins/gcp/fix_plugin_gcp/gcp_client.py b/plugins/gcp/fix_plugin_gcp/gcp_client.py index 7e76ece0a..70cc1ec46 100644 --- a/plugins/gcp/fix_plugin_gcp/gcp_client.py +++ b/plugins/gcp/fix_plugin_gcp/gcp_client.py @@ -11,6 +11,8 @@ from fixlib.json import value_in_path from fixlib.types import Json +from .utils import get_universe_domain_api + InternalZoneProp = "_zone" ZoneProp = "zone" RegionProp = "region" @@ -134,7 +136,11 @@ def set_labels(self, api_spec: GcpApiSpec, body: Dict[str, Any], **kwargs: Any) def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs: Any) -> Json: client = _discovery_function( - api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache() + api_spec.service, + api_spec.version, + credentials=self.credentials, + cache=MemoryCache(), + discoveryServiceUrl=get_universe_domain_api(service="discovery"), ) executor = client for accessor in api_spec.accessors: @@ -150,7 +156,11 @@ def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs def list(self, api_spec: GcpApiSpec, **kwargs: Any) -> List[Json]: # todo add caching client = _discovery_function( - api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache() + api_spec.service, + api_spec.version, + credentials=self.credentials, + cache=MemoryCache(), + discoveryServiceUrl=get_universe_domain_api(service="discovery"), ) executor = client for accessor in api_spec.accessors: diff --git a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py index 857c2d357..ca7aaffdc 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py +++ b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py @@ -9,6 +9,8 @@ from fixlib.baseresources import BaseServerlessFunction, MetricName from fixlib.json_bender import Bender, S, Bend, ForallBend +from fix_plugin_gcp.utils import get_universe_domain_api + @define(eq=False, slots=False) class GcpRepoSource: @@ -307,7 +309,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudfunctions.googleapis.com/function/execution_count", + query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Invocations, @@ -327,7 +329,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudfunctions.googleapis.com/function/execution_count", + query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Errors, @@ -347,7 +349,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudfunctions.googleapis.com/function/execution_times", + query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_times", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Duration, diff --git a/plugins/gcp/fix_plugin_gcp/resources/compute.py b/plugins/gcp/fix_plugin_gcp/resources/compute.py index 93945b7a5..eac4f9f87 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/compute.py +++ b/plugins/gcp/fix_plugin_gcp/resources/compute.py @@ -37,6 +37,8 @@ from fixlib.json_bender import Bender, S, Bend, ForallBend, MapDict, F, MapEnum, AsInt from fixlib.types import Json +from fix_plugin_gcp.utils import get_universe_domain_api + log = logging.getLogger("fix.plugins.gcp") @@ -1257,7 +1259,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="compute.googleapis.com/instance/disk/average_io_queue_depth", + query_name=f"{get_universe_domain_api(service="compute")}/instance/disk/average_io_queue_depth", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.VolumeQueueLength, @@ -1284,8 +1286,11 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_ops_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_ops_count", MetricName.DiskWrite), + (f"{get_universe_domain_api(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_ops_count", + MetricName.DiskWrite, + ), ] ] ) @@ -1304,8 +1309,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_bytes_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_bytes_count", MetricName.DiskWrite), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/read_bytes_count", + MetricName.DiskRead, + ), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_bytes_count", + MetricName.DiskWrite, + ), ] ] ) @@ -1812,10 +1823,16 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - (f"loadbalancing.googleapis.com/https/{lb_type}/request_count", MetricName.RequestCount), - (f"loadbalancing.googleapis.com/https/{lb_type}/request_bytes_count", MetricName.RequestBytesCount), ( - f"loadbalancing.googleapis.com/https/{lb_type}/response_bytes_count", + f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/request_count", + MetricName.RequestCount, + ), + ( + f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/request_bytes_count", + MetricName.RequestBytesCount, + ), + ( + f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/response_bytes_count", MetricName.ResponseBytesCount, ), ] @@ -1824,7 +1841,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name=f"loadbalancing.googleapis.com/https/{lb_type}/backend_latencies", + query_name=f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/backend_latencies", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.Latency, @@ -3744,7 +3761,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="compute.googleapis.com/instance/cpu/utilization", + query_name=f"{get_universe_domain_api(service="compute")}/instance/cpu/utilization", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.CpuUtilization, @@ -3770,8 +3787,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/network/received_bytes_count", MetricName.NetworkIn), - ("compute.googleapis.com/instance/network/sent_bytes_count", MetricName.NetworkOut), + ( + f"{get_universe_domain_api(service="compute")}/instance/network/received_bytes_count", + MetricName.NetworkIn, + ), + ( + f"{get_universe_domain_api(service="compute")}/instance/network/sent_bytes_count", + MetricName.NetworkOut, + ), ] ] ) @@ -3790,8 +3813,11 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_ops_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_ops_count", MetricName.DiskWrite), + (f"{get_universe_domain_api(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_ops_count", + MetricName.DiskWrite, + ), ] ] ) @@ -3810,8 +3836,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("compute.googleapis.com/instance/disk/read_bytes_count", MetricName.DiskRead), - ("compute.googleapis.com/instance/disk/write_bytes_count", MetricName.DiskWrite), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/read_bytes_count", + MetricName.DiskRead, + ), + ( + f"{get_universe_domain_api(service="compute")}/instance/disk/write_bytes_count", + MetricName.DiskWrite, + ), ] ] ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py index 2402768f9..16aa40036 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py +++ b/plugins/gcp/fix_plugin_gcp/resources/sqladmin.py @@ -12,6 +12,8 @@ from fixlib.json_bender import F, Bender, S, Bend, ForallBend, K, MapEnum, AsInt from fixlib.types import Json +from fix_plugin_gcp.utils import get_universe_domain_api + log = logging.getLogger("fix.plugins.gcp") service_name = "sqladmin" @@ -773,7 +775,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="cloudsql.googleapis.com/database/cpu/utilization", + query_name=f"{get_universe_domain_api(service="cloudsql")}/database/cpu/utilization", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.CpuUtilization, @@ -805,9 +807,18 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("cloudsql.googleapis.com/database/network/connections", MetricName.DatabaseConnections), - ("cloudsql.googleapis.com/database/network/sent_bytes_count", MetricName.NetworkBytesSent), - ("cloudsql.googleapis.com/database/network/received_bytes_count", MetricName.NetworkBytesReceived), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/network/connections", + MetricName.DatabaseConnections, + ), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/network/sent_bytes_count", + MetricName.NetworkBytesSent, + ), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/network/received_bytes_count", + MetricName.NetworkBytesReceived, + ), ] ] ) @@ -828,8 +839,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer ) for stat in STANDART_STAT_MAP for name, metric_name in [ - ("cloudsql.googleapis.com/database/disk/read_ops_count", MetricName.DiskRead), - ("cloudsql.googleapis.com/database/disk/write_ops_count", MetricName.DiskWrite), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/disk/read_ops_count", + MetricName.DiskRead, + ), + ( + f"{get_universe_domain_api(service="cloudsql")}/database/disk/write_ops_count", + MetricName.DiskWrite, + ), ] ] ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/storage.py b/plugins/gcp/fix_plugin_gcp/resources/storage.py index ec4fbae66..61a420bd7 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/storage.py +++ b/plugins/gcp/fix_plugin_gcp/resources/storage.py @@ -16,6 +16,8 @@ from fixlib.graph import Graph from fixlib.json_bender import Bender, S, Bend, ForallBend, AsBool +from fix_plugin_gcp.utils import get_universe_domain_api + service_name = "storage" @@ -432,7 +434,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="storage.googleapis.com/storage/total_bytes", + query_name=f"{get_universe_domain_api(service="storage")}/storage/total_bytes", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.BucketSizeBytes, @@ -450,7 +452,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer queries.extend( [ GcpMonitoringQuery.create( - query_name="storage.googleapis.com/storage/object_count", + query_name=f"{get_universe_domain_api(service="storage")}/storage/object_count", period=delta, ref_id=f"{self.kind}/{self.id}/{self.region().id}", metric_name=MetricName.NumberOfObjects, diff --git a/plugins/gcp/fix_plugin_gcp/utils.py b/plugins/gcp/fix_plugin_gcp/utils.py index 9f9c6063f..356f2b6eb 100644 --- a/plugins/gcp/fix_plugin_gcp/utils.py +++ b/plugins/gcp/fix_plugin_gcp/utils.py @@ -92,6 +92,16 @@ def load_credentials(path: str): raise ValueError(f"No credentials file found at {file}") +def get_universe_domain_api(service: str = "endpoint"): + universe_domain = Config.gcp.universe_domain + if service == "endpoint": + return universe_domain + elif service == "discovery": + return f"{{api}}.{universe_domain}/$discovery/rest?version={{apiVersion}}" + else: + return f"{service}.{universe_domain}" + + @retry( stop_max_attempt_number=10, wait_exponential_multiplier=3000, @@ -99,7 +109,13 @@ def load_credentials(path: str): retry_on_exception=retry_on_error, ) def gcp_client(service: str, version: str, credentials: str): - client = discovery.build(service, version, credentials=credentials, cache=MemoryCache()) + client = discovery.build( + service, + version, + credentials=credentials, + cache=MemoryCache(), + discoveryServiceUrl=get_universe_domain_api(service="discovery"), + ) return client