feat(gcp universe domain) add universe domain config for request endpoint

plugins/gcp/fix_plugin_gcp/config.py

@@ -7,6 +7,7 @@ class GcpConfig:
     kind: ClassVar[str] = "gcp"
     service_account: List[str] = field(factory=list, metadata={"description": "GCP service account file(s)"})
     project: List[str] = field(factory=list, metadata={"description": "GCP project(s)"})
+    universe_domain: str = field(default="googleapis.com", metadata={"description": "Universe domain for API"})
     collect: List[str] = field(
         factory=list,
         metadata={"description": "GCP services to collect (default: all)"},

plugins/gcp/fix_plugin_gcp/gcp_client.py

@@ -11,6 +11,8 @@
 from fixlib.json import value_in_path
 from fixlib.types import Json
 
+from .utils import get_universe_domain_api
+
 InternalZoneProp = "_zone"
 ZoneProp = "zone"
 RegionProp = "region"
@@ -134,7 +136,11 @@ def set_labels(self, api_spec: GcpApiSpec, body: Dict[str, Any], **kwargs: Any)
 
     def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs: Any) -> Json:
         client = _discovery_function(
-            api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache()
+            api_spec.service,
+            api_spec.version,
+            credentials=self.credentials,
+            cache=MemoryCache(),
+            discoveryServiceUrl=get_universe_domain_api(service="discovery"),
         )
         executor = client
         for accessor in api_spec.accessors:
@@ -150,7 +156,11 @@ def call_single(self, api_spec: GcpApiSpec, body: Optional[Any] = None, **kwargs
     def list(self, api_spec: GcpApiSpec, **kwargs: Any) -> List[Json]:
         # todo add caching
         client = _discovery_function(
-            api_spec.service, api_spec.version, credentials=self.credentials, cache=MemoryCache()
+            api_spec.service,
+            api_spec.version,
+            credentials=self.credentials,
+            cache=MemoryCache(),
+            discoveryServiceUrl=get_universe_domain_api(service="discovery"),
         )
         executor = client
         for accessor in api_spec.accessors:

plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py

@@ -9,6 +9,8 @@
 from fixlib.baseresources import BaseServerlessFunction, MetricName
 from fixlib.json_bender import Bender, S, Bend, ForallBend
 
+from fix_plugin_gcp.utils import get_universe_domain_api
+
 
 @define(eq=False, slots=False)
 class GcpRepoSource:
@@ -307,7 +309,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="cloudfunctions.googleapis.com/function/execution_count",
+                    query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_count",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.Invocations,
@@ -327,7 +329,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="cloudfunctions.googleapis.com/function/execution_count",
+                    query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_count",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.Errors,
@@ -347,7 +349,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="cloudfunctions.googleapis.com/function/execution_times",
+                    query_name=f"{get_universe_domain_api(service="cloudfunctions")}/function/execution_times",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.Duration,

plugins/gcp/fix_plugin_gcp/resources/compute.py

@@ -37,6 +37,8 @@
 from fixlib.json_bender import Bender, S, Bend, ForallBend, MapDict, F, MapEnum, AsInt
 from fixlib.types import Json
 
+from fix_plugin_gcp.utils import get_universe_domain_api
+
 log = logging.getLogger("fix.plugins.gcp")
 
 
@@ -1257,7 +1259,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="compute.googleapis.com/instance/disk/average_io_queue_depth",
+                    query_name=f"{get_universe_domain_api(service="compute")}/instance/disk/average_io_queue_depth",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.VolumeQueueLength,
@@ -1284,8 +1286,11 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("compute.googleapis.com/instance/disk/read_ops_count", MetricName.DiskRead),
-                    ("compute.googleapis.com/instance/disk/write_ops_count", MetricName.DiskWrite),
+                    (f"{get_universe_domain_api(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/disk/write_ops_count",
+                        MetricName.DiskWrite,
+                    ),
                 ]
             ]
         )
@@ -1304,8 +1309,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("compute.googleapis.com/instance/disk/read_bytes_count", MetricName.DiskRead),
-                    ("compute.googleapis.com/instance/disk/write_bytes_count", MetricName.DiskWrite),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/disk/read_bytes_count",
+                        MetricName.DiskRead,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/disk/write_bytes_count",
+                        MetricName.DiskWrite,
+                    ),
                 ]
             ]
         )
@@ -1812,10 +1823,16 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    (f"loadbalancing.googleapis.com/https/{lb_type}/request_count", MetricName.RequestCount),
-                    (f"loadbalancing.googleapis.com/https/{lb_type}/request_bytes_count", MetricName.RequestBytesCount),
                     (
-                        f"loadbalancing.googleapis.com/https/{lb_type}/response_bytes_count",
+                        f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/request_count",
+                        MetricName.RequestCount,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/request_bytes_count",
+                        MetricName.RequestBytesCount,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/response_bytes_count",
                         MetricName.ResponseBytesCount,
                     ),
                 ]
@@ -1824,7 +1841,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name=f"loadbalancing.googleapis.com/https/{lb_type}/backend_latencies",
+                    query_name=f"{get_universe_domain_api(service="loadbalancing")}/https/{lb_type}/backend_latencies",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.Latency,
@@ -3744,7 +3761,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="compute.googleapis.com/instance/cpu/utilization",
+                    query_name=f"{get_universe_domain_api(service="compute")}/instance/cpu/utilization",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.CpuUtilization,
@@ -3770,8 +3787,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("compute.googleapis.com/instance/network/received_bytes_count", MetricName.NetworkIn),
-                    ("compute.googleapis.com/instance/network/sent_bytes_count", MetricName.NetworkOut),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/network/received_bytes_count",
+                        MetricName.NetworkIn,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/network/sent_bytes_count",
+                        MetricName.NetworkOut,
+                    ),
                 ]
             ]
         )
@@ -3790,8 +3813,11 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("compute.googleapis.com/instance/disk/read_ops_count", MetricName.DiskRead),
-                    ("compute.googleapis.com/instance/disk/write_ops_count", MetricName.DiskWrite),
+                    (f"{get_universe_domain_api(service="compute")}/instance/disk/read_ops_count", MetricName.DiskRead),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/disk/write_ops_count",
+                        MetricName.DiskWrite,
+                    ),
                 ]
             ]
         )
@@ -3810,8 +3836,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("compute.googleapis.com/instance/disk/read_bytes_count", MetricName.DiskRead),
-                    ("compute.googleapis.com/instance/disk/write_bytes_count", MetricName.DiskWrite),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/disk/read_bytes_count",
+                        MetricName.DiskRead,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="compute")}/instance/disk/write_bytes_count",
+                        MetricName.DiskWrite,
+                    ),
                 ]
             ]
         )

plugins/gcp/fix_plugin_gcp/resources/sqladmin.py

@@ -12,6 +12,8 @@
 from fixlib.json_bender import F, Bender, S, Bend, ForallBend, K, MapEnum, AsInt
 from fixlib.types import Json
 
+from fix_plugin_gcp.utils import get_universe_domain_api
+
 log = logging.getLogger("fix.plugins.gcp")
 service_name = "sqladmin"
 
@@ -773,7 +775,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="cloudsql.googleapis.com/database/cpu/utilization",
+                    query_name=f"{get_universe_domain_api(service="cloudsql")}/database/cpu/utilization",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.CpuUtilization,
@@ -805,9 +807,18 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("cloudsql.googleapis.com/database/network/connections", MetricName.DatabaseConnections),
-                    ("cloudsql.googleapis.com/database/network/sent_bytes_count", MetricName.NetworkBytesSent),
-                    ("cloudsql.googleapis.com/database/network/received_bytes_count", MetricName.NetworkBytesReceived),
+                    (
+                        f"{get_universe_domain_api(service="cloudsql")}/database/network/connections",
+                        MetricName.DatabaseConnections,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="cloudsql")}/database/network/sent_bytes_count",
+                        MetricName.NetworkBytesSent,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="cloudsql")}/database/network/received_bytes_count",
+                        MetricName.NetworkBytesReceived,
+                    ),
                 ]
             ]
         )
@@ -828,8 +839,14 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
                 )
                 for stat in STANDART_STAT_MAP
                 for name, metric_name in [
-                    ("cloudsql.googleapis.com/database/disk/read_ops_count", MetricName.DiskRead),
-                    ("cloudsql.googleapis.com/database/disk/write_ops_count", MetricName.DiskWrite),
+                    (
+                        f"{get_universe_domain_api(service="cloudsql")}/database/disk/read_ops_count",
+                        MetricName.DiskRead,
+                    ),
+                    (
+                        f"{get_universe_domain_api(service="cloudsql")}/database/disk/write_ops_count",
+                        MetricName.DiskWrite,
+                    ),
                 ]
             ]
         )

plugins/gcp/fix_plugin_gcp/resources/storage.py

@@ -16,6 +16,8 @@
 from fixlib.graph import Graph
 from fixlib.json_bender import Bender, S, Bend, ForallBend, AsBool
 
+from fix_plugin_gcp.utils import get_universe_domain_api
+
 service_name = "storage"
 
 
@@ -432,7 +434,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="storage.googleapis.com/storage/total_bytes",
+                    query_name=f"{get_universe_domain_api(service="storage")}/storage/total_bytes",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.BucketSizeBytes,
@@ -450,7 +452,7 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[GcpMonitoringQuer
         queries.extend(
             [
                 GcpMonitoringQuery.create(
-                    query_name="storage.googleapis.com/storage/object_count",
+                    query_name=f"{get_universe_domain_api(service="storage")}/storage/object_count",
                     period=delta,
                     ref_id=f"{self.kind}/{self.id}/{self.region().id}",
                     metric_name=MetricName.NumberOfObjects,

plugins/gcp/fix_plugin_gcp/utils.py

@@ -92,14 +92,30 @@ def load_credentials(path: str):
         raise ValueError(f"No credentials file found at {file}")
 
 
+def get_universe_domain_api(service: str = "endpoint"):
+    universe_domain = Config.gcp.universe_domain
+    if service == "endpoint":
+        return universe_domain
+    elif service == "discovery":
+        return f"{{api}}.{universe_domain}/$discovery/rest?version={{apiVersion}}"
+    else:
+        return f"{service}.{universe_domain}"
+
+
 @retry(
     stop_max_attempt_number=10,
     wait_exponential_multiplier=3000,
     wait_exponential_max=300000,
     retry_on_exception=retry_on_error,
 )
 def gcp_client(service: str, version: str, credentials: str):
-    client = discovery.build(service, version, credentials=credentials, cache=MemoryCache())
+    client = discovery.build(
+        service,
+        version,
+        credentials=credentials,
+        cache=MemoryCache(),
+        discoveryServiceUrl=get_universe_domain_api(service="discovery"),
+    )
     return client