feat: detect monitored resources on all GCP environments (#200)

Author: daniel-sanche

google/cloud/logging_v2/client.py

@@ -29,7 +29,6 @@
 from google.cloud.client import ClientWithProject
 from google.cloud.environment_vars import DISABLE_GRPC
 from google.cloud.logging_v2._helpers import _add_defaults_to_filter
-from google.cloud.logging_v2._helpers import retrieve_metadata_server
 from google.cloud.logging_v2._http import Connection
 from google.cloud.logging_v2._http import _LoggingAPI as JSONLoggingAPI
 from google.cloud.logging_v2._http import _MetricsAPI as JSONMetricsAPI
@@ -39,6 +38,9 @@
 from google.cloud.logging_v2.handlers import ContainerEngineHandler
 from google.cloud.logging_v2.handlers import setup_logging
 from google.cloud.logging_v2.handlers.handlers import EXCLUDED_LOGGER_DEFAULTS
+from google.cloud.logging_v2.resource import Resource
+from google.cloud.logging_v2.handlers._monitored_resources import detect_resource
+
 
 from google.cloud.logging_v2.logger import Logger
 from google.cloud.logging_v2.metric import Metric
@@ -48,14 +50,8 @@
 _DISABLE_GRPC = os.getenv(DISABLE_GRPC, False)
 _USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC
 
-_APPENGINE_FLEXIBLE_ENV_VM = "GAE_APPENGINE_HOSTNAME"
-"""Environment variable set in App Engine when vm:true is set."""
-
-_APPENGINE_INSTANCE_ID = "GAE_INSTANCE"
-"""Environment variable set in App Engine standard and flexible environment."""
-
-_GKE_CLUSTER_NAME = "instance/attributes/cluster-name"
-"""Attribute in metadata server when in GKE environment."""
+_GAE_RESOURCE_TYPE = "gae_app"
+_GKE_RESOURCE_TYPE = "k8s_container"
 
 
 class Client(ClientWithProject):
@@ -348,17 +344,20 @@ def get_default_handler(self, **kw):
         Returns:
             logging.Handler: The default log handler based on the environment
         """
-        gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME)
+        monitored_resource = kw.pop("resource", detect_resource(self.project))
 
         if (
-            _APPENGINE_FLEXIBLE_ENV_VM in os.environ
-            or _APPENGINE_INSTANCE_ID in os.environ
+            isinstance(monitored_resource, Resource)
+            and monitored_resource.type == _GAE_RESOURCE_TYPE
         ):
             return AppEngineHandler(self, **kw)
-        elif gke_cluster_name is not None:
+        elif (
+            isinstance(monitored_resource, Resource)
+            and monitored_resource.type == _GKE_RESOURCE_TYPE
+        ):
             return ContainerEngineHandler(**kw)
         else:
-            return CloudLoggingHandler(self, **kw)
+            return CloudLoggingHandler(self, resource=monitored_resource, **kw)
 
     def setup_logging(
         self, *, log_level=logging.INFO, excluded_loggers=EXCLUDED_LOGGER_DEFAULTS, **kw

google/cloud/logging_v2/handlers/_monitored_resources.py

@@ -0,0 +1,195 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from google.cloud.logging_v2.resource import Resource
+from google.cloud.logging_v2._helpers import retrieve_metadata_server
+
+_GAE_SERVICE_ENV = "GAE_SERVICE"
+_GAE_VERSION_ENV = "GAE_VERSION"
+_GAE_INSTANCE_ENV = "GAE_INSTANCE"
+_GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV]
+"""Environment variables set in App Engine environment."""
+
+_CLOUD_RUN_SERVICE_ID = "K_SERVICE"
+_CLOUD_RUN_REVISION_ID = "K_REVISION"
+_CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION"
+_CLOUD_RUN_ENV_VARS = [
+    _CLOUD_RUN_SERVICE_ID,
+    _CLOUD_RUN_REVISION_ID,
+    _CLOUD_RUN_CONFIGURATION_ID,
+]
+"""Environment variables set in Cloud Run environment."""
+
+_FUNCTION_TARGET = "FUNCTION_TARGET"
+_FUNCTION_SIGNATURE = "FUNCTION_SIGNATURE_TYPE"
+_FUNCTION_NAME = "FUNCTION_NAME"
+_FUNCTION_REGION = "FUNCTION_REGION"
+_FUNCTION_ENTRY = "ENTRY_POINT"
+_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE, _CLOUD_RUN_SERVICE_ID]
+_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY]
+"""Environment variables set in Cloud Functions environments."""
+
+
+_REGION_ID = "instance/region"
+_ZONE_ID = "instance/zone"
+_GCE_INSTANCE_ID = "instance/id"
+"""Attribute in metadata server for compute region and instance."""
+
+_GKE_CLUSTER_NAME = "instance/attributes/cluster-name"
+"""Attribute in metadata server when in GKE environment."""
+
+
+def _create_functions_resource(project):
+    """Create a standardized Cloud Functions resource.
+    Args:
+        project (str): The project ID to pass on to the resource
+    Returns:
+        google.cloud.logging.Resource
+    """
+    region = retrieve_metadata_server(_REGION_ID)
+    if _FUNCTION_NAME in os.environ:
+        function_name = os.environ.get(_FUNCTION_NAME)
+    elif _CLOUD_RUN_SERVICE_ID in os.environ:
+        function_name = os.environ.get(_CLOUD_RUN_SERVICE_ID)
+    else:
+        function_name = ""
+    resource = Resource(
+        type="cloud_function",
+        labels={
+            "project_id": project,
+            "function_name": function_name,
+            "region": region if region else "",
+        },
+    )
+    return resource
+
+
+def _create_kubernetes_resource(project):
+    """Create a standardized Kubernetes resource.
+    Args:
+        project (str): The project ID to pass on to the resource
+    Returns:
+        google.cloud.logging.Resource
+    """
+    zone = retrieve_metadata_server(_ZONE_ID)
+    cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME)
+
+    resource = Resource(
+        type="k8s_container",
+        labels={
+            "project_id": project,
+            "location": zone if zone else "",
+            "cluster_name": cluster_name if cluster_name else "",
+        },
+    )
+    return resource
+
+
+def _create_compute_resource(project):
+    """Create a standardized Compute Engine resource.
+    Args:
+        project (str): The project ID to pass on to the resource
+    Returns:
+        google.cloud.logging.Resource
+    """
+    instance = retrieve_metadata_server(_GCE_INSTANCE_ID)
+    zone = retrieve_metadata_server(_ZONE_ID)
+    resource = Resource(
+        type="gce_instance",
+        labels={
+            "project_id": project,
+            "instance_id": instance if instance else "",
+            "zone": zone if zone else "",
+        },
+    )
+    return resource
+
+
+def _create_cloud_run_resource(project):
+    """Create a standardized Cloud Run resource.
+    Args:
+        project (str): The project ID to pass on to the resource
+    Returns:
+        google.cloud.logging.Resource
+    """
+    region = retrieve_metadata_server(_REGION_ID)
+    resource = Resource(
+        type="cloud_run_revision",
+        labels={
+            "project_id": project,
+            "service_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""),
+            "revision_name": os.environ.get(_CLOUD_RUN_REVISION_ID, ""),
+            "location": region if region else "",
+            "configuration_name": os.environ.get(_CLOUD_RUN_CONFIGURATION_ID, ""),
+        },
+    )
+    return resource
+
+
+def _create_app_engine_resource(project):
+    """Create a standardized App Engine resource.
+    Args:
+        project (str): The project ID to pass on to the resource
+    Returns:
+        google.cloud.logging.Resource
+    """
+    zone = retrieve_metadata_server(_ZONE_ID)
+    resource = Resource(
+        type="gae_app",
+        labels={
+            "project_id": project,
+            "module_id": os.environ.get(_GAE_SERVICE_ENV, ""),
+            "version_id": os.environ.get(_GAE_VERSION_ENV, ""),
+            "zone": zone if zone else "",
+        },
+    )
+    return resource
+
+
+def _create_global_resource(project):
+    return Resource(type="global", labels={"project_id": project})
+
+
+def detect_resource(project):
+    """Return the default monitored resource based on the local environment.
+    Args:
+        project (str): The project ID to pass on to the resource
+    Returns:
+        google.cloud.logging.Resource: The default resource based on the environment
+    """
+    gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME)
+    gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID)
+
+    if all([env in os.environ for env in _GAE_ENV_VARS]):
+        # App Engine Flex or Standard
+        return _create_app_engine_resource(project)
+    elif gke_cluster_name is not None:
+        # Kubernetes Engine
+        return _create_kubernetes_resource(project)
+    elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) or all(
+        [env in os.environ for env in _FUNCTION_ENV_VARS]
+    ):
+        # Cloud Functions
+        return _create_functions_resource(project)
+    elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]):
+        # Cloud Run
+        return _create_cloud_run_resource(project)
+    elif gce_instance_name is not None:
+        # Compute Engine
+        return _create_compute_resource(project)
+    else:
+        # use generic global resource
+        return _create_global_resource(project)

google/cloud/logging_v2/handlers/app_engine.py

@@ -22,8 +22,10 @@
 import os
 
 from google.cloud.logging_v2.handlers._helpers import get_request_data
+from google.cloud.logging_v2.handlers._monitored_resources import (
+    _create_app_engine_resource,
+)
 from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport
-from google.cloud.logging_v2.resource import Resource
 
 _DEFAULT_GAE_LOGGER_NAME = "app"
 
@@ -75,15 +77,7 @@ def get_gae_resource(self):
         Returns:
             google.cloud.logging_v2.resource.Resource: Monitored resource for GAE.
         """
-        gae_resource = Resource(
-            type="gae_app",
-            labels={
-                "project_id": self.project_id,
-                "module_id": self.module_id,
-                "version_id": self.version_id,
-            },
-        )
-        return gae_resource
+        return _create_app_engine_resource(self.project_id)
 
     def get_gae_labels(self):
         """Return the labels for GAE app.

google/cloud/logging_v2/handlers/handlers.py

@@ -17,7 +17,7 @@
 import logging
 
 from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport
-from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE
+from google.cloud.logging_v2.handlers._monitored_resources import detect_resource
 
 DEFAULT_LOGGER_NAME = "python"
 
@@ -59,7 +59,7 @@ def __init__(
         *,
         name=DEFAULT_LOGGER_NAME,
         transport=BackgroundThreadTransport,
-        resource=_GLOBAL_RESOURCE,
+        resource=None,
         labels=None,
         stream=None,
     ):
@@ -78,12 +78,15 @@ def __init__(
                 :class:`.BackgroundThreadTransport`. The other
                 option is :class:`.SyncTransport`.
             resource (~logging_v2.resource.Resource):
-                Resource for this Handler. Defaults to ``GLOBAL_RESOURCE``.
+                Resource for this Handler. If not given, will be inferred from the environment.
             labels (Optional[dict]): Monitored resource of the entry, defaults
                 to the global resource type.
             stream (Optional[IO]): Stream to be used by the handler.
         """
         super(CloudLoggingHandler, self).__init__(stream)
+        if not resource:
+            # infer the correct monitored resource from the local environment
+            resource = detect_resource(client.project)
         self.name = name
         self.client = client
         self.transport = transport(client, name)