Add Agent Control Health Checks (#1294)

* Initial commit.

* Add status change logic.

* Checkpoint.

* Remove unused arg.

* Clear configuration leaking from test_configuration.py.

* Remove log statement when fleet ID is not found.

* Add support testing and support for new status codes.

* Update thread names to use NR-Control.

* Refactoring.

* Address review feedback.

* Capture urlparse logic in try except.

* Add additional shutdown check in shutdown_agent.

* Refactor status code and message usage in superagent

* Update regex assertion

Co-authored-by: Timothy Pansino <[email protected]>

* Add super agent supportability metric.

* Add newline.

* Rename super agent to agent control.

* Fix supportability metric test.

* Update agent control config.

* Add explicit check for empty delivery location.

* Change delivery location to property on health check class.

* Initial commit.

* Add status change logic.

* Checkpoint.

* Remove unused arg.

* Clear configuration leaking from test_configuration.py.

* Remove log statement when fleet ID is not found.

* Add support testing and support for new status codes.

* Update thread names to use NR-Control.

* Refactoring.

* Address review feedback.

* Capture urlparse logic in try except.

* Add additional shutdown check in shutdown_agent.

* Refactor status code and message usage in superagent

* Update regex assertion

Co-authored-by: Timothy Pansino <[email protected]>

* Add super agent supportability metric.

* Add newline.

* Rename super agent to agent control.

* Fix supportability metric test.

* Use environ_as_bool/ int.

* Add max 3 app names unhealthy status code.

* Switch order of tests to avoid config collisions.

* Reset app name.

* Reuse app name list variable.

* Fix line spacing.

* [Mega-Linter] Apply linters fixes

* Initial commit.

* Add status change logic.

* Checkpoint.

* Remove unused arg.

* Clear configuration leaking from test_configuration.py.

* Remove log statement when fleet ID is not found.

* Add support testing and support for new status codes.

* Update thread names to use NR-Control.

* Refactoring.

* Address review feedback.

* Capture urlparse logic in try except.

* Add additional shutdown check in shutdown_agent.

* Refactor status code and message usage in superagent

* Update regex assertion

Co-authored-by: Timothy Pansino <[email protected]>

* Add super agent supportability metric.

* Add newline.

* Rename super agent to agent control.

* Fix supportability metric test.

* Update agent control config.

* Add explicit check for empty delivery location.

* Change delivery location to property on health check class.

* Initial commit.

* Add status change logic.

* Checkpoint.

* Remove unused arg.

* Clear configuration leaking from test_configuration.py.

* Remove log statement when fleet ID is not found.

* Add support testing and support for new status codes.

* Update thread names to use NR-Control.

* Refactoring.

* Address review feedback.

* Capture urlparse logic in try except.

* Add additional shutdown check in shutdown_agent.

* Refactor status code and message usage in superagent

* Update regex assertion

Co-authored-by: Timothy Pansino <[email protected]>

* Rename super agent to agent control.

* Use environ_as_bool/ int.

* Add license header to HTTP client recorder.

---------

Co-authored-by: Tim Pansino <[email protected]>
Co-authored-by: Timothy Pansino <[email protected]>
Co-authored-by: umaannamalai <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 3 people

newrelic/config.py

@@ -17,6 +17,8 @@
 import logging
 import os
 import sys
+import threading
+import time
 import traceback
 
 import newrelic.api.application
@@ -40,12 +42,19 @@
 from newrelic.common.log_file import initialize_logging
 from newrelic.common.object_names import callable_name, expand_builtin_exception_name
 from newrelic.core import trace_cache
+from newrelic.core.agent_control_health import (
+    HealthStatus,
+    agent_control_health_instance,
+    agent_control_healthcheck_loop,
+)
 from newrelic.core.config import (
     Settings,
     apply_config_setting,
     default_host,
     fetch_config_setting,
 )
+from newrelic.core.agent_control_health import HealthStatus, agent_control_health_instance, agent_control_healthcheck_loop
+
 
 __all__ = ["initialize", "filter_app_factory"]
 
@@ -100,6 +109,7 @@ def _map_aws_account_id(s):
 # all the settings have been read.
 
 _cache_object = []
+agent_control_health = agent_control_health_instance()
 
 
 def _reset_config_parser():
@@ -592,12 +602,16 @@ def _process_app_name_setting():
     # primary application name and link it with the other applications.
     # When activating the application the linked names will be sent
     # along to the core application where the association will be
-    # created if the do not exist.
+    # created if it does not exist.
+
+    app_name_list = _settings.app_name.split(";")
+    name = app_name_list[0].strip() or "Python Application"
 
-    name = _settings.app_name.split(";")[0].strip() or "Python Application"
+    if len(app_name_list) > 3:
+        agent_control_health.set_health_status(HealthStatus.MAX_APP_NAME.value)
 
     linked = []
-    for altname in _settings.app_name.split(";")[1:]:
+    for altname in app_name_list[1:]:
         altname = altname.strip()
         if altname:
             linked.append(altname)
@@ -1033,21 +1047,25 @@ def _load_configuration(
 
     # Now read in the configuration file. Cache the config file
     # name in internal settings object as indication of succeeding.
-    if config_file.endswith(".toml"):
-        try:
-            import tomllib
-        except ImportError:
-            raise newrelic.api.exceptions.ConfigurationError(
-                "TOML configuration file can only be used if tomllib is available (Python 3.11+)."
-            )
-        with open(config_file, "rb") as f:
-            content = tomllib.load(f)
-            newrelic_section = content.get("tool", {}).get("newrelic")
-            if not newrelic_section:
-                raise newrelic.api.exceptions.ConfigurationError("New Relic configuration not found in TOML file.")
-            _config_object.read_dict(_toml_config_to_configparser_dict(newrelic_section))
-    elif not _config_object.read([config_file]):
-        raise newrelic.api.exceptions.ConfigurationError(f"Unable to open configuration file {config_file}.")
+    try:
+        if config_file.endswith(".toml"):
+            try:
+                import tomllib
+            except ImportError:
+                raise newrelic.api.exceptions.ConfigurationError(
+                    "TOML configuration file can only be used if tomllib is available (Python 3.11+)."
+                )
+            with open(config_file, "rb") as f:
+                content = tomllib.load(f)
+                newrelic_section = content.get("tool", {}).get("newrelic")
+                if not newrelic_section:
+                    raise newrelic.api.exceptions.ConfigurationError("New Relic configuration not found in TOML file.")
+                _config_object.read_dict(_toml_config_to_configparser_dict(newrelic_section))
+        elif not _config_object.read([config_file]):
+            raise newrelic.api.exceptions.ConfigurationError(f"Unable to open configuration file {config_file}.")
+    except Exception:
+        agent_control_health.set_health_status(HealthStatus.INVALID_CONFIG.value)
+        raise
 
     _settings.config_file = config_file
 
@@ -4826,13 +4844,29 @@ def _setup_agent_console():
         newrelic.core.agent.Agent.run_on_startup(_startup_agent_console)
 
 
+agent_control_health_thread = threading.Thread(
+    name="Agent-Control-Health-Main-Thread", target=agent_control_healthcheck_loop
+)
+agent_control_health_thread.daemon = True
+
+
+def _setup_agent_control_health():
+    if agent_control_health_thread.is_alive():
+        return
+
+    if agent_control_health.health_check_enabled:
+        agent_control_health_thread.start()
+
+
 def initialize(
     config_file=None,
     environment=None,
     ignore_errors=None,
     log_file=None,
     log_level=None,
 ):
+    agent_control_health.start_time_unix_nano = time.time_ns()
+
     if config_file is None:
         config_file = os.environ.get("NEW_RELIC_CONFIG_FILE", None)
 
@@ -4844,6 +4878,12 @@ def initialize(
 
     _load_configuration(config_file, environment, ignore_errors, log_file, log_level)
 
+    _setup_agent_control_health()
+
+    if _settings.monitor_mode:
+        if not _settings.license_key:
+            agent_control_health.set_health_status(HealthStatus.MISSING_LICENSE.value)
+
     if _settings.monitor_mode or _settings.developer_mode:
         _settings.enabled = True
         _setup_instrumentation()
@@ -4852,6 +4892,7 @@ def initialize(
         _setup_agent_console()
     else:
         _settings.enabled = False
+        agent_control_health.set_health_status(HealthStatus.AGENT_DISABLED.value)
 
 
 def filter_app_factory(app, global_conf, config_file, environment=None):

newrelic/console.py

@@ -36,6 +36,7 @@
 from newrelic.common.object_wrapper import ObjectProxy
 from newrelic.core.agent import agent_instance
 from newrelic.core.config import flatten_settings, global_settings
+from newrelic.core.agent_control_health import HealthStatus, agent_control_health_instance
 from newrelic.core.trace_cache import trace_cache
 
 
@@ -512,6 +513,8 @@ def __init__(self, config_file, stdin=None, stdout=None, log=None):
         self.__log_object = log
 
         if not self.__config_object.read([config_file]):
+            agent_control_instance = agent_control_health_instance()
+            agent_control_instance.set_health_status(HealthStatus.INVALID_CONFIG.value)
             raise RuntimeError(f"Unable to open configuration file {config_file}.")
 
         listener_socket = self.__config_object.get("newrelic", "console.listener_socket") % {"pid": "*"}

newrelic/core/agent.py

@@ -35,6 +35,7 @@
 from newrelic.samplers.cpu_usage import cpu_usage_data_source
 from newrelic.samplers.gc_data import garbage_collector_data_source
 from newrelic.samplers.memory_usage import memory_usage_data_source
+from newrelic.core.agent_control_health import HealthStatus, agent_control_health_instance
 
 _logger = logging.getLogger(__name__)
 
@@ -217,6 +218,9 @@ def __init__(self, config):
         self._scheduler = sched.scheduler(self._harvest_timer, self._harvest_shutdown.wait)
 
         self._process_shutdown = False
+        self._agent_control = agent_control_health_instance()
+
+        self._agent_control = agent_control_health_instance()
 
         self._lock = threading.Lock()
 
@@ -734,6 +738,11 @@ def shutdown_agent(self, timeout=None):
         if self._harvest_shutdown_is_set():
             return
 
+        self._agent_control.set_health_status(HealthStatus.AGENT_SHUTDOWN.value)
+
+        if self._agent_control.health_check_enabled:
+            self._agent_control.write_to_health_file()
+
         if timeout is None:
             timeout = self._config.shutdown_timeout