RCA Helpers (#265)

* Better package version management

* New snapshot fields; fix package version processing

* Log boot message to host agent

* Package init cleanup

* Fix double import

Author: Peter Giacomo Lombardo

instana/__init__.py

@@ -22,27 +22,38 @@
 from threading import Timer
 import pkg_resources
 
+from .version import VERSION
+
 __author__ = 'Instana Inc.'
 __copyright__ = 'Copyright 2020 Instana Inc.'
 __credits__ = ['Pavlo Baron', 'Peter Giacomo Lombardo', 'Andrey Slotin']
 __license__ = 'MIT'
 __maintainer__ = 'Peter Giacomo Lombardo'
 __email__ = '[email protected]'
+__version__ = VERSION
+
+# User configurable EUM API key for instana.helpers.eum_snippet()
+# pylint: disable=invalid-name
+eum_api_key = ''
 
-try:
-    __version__ = pkg_resources.get_distribution('instana').version
-except pkg_resources.DistributionNotFound:
-    __version__ = 'unknown'
+# This Python package can be loaded into Python processes one of three ways:
+#   1. manual import statement
+#   2. autowrapt hook
+#   3. dynamically injected remotely
+#
+# With such magic, we may get pulled into Python processes that we have no interest being in.
+# As a safety measure, we maintain a "do not load list" and if this process matches something
+# in that list, then we go sit in a corner quietly and don't load anything at all.
+do_not_load_list = ["pip", "pip2", "pip3", "pipenv", "docker-compose", "easy_install", "easy_install-2.7",
+                    "smtpd.py", "twine", "ufw", "unattended-upgrade"]
 
 
 def load(_):
     """
     Method used to activate the Instana sensor via AUTOWRAPT_BOOTSTRAP
     environment variable.
     """
-    if "INSTANA_DEBUG" in os.environ:
-        print("Instana: activated via AUTOWRAPT_BOOTSTRAP")
-
+    return None
 
 def get_lambda_handler_or_default():
     """
@@ -95,7 +106,7 @@ def lambda_handler(event, context):
 def boot_agent_later():
     """ Executes <boot_agent> in the future! """
     if 'gevent' in sys.modules:
-        import gevent
+        import gevent # pylint: disable=import-outside-toplevel
         gevent.spawn_later(2.0, boot_agent)
     else:
         Timer(2.0, boot_agent).start()
@@ -148,42 +159,20 @@ def boot_agent():
     # Hooks
     from .hooks import hook_uwsgi
 
-
-if "INSTANA_MAGIC" in os.environ:
-    pkg_resources.working_set.add_entry("/tmp/.instana/python")
-    # The following path is deprecated: To be removed at a future date
-    pkg_resources.working_set.add_entry("/tmp/instana/python")
-
-    if "INSTANA_DEBUG" in os.environ:
-        print("Instana: activated via AutoTrace")
-else:
-    if ("INSTANA_DEBUG" in os.environ) and ("AUTOWRAPT_BOOTSTRAP" not in os.environ):
-        print("Instana: activated via manual import")
-
-# User configurable EUM API key for instana.helpers.eum_snippet()
-# pylint: disable=invalid-name
-eum_api_key = ''
-
-# This Python package can be loaded into Python processes one of three ways:
-#   1. manual import statement
-#   2. autowrapt hook
-#   3. dynamically injected remotely
-#
-# With such magic, we may get pulled into Python processes that we have no interest being in.
-# As a safety measure, we maintain a "do not load list" and if this process matches something
-# in that list, then we go sit in a corner quietly and don't load anything at all.
-do_not_load_list = ["pip", "pip2", "pip3", "pipenv", "docker-compose", "easy_install", "easy_install-2.7",
-                    "smtpd.py", "twine", "ufw", "unattended-upgrade"]
-
-# There are cases when sys.argv may not be defined at load time.  Seems to happen in embedded Python,
-# and some Pipenv installs.  If this is the case, it's best effort.
-if hasattr(sys, 'argv') and len(sys.argv) > 0 and (os.path.basename(sys.argv[0]) in do_not_load_list):
-    if "INSTANA_DEBUG" in os.environ:
-        print("Instana: No use in monitoring this process type (%s).  "
-              "Will go sit in a corner quietly." % os.path.basename(sys.argv[0]))
-else:
-    if "INSTANA_MAGIC" in os.environ:
-        # If we're being loaded into an already running process, then delay agent initialization
-        boot_agent_later()
+if 'INSTANA_DISABLE' not in os.environ:
+    # There are cases when sys.argv may not be defined at load time.  Seems to happen in embedded Python,
+    # and some Pipenv installs.  If this is the case, it's best effort.
+    if hasattr(sys, 'argv') and len(sys.argv) > 0 and (os.path.basename(sys.argv[0]) in do_not_load_list):
+        if "INSTANA_DEBUG" in os.environ:
+            print("Instana: No use in monitoring this process type (%s).  "
+                  "Will go sit in a corner quietly." % os.path.basename(sys.argv[0]))
     else:
-        boot_agent()
+        if "INSTANA_MAGIC" in os.environ:
+            pkg_resources.working_set.add_entry("/tmp/.instana/python")
+            # The following path is deprecated: To be removed at a future date
+            pkg_resources.working_set.add_entry("/tmp/instana/python")
+
+            # If we're being loaded into an already running process, then delay agent initialization
+            boot_agent_later()
+        else:
+            boot_agent()

instana/agent/aws_fargate.py

@@ -6,8 +6,9 @@
 from instana.options import AWSFargateOptions
 from instana.collector.aws_fargate import AWSFargateCollector
 from ..log import logger
-from ..util import to_json, package_version
+from ..util import to_json
 from .base import BaseAgent
+from ..version import VERSION
 
 
 class AWSFargateFrom(object):
@@ -34,7 +35,7 @@ def __init__(self):
         # Update log level (if INSTANA_LOG_LEVEL was set)
         self.update_log_level()
 
-        logger.info("Stan is on the AWS Fargate scene.  Starting Instana instrumentation version: %s", package_version())
+        logger.info("Stan is on the AWS Fargate scene.  Starting Instana instrumentation version: %s", VERSION)
 
         if self._validate_options():
             self._can_send = True

instana/agent/aws_lambda.py

@@ -4,8 +4,9 @@
 """
 import time
 from ..log import logger
-from ..util import to_json, package_version
+from ..util import to_json
 from .base import BaseAgent
+from ..version import VERSION
 from ..collector.aws_lambda import AWSLambdaCollector
 from ..options import AWSLambdaOptions
 
@@ -34,7 +35,7 @@ def __init__(self):
         # Update log level from what Options detected
         self.update_log_level()
 
-        logger.info("Stan is on the AWS Lambda scene.  Starting Instana instrumentation version: %s", package_version())
+        logger.info("Stan is on the AWS Lambda scene.  Starting Instana instrumentation version: %s", VERSION)
 
         if self._validate_options():
             self._can_send = True

instana/agent/host.py

@@ -14,9 +14,10 @@
 from ..log import logger
 from .base import BaseAgent
 from ..fsm import TheMachine
+from ..version import VERSION
 from ..options import StandardOptions
 from ..collector.host import HostCollector
-from ..util import to_json, get_py_source, package_version
+from ..util import to_json, get_py_source
 
 
 class AnnounceData(object):
@@ -50,8 +51,8 @@ def __init__(self):
 
         # Update log level from what Options detected
         self.update_log_level()
-        
-        logger.info("Stan is on the scene.  Starting Instana instrumentation version: %s", package_version())
+
+        logger.info("Stan is on the scene.  Starting Instana instrumentation version: %s", VERSION)
 
         self.collector = HostCollector(self)
         self.machine = TheMachine(self)
@@ -186,6 +187,27 @@ def announce(self, discovery):
             logger.debug("announce: connection error (%s)", type(exc))
         return response
 
+    def log_message_to_host_agent(self, message):
+        """
+        Log a message to the discovered host agent
+        """
+        response = None
+        try:
+            payload = dict()
+            payload["m"] = message
+
+            url = self.__agent_logger_url()
+            response = self.client.post(url,
+                                       data=to_json(payload),
+                                       headers={"Content-Type": "application/json",
+                                                "X-Log-Level": "INFO"},
+                                       timeout=0.8)
+
+            if 200 <= response.status_code <= 204:
+                self.last_seen = datetime.now()
+        except Exception as exc:
+            logger.debug("agent logging: connection error (%s)", type(exc))
+
     def is_agent_ready(self):
         """
         Used after making a successful announce to test when the agent is ready to accept data.
@@ -214,7 +236,7 @@ def report_data_payload(self, payload):
                                             data=to_json(payload['spans']),
                                             headers={"Content-Type": "application/json"},
                                             timeout=0.8)
-            
+
             if response is not None and 200 <= response.status_code <= 204:
                 self.last_seen = datetime.now()
 
@@ -251,7 +273,7 @@ def handle_agent_tasks(self, task):
                 payload = get_py_source(task["args"]["file"])
             else:
                 message = "Unrecognized action: %s. An newer Instana package may be required " \
-                          "for this. Current version: %s" % (task["action"], package_version())
+                          "for this. Current version: %s" % (task["action"], VERSION)
                 payload = {"error": message}
         else:
             payload = {"error": "Instana Python: No action specified in request."}
@@ -303,3 +325,9 @@ def __response_url(self, message_id):
         """
         path = "com.instana.plugin.python/response.%d?messageId=%s" % (int(self.announce_data.pid), message_id)
         return "http://%s:%s/%s" % (self.options.agent_host, self.options.agent_port, path)
+
+    def __agent_logger_url(self):
+        """
+        URL for logging messages to the discovered host agent.
+        """
+        return "http://%s:%s/%s" % (self.options.agent_host, self.options.agent_port, "com.instana.agent.logger")

instana/collector/helpers/runtime.py

@@ -9,6 +9,7 @@
 from pkg_resources import DistributionNotFound, get_distribution
 
 from instana.log import logger
+from instana.version import VERSION
 from instana.util import DictionaryOfStan, determine_service_name
 
 from .base import BaseHelper
@@ -166,6 +167,14 @@ def _collect_runtime_snapshot(self,plugin_data):
             snapshot_payload['f'] = platform.python_implementation() # flavor
             snapshot_payload['a'] = platform.architecture()[0] # architecture
             snapshot_payload['versions'] = self.gather_python_packages()
+            snapshot_payload['iv'] = VERSION
+
+            if 'AUTOWRAPT_BOOTSTRAP' in os.environ:
+                snapshot_payload['m'] = 'Autowrapt'
+            elif 'INSTANA_MAGIC' in os.environ:
+                snapshot_payload['m'] = 'AutoTrace'
+            else:
+                snapshot_payload['m'] = 'Manual'
 
             try:
                 from django.conf import settings # pylint: disable=import-outside-toplevel
@@ -191,23 +200,30 @@ def gather_python_packages(self):
                 # Skip modules that begin with underscore
                 if ('.' in pkg_name) or pkg_name[0] == '_':
                     continue
+
+                # Skip builtins
+                if pkg_name in ["sys", "curses"]:
+                    continue
+
                 if sys_packages[pkg_name]:
                     try:
                         pkg_info = sys_packages[pkg_name].__dict__
-                        if "version" in pkg_info:
-                            versions[pkg_name] = self.jsonable(pkg_info["version"])
-                        elif "__version__" in pkg_info:
+                        if "__version__" in pkg_info:
                             if isinstance(pkg_info["__version__"], str):
                                 versions[pkg_name] = pkg_info["__version__"]
                             else:
                                 versions[pkg_name] = self.jsonable(pkg_info["__version__"])
+                        elif "version" in pkg_info:
+                            versions[pkg_name] = self.jsonable(pkg_info["version"])
                         else:
                             versions[pkg_name] = get_distribution(pkg_name).version
                     except DistributionNotFound:
                         pass
                     except Exception:
                         logger.debug("gather_python_packages: could not process module: %s", pkg_name)
-
+            
+            # Manually set our package version
+            versions['instana'] = VERSION
         except Exception:
             logger.debug("gather_python_packages", exc_info=True)
 

instana/fsm.py

@@ -11,6 +11,7 @@
 
 from .log import logger
 from .util import get_default_gateway
+from .version import VERSION
 
 
 class Discovery(object):
@@ -58,7 +59,8 @@ def __init__(self, agent):
                 # "onchangestate":  self.print_state_change,
                 "onlookup":       self.lookup_agent_host,
                 "onannounce":     self.announce_sensor,
-                "onpending":      self.on_ready}})
+                "onpending":      self.on_ready,
+                "ongood2go":      self.on_good2go}})
 
         self.timer = threading.Timer(1, self.fsm.lookup)
         self.timer.daemon = True
@@ -173,8 +175,17 @@ def schedule_retry(self, fun, e, name):
 
     def on_ready(self, _):
         self.agent.start()
-        logger.info("Instana host agent available. We're in business. Announced pid: %s (true pid: %s)",
-                    str(os.getpid()), str(self.agent.announce_data.pid))
+
+        ns_pid = str(os.getpid())
+        true_pid = str(self.agent.announce_data.pid)
+
+        logger.info("Instana host agent available. We're in business. Announced PID: %s (true pid: %s)", ns_pid, true_pid)
+
+    def on_good2go(self, _):
+        ns_pid = str(os.getpid())
+        true_pid = str(self.agent.announce_data.pid)
+
+        self.agent.log_message_to_host_agent("Instana Python Package %s: PID %s (true pid: %s) is now online and reporting" % (VERSION, ns_pid, true_pid))
 
     def __get_real_pid(self):
         """

instana/version.py

@@ -0,0 +1,3 @@
+# Module version file.  Used by setup.py and snapshot reporting.
+
+VERSION = '1.25.6dev1'

setup.py

@@ -1,10 +1,14 @@
 # coding: utf-8
+import os
 import sys
 from os import path
 from distutils.version import LooseVersion
 from setuptools import find_packages, setup
 
-VERSION = '1.25.5'
+os.environ["INSTANA_DISABLE"] = "true"
+
+# pylint: disable=wrong-import-position
+from instana.version import VERSION
 
 # Import README.md into long_description
 pwd = path.abspath(path.dirname(__file__))