Add the ability to have activation config secrets in a separate file (#95)

* Add --secrets flag to dt-sdk run command

* Add secrets.json to default .gitignore file

* Add secrets.json to template, adjust activation.json template, update docs

* Make secrets DebugClient argument optional, revert tests, add secrets.json to docs file structure chart

Author: tomas-s-roos

docs/guides/extension_structure.rst

@@ -14,6 +14,7 @@ The basic bare-bone structure of an extension is as follows:
     ├── my_extension
     │   ├── __init__.py
     │   └── __main__.py
+    ├── secrets.json
     └── setup.py
 
 This structure can be generated by running the :doc:`/cli/create` command
@@ -195,7 +196,8 @@ Activation Config
 
 This is a config file that can be used for local testing, when extension instance
 is launched using the :doc:`/cli/run` command. It must contain all of the mandatory
-fields as defined in the `Activation Schema`_.
+fields as defined in the `Activation Schema`_. This file also supports using secrets
+from the ``secrets.json`` file.
 
 When extension is deployed to the Dynatrace environment and monitoring configuration
 is created, then the environment provides an individual activation config for each
@@ -215,12 +217,32 @@ Here is what a sample activation config looks like:
                 {
                     "url": "http://127.0.0.1:15672",
                     "user": "guest",
-                    "password": "guest"
+                    "password": "{{myPassword}}"
                 }
             ]
         }
     }
 
+
+Secrets
+-----------------
+
+``secrets.json``
+
+This file defines the secrets that can be used in ``activation.json``. Specify 
+each secret as key-value pair and then use ``{{key}}`` in the activation config 
+file to use the corresponding secret's value. Only secrets of type ``string`` 
+are supported.
+
+This file is by default in ``.gitignore``.
+
+.. code:: json
+
+    {
+        "myPassword": "secretPassword"
+    }
+
+
 Setup.py
 --------
 

docs/index.rst

@@ -78,6 +78,7 @@ structure:
    ├── my_extension
    │   ├── __init__.py
    │   └── __main__.py
+   ├── secrets.json
    └── setup.py
 
 .. admonition:: What do these files mean?

dynatrace_extension/__about__.py

@@ -3,4 +3,4 @@
 # SPDX-License-Identifier: MIT
 
 
-__version__ = "1.2.14"
+__version__ = "1.3.0"

dynatrace_extension/cli/create/extension_template/.gitignore.template

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Secrets
+secrets.json

dynatrace_extension/cli/create/extension_template/activation.json.template

@@ -8,7 +8,7 @@
 			{
 				"url": "http://127.0.0.1:15672",
 				"user": "guest",
-				"password": "guest"
+				"password": "{{myPassword}}"
 			}
 		]
 	}

dynatrace_extension/cli/create/extension_template/secrets.json.template

@@ -0,0 +1,3 @@
+{
+	"myPassword": "secretPassword"
+}

dynatrace_extension/cli/main.py

@@ -40,6 +40,7 @@ def version():
 def run(
     extension_dir: Path = typer.Argument("."),
     activation_config: str = "activation.json",
+    secrets: str = "secrets.json",
     fast_check: bool = typer.Option(False, "--fastcheck"),
     local_ingest: bool = typer.Option(False, "--local-ingest"),
     local_ingest_port: int = typer.Option(14499, "--local-ingest-port"),
@@ -50,6 +51,7 @@ def run(
 
     :param extension_dir: The directory of the extension, by default this is the current directory
     :param activation_config: The activation config file, by default this is activation.json
+    :param secrets: The secrets file to be used to enrich the activation config, by default this is secrets.json
     :param fast_check: If true, run a fastcheck and exits
     :param local_ingest: If true, send metrics to localhost:14499 on top of printing them
     :param local_ingest_port: The port to send metrics to, by default this is 14499
@@ -59,7 +61,15 @@ def run(
     # This parses the yaml, which validates it before running
     extension_yaml = ExtensionYaml(extension_dir / "extension/extension.yaml")
     try:
-        command = [sys.executable, "-m", extension_yaml.python.runtime.module, "--activationconfig", activation_config]
+        command = [
+            sys.executable,
+            "-m",
+            extension_yaml.python.runtime.module,
+            "--activationconfig",
+            activation_config,
+            "--secrets",
+            secrets,
+        ]
         if fast_check:
             command.append("--fastcheck")
         if local_ingest:

dynatrace_extension/sdk/communication.py

@@ -347,14 +347,24 @@ def __init__(
         activation_config_path: str,
         extension_config_path: str,
         logger: logging.Logger,
+        secrets_path: str = "secrets.json",
         local_ingest: bool = False,
         local_ingest_port: int = 14499,
         print_metrics: bool = True,
     ):
+
+        self.secrets = {}
+        if secrets_path and Path(secrets_path).exists():
+            with open(secrets_path) as f:
+                self.secrets = json.load(f)
+
         self.activation_config = {}
         if activation_config_path and Path(activation_config_path).exists():
             with open(activation_config_path) as f:
-                self.activation_config = json.load(f)
+                raw_activation_config = f.read()
+                self.activation_config = json.loads(
+                    self.replace_secrets_in_activation_config(self.secrets, raw_activation_config)
+                )
 
         self.extension_config = ""
         if not extension_config_path:
@@ -453,6 +463,12 @@ def send_sfm_metrics(self, mint_lines: list[str]) -> MintResponse:
     def get_cluster_time_diff(self) -> int:
         return 0
 
+    def replace_secrets_in_activation_config(self, secrets: dict, activation_config_string: str) -> str:
+        for secret_name, secret_value in secrets.items():
+            activation_config_string = activation_config_string.replace(f"{{{{{secret_name}}}}}", str(secret_value))
+
+        return activation_config_string
+
 
 def divide_into_batches(
     items: Sequence[dict | str], max_size_bytes: int, join_with: str | None = None

dynatrace_extension/sdk/extension.py

@@ -715,6 +715,7 @@ def _parse_args(self):
         # Debug parameters, these are used when running the extension locally
         parser.add_argument("--extensionconfig", required=False, default=None)
         parser.add_argument("--activationconfig", required=False, default="activation.json")
+        parser.add_argument("--secrets", required=False, default="secrets.json")
         parser.add_argument("--no-print-metrics", required=False, action="store_true")
 
         args, unknown = parser.parse_known_args()
@@ -727,6 +728,7 @@ def _parse_args(self):
                 activation_config_path=args.activationconfig,
                 extension_config_path=args.extensionconfig,
                 logger=api_logger,
+                secrets_path=args.secrets,
                 local_ingest=args.local_ingest,
                 local_ingest_port=args.local_ingest_port,
                 print_metrics=print_metrics,