Opta destroy now purges the layer of modules w/o existing resources (#473)

* Opta destroy now purges the layer of modules w/o existing resources

I think that this makes sense-- if there are no resources to destroy, then forget about it. Takes a lot of headache away.

* testfixing

Author: juandiegopalomino

opta/commands/destroy.py

@@ -96,7 +96,8 @@ def destroy(
             abort=True,
         )
         tf_flags.append("-auto-approve")
-
+    modules = Terraform.get_existing_modules(layer)
+    layer.modules = [x for x in layer.modules if x.name in modules]
     gen_all(layer)
     Terraform.init(False, "-reconfigure", layer=layer)
     logger.info(f"Destroying {layer.name}")

opta/core/terraform.py

@@ -163,22 +163,17 @@ def destroy_resources(
 
     @classmethod
     def destroy_all(cls, layer: "Layer", *tf_flags: str) -> None:
-        existing_modules = Terraform.get_existing_modules(layer)
-        targets = list(map(lambda x: f"-target=module.{x}", sorted(existing_modules)))
 
         # Refreshing the state is necessary to update terraform outputs.
         # This includes fetching the latest EKS cluster auth token, which is
         # necessary for destroying many k8s resources.
-        cls.refresh(layer, *targets)
+        cls.refresh(layer)
         kwargs = cls.insert_extra_env(layer)
 
         idx = len(layer.modules) - 1
         for module in reversed(layer.modules):
             module_address_prefix = f"module.{module.name}"
 
-            if module.name not in existing_modules:
-                idx -= 1
-                continue
             cls.refresh(layer, f"-target={module_address_prefix}")
             nice_run(
                 ["terraform", "destroy", f"-target={module_address_prefix}", *tf_flags],

opta/module_processors/datadog.py

@@ -9,7 +9,6 @@
 from opta.core.kubernetes import configure_kubectl
 from opta.exceptions import UserErrors
 from opta.module_processors.base import ModuleProcessor
-from opta.utils import exp_backoff, logger
 
 if TYPE_CHECKING:
     from opta.layer import Layer
@@ -22,52 +21,40 @@ def __init__(self, module: "Module", layer: "Layer"):
             raise Exception(
                 f"The module {module.name} was expected to be of type datadog"
             )
-        # If the k8s cluster was recently created, it may take some time for it to be ready.
-        for _ in exp_backoff(num_tries=3):
-            try:
-                configure_kubectl(layer)
-                break
-            except Exception as err:
-                logger.exception(str(err))
-                logger.info("Retrying attempt to talk to K8s cluster...")
-        else:
-            raise Exception("Couldn't connect to the K8s cluster")
-
-        load_kube_config()
-        self.v1 = CoreV1Api()
         super(DatadogProcessor, self).__init__(module, layer)
 
     def process(self, module_idx: int) -> None:
+        configure_kubectl(self.layer)
+        load_kube_config()
+        v1 = CoreV1Api()
         # Update the secrets
-        namespaces = self.v1.list_namespace(
-            field_selector=f"metadata.name={self.layer.name}"
-        )
+        namespaces = v1.list_namespace(field_selector=f"metadata.name={self.layer.name}")
         if len(namespaces.items) == 0:
-            self.v1.create_namespace(
+            v1.create_namespace(
                 body=V1Namespace(metadata=V1ObjectMeta(name=self.layer.name))
             )
         try:
-            secret = self.v1.read_namespaced_secret("secret", self.layer.name)
+            secret = v1.read_namespaced_secret("secret", self.layer.name)
             if (
                 "DATADOG_API_KEY" not in secret.data
                 or secret.data["DATADOG_API_KEY"] == ""
             ):
-                api_key = self.create_secret()
+                api_key = self.create_secret(v1)
             else:
                 api_key = base64.b64decode(secret.data["DATADOG_API_KEY"]).decode("utf-8")
         except ApiException:
-            self.v1.create_namespaced_secret(
+            v1.create_namespaced_secret(
                 namespace=self.layer.name,
                 body=V1Secret(
                     metadata=V1ObjectMeta(name="secret"),
                     string_data={"DATADOG_API_KEY": ""},
                 ),
             )
-            api_key = self.create_secret()
+            api_key = self.create_secret(v1)
         self.module.data["api_key"] = api_key
         super(DatadogProcessor, self).process(module_idx)
 
-    def create_secret(self) -> str:
+    def create_secret(self, v1: CoreV1Api) -> str:
         value = self.module.data.get("api_key") or click.prompt(
             "Please enter your datadog api key (from https://app.datadoghq.com/account/settings#api)",
             type=click.STRING,
@@ -83,7 +70,7 @@ def create_secret(self) -> str:
         patch = [
             {"op": "replace", "path": "/data/DATADOG_API_KEY", "value": secret_value}
         ]
-        self.v1.patch_namespaced_secret("secret", self.layer.name, patch)
+        v1.patch_namespaced_secret("secret", self.layer.name, patch)
         return value
 
     def validate_api_key(self, api_key: str) -> bool:

tests/commands/test_destroy.py

@@ -4,6 +4,7 @@
 from pytest_mock import MockFixture
 
 from opta.commands.destroy import destroy
+from opta.exceptions import UserErrors
 from tests.util import get_call_args
 
 FAKE_ENV_CONFIG = os.path.join(
@@ -42,6 +43,7 @@ def test_destroy_env_with_children(mocker: MockFixture) -> None:
     result = runner.invoke(destroy, ["--config", FAKE_ENV_CONFIG])
 
     assert result.exit_code == 1
+    assert isinstance(result.exception, UserErrors)
 
     assert not mocked_gen_all.called
 
@@ -53,6 +55,9 @@ def test_destroy_env_without_children(mocker: MockFixture) -> None:
     mocker.patch("opta.commands.destroy.amplitude_client.send_event")
     mocker.patch("opta.commands.destroy.Terraform.init")
     mocker.patch("opta.commands.destroy.Terraform.destroy_all")
+    mocker.patch(
+        "opta.commands.destroy.Terraform.get_existing_modules", return_value={"base"}
+    )
     mocker.patch("opta.commands.destroy.Terraform.download_state", return_value=True)
     mocker.patch("opta.commands.destroy.Layer.verify_cloud_credentials")
 
@@ -80,6 +85,9 @@ def test_destroy_service(mocker: MockFixture) -> None:
     mocker.patch("opta.commands.destroy.amplitude_client.send_event")
     mocker.patch("opta.commands.destroy.Terraform.init")
     mocker.patch("opta.commands.destroy.Terraform.destroy_all")
+    mocker.patch(
+        "opta.commands.destroy.Terraform.get_existing_modules", return_value={"base"}
+    )
     mocker.patch("opta.commands.destroy.Terraform.download_state", return_value=True)
     mocker.patch("opta.commands.destroy.Layer.verify_cloud_credentials")