Resource retention Integration test (#194)

Issue #, if available:

Description of changes:
`test/e2e/resources/notebook_instance_lifecycle_retain.yaml` - Notebook Lifecycle CR with retain annotation
`test/e2e/tests/test_retain_policy.py` - Resource retention Integration test.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: ananth102

test/e2e/common/config.py

@@ -19,6 +19,7 @@
 DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL = "dataqualityjobdefinitions"
 MODEL_PACKAGE_GROUP_RESOURCE_PLURAL = "modelpackagegroups"
 MODEL_PACKAGE_RESOURCE_PLURAL = "modelpackages"
+NOTEBOOK_INSTANCE_LIFECYCLE_RESOURCE_PLURAL = "notebookinstancelifecycleconfigs"
 
 # Job Type Resource Statuses
 LIST_JOB_STATUS_STOPPED = ("Stopped", "Stopping", "Completed")

test/e2e/resources/notebook_instance_lifecycle_config.yaml

@@ -2,6 +2,8 @@ apiVersion: sagemaker.services.k8s.aws/v1alpha1
 kind: NotebookInstanceLifecycleConfig
 metadata:
   name: $NOTEBOOK_INSTANCE_LFC_NAME
+  annotations:
+    services.k8s.aws/deletion-policy: $DELETION_POLICY
 spec:
   notebookInstanceLifecycleConfigName: $NOTEBOOK_INSTANCE_LFC_NAME
   onStart:

test/e2e/tests/test_notebook_instance_lifecycle.py

@@ -32,18 +32,22 @@
 
 from e2e.replacement_values import REPLACEMENT_VALUES
 from time import sleep
+from e2e.common import config as cfg
 
 DELETE_WAIT_PERIOD = 16
 DELETE_PERIOD_LENGTH = 30
+UPDATE_WAIT_PERIOD_IN_SECONDS = 5
+RESOURCE_PLURAL = cfg.NOTEBOOK_INSTANCE_LIFECYCLE_RESOURCE_PLURAL
 
 
 @pytest.fixture(scope="module")
 def notebook_instance_lifecycleConfig():
     notebook_instance_lfc_name = random_suffix_name("notebookinstancelfc", 50)
     replacements = REPLACEMENT_VALUES.copy()
     replacements["NOTEBOOK_INSTANCE_LFC_NAME"] = notebook_instance_lfc_name
+    replacements["DELETION_POLICY"] = "delete"
     reference, spec, resource = create_sagemaker_resource(
-        resource_plural="notebookinstancelifecycleconfigs",
+        resource_plural=RESOURCE_PLURAL,
         resource_name=notebook_instance_lfc_name,
         spec_file="notebook_instance_lifecycle_config",
         replacements=replacements,
@@ -110,6 +114,10 @@ def test_create_update_delete(self, notebook_instance_lifecycleConfig):
         # We need to keep track of the current time so its best to just do
         # the update test with the create test.
         # update content is pip install six
+
+        # Sometimes creation can happen in under a second.
+        sleep(UPDATE_WAIT_PERIOD_IN_SECONDS)
+
         assert "lastModifiedTime" in resource["status"]
         last_modified_time = resource["status"]["lastModifiedTime"]
         update_content = "cGlwIGluc3RhbGwgc2l4"

test/e2e/tests/test_retain_policy.py

@@ -0,0 +1,113 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+"""Integration tests for the Resource retain policy
+"""
+
+import pytest
+import logging
+import botocore
+
+from acktest.k8s import resource as k8s
+from acktest.resources import random_suffix_name
+from e2e import (
+    service_marker,
+    create_sagemaker_resource,
+    sagemaker_client,
+)
+
+
+from e2e.replacement_values import REPLACEMENT_VALUES
+from e2e.common import config as cfg
+
+RESOURCE_PLURAL = cfg.NOTEBOOK_INSTANCE_LIFECYCLE_RESOURCE_PLURAL
+RESOURCE_SPEC_FILE = "notebook_instance_lifecycle_config"
+
+
+@pytest.fixture(scope="module")
+def retained_notebook_instance_lifecycle_config():
+    notebook_instance_lfc_name = random_suffix_name("notebookinstancelfc", 40)
+    replacements = REPLACEMENT_VALUES.copy()
+    replacements["NOTEBOOK_INSTANCE_LFC_NAME"] = notebook_instance_lfc_name
+    replacements["DELETION_POLICY"] = "retain"
+    reference, spec, resource = create_sagemaker_resource(
+        resource_plural=RESOURCE_PLURAL,
+        resource_name=notebook_instance_lfc_name,
+        spec_file=RESOURCE_SPEC_FILE,
+        replacements=replacements,
+    )
+    assert resource is not None
+    yield (reference, resource, spec)
+
+    deletion_resp = delete_notebook_instance_lifecycle_config(
+        notebook_instance_lfc_name
+    )
+    assert deletion_resp is not None
+
+
+def get_notebook_instance_lifecycle_config(notebook_instance_lfc_name: str):
+    try:
+        resp = sagemaker_client().describe_notebook_instance_lifecycle_config(
+            NotebookInstanceLifecycleConfigName=notebook_instance_lfc_name
+        )
+        return resp
+    except botocore.exceptions.ClientError as error:
+        logging.error(
+            f"SageMaker could not find a Notebook Instance Lifecycle Configuration with the name {notebook_instance_lfc_name}. Error {error}"
+        )
+        return None
+
+
+def delete_notebook_instance_lifecycle_config(notebook_instance_lfc_name: str):
+    try:
+        resp = sagemaker_client().delete_notebook_instance_lifecycle_config(
+            NotebookInstanceLifecycleConfigName=notebook_instance_lfc_name
+        )
+        return resp
+    except botocore.exceptions.ClientError as error:
+        logging.error(
+            f"SageMaker could not find a Notebook Instance Lifecycle Configuration with the name {notebook_instance_lfc_name}. Error {error}"
+        )
+        return None
+
+
+@service_marker
+class TestRetainPolicy:
+    def test_retain_notebook_instance_lifecycle(
+        self, retained_notebook_instance_lifecycle_config
+    ):
+        (reference, resource, _) = retained_notebook_instance_lifecycle_config
+        assert k8s.get_resource_exists(reference)
+
+        # Getting the resource name
+        notebook_instance_lfc_name = resource["spec"].get(
+            "notebookInstanceLifecycleConfigName", None
+        )
+        assert notebook_instance_lfc_name is not None
+        notebook_instance_lfc_desc = get_notebook_instance_lifecycle_config(
+            notebook_instance_lfc_name
+        )
+        assert (
+            k8s.get_resource_arn(resource)
+            == notebook_instance_lfc_desc["NotebookInstanceLifecycleConfigArn"]
+        )
+
+        # Delete the CR in Kubernetes
+        _, deleted = k8s.delete_custom_resource(reference)
+
+        assert deleted
+
+        # Verify that the resource was not deleted on SageMaker
+        assert (
+            get_notebook_instance_lifecycle_config(notebook_instance_lfc_name)
+            is not None
+        )