integ-test: skip LMI test and improve test quality

.github/workflows/integration-tests.yml

@@ -24,7 +24,7 @@ env:
   CARGO_LAMBDA_VERSION: "v0.17.1"
   NOSE_PARAMETERIZED_NO_WARN: 1
   BY_CANARY: true
-  UV_PYTHON: python3.9
+  UV_PYTHON: python3.11
   CREDENTIAL_DISTRIBUTION_LAMBDA_ARN: ${{ secrets.CREDENTIAL_DISTRIBUTION_LAMBDA_ARN }}
   ACCOUNT_RESET_LAMBDA_ARN: ${{ secrets.ACCOUNT_RESET_LAMBDA_ARN }}
 
@@ -297,33 +297,52 @@ jobs:
       - name: Get testing resources and credentials
         run: |
           # Try with skip_role_deletion parameter first
-          test_env_var=$(python3.9 tests/get_testing_resources.py skip_role_deletion)
+          test_env_var=$(python3.11 tests/get_testing_resources.py skip_role_deletion)
 
           if [ $? -ne 0 ]; then
             echo "First attempt with skip_role_deletion failed, trying without parameter..."
-            test_env_var=$(python3.9 tests/get_testing_resources.py)
+            test_env_var=$(python3.11 tests/get_testing_resources.py)
 
             if [ $? -ne 0 ]; then
               echo "get_testing_resources failed. Failed to acquire credentials or test resources."
               exit 1
             fi
           fi
 
-          # Save current credentials for account reset later
+          # Save current credentials for account reset later (mask them first)
+          echo "::add-mask::$AWS_ACCESS_KEY_ID"
+          echo "::add-mask::$AWS_SECRET_ACCESS_KEY"
+          echo "::add-mask::$AWS_SESSION_TOKEN"
           echo "CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV
           echo "CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV
           echo "CI_ACCESS_ROLE_AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV
 
-          # Set test credentials
-          echo "AWS_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")" >> $GITHUB_ENV
-          echo "AWS_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")" >> $GITHUB_ENV
-          echo "AWS_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")" >> $GITHUB_ENV
-          echo "TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")" >> $GITHUB_ENV
+          # Extract test credentials and mask them before setting as env vars
+          TEST_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")
+          TEST_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")
+          TEST_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")
+          TEST_TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")
+
+          # Mask sensitive credentials so they don't appear in logs
+          echo "::add-mask::$TEST_ACCESS_KEY_ID"
+          echo "::add-mask::$TEST_SECRET_ACCESS_KEY"
+          echo "::add-mask::$TEST_SESSION_TOKEN"
+          echo "::add-mask::$TEST_TASK_TOKEN"
+
+          # Set test credentials as environment variables
+          echo "AWS_ACCESS_KEY_ID=$TEST_ACCESS_KEY_ID" >> $GITHUB_ENV
+          echo "AWS_SECRET_ACCESS_KEY=$TEST_SECRET_ACCESS_KEY" >> $GITHUB_ENV
+          echo "AWS_SESSION_TOKEN=$TEST_SESSION_TOKEN" >> $GITHUB_ENV
+          echo "TASK_TOKEN=$TEST_TASK_TOKEN" >> $GITHUB_ENV
+
+          # Set other test resources (non-sensitive)
           echo "AWS_S3_TESTING=$(echo "$test_env_var" | jq -j ".TestBucketName")" >> $GITHUB_ENV
           echo "AWS_ECR_TESTING=$(echo "$test_env_var" | jq -j ".TestECRURI")" >> $GITHUB_ENV
           echo "AWS_KMS_KEY=$(echo "$test_env_var" | jq -j ".TestKMSKeyArn")" >> $GITHUB_ENV
           echo "AWS_SIGNING_PROFILE_NAME=$(echo "$test_env_var" | jq -j ".TestSigningProfileName")" >> $GITHUB_ENV
           echo "AWS_SIGNING_PROFILE_VERSION_ARN=$(echo "$test_env_var" | jq -j ".TestSigningProfileARN")" >> $GITHUB_ENV
+          echo "LMI_SUBNET_ID=$(echo "$test_env_var" | jq -j ".LMISubnetId")" >> $GITHUB_ENV
+          echo "LMI_SECURITY_GROUP_ID=$(echo "$test_env_var" | jq -j ".LMISecurityGroupId")" >> $GITHUB_ENV
 
       - name: Login to Public ECR
         if: matrix.container_runtime != 'no-container' && env.BY_CANARY == 'true'
@@ -368,13 +387,13 @@ jobs:
               pytest -vv -n 2 --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py -m 'java' -k "${CONTAINER_FILTER}" --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64-java-${{ matrix.container_runtime }}.json
               ;;
             "terraform-build")
-              pytest -vv -n 4 --reruns 4 tests/integration/buildcmd/test_build_terraform_applications.py tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-terraform-${{ matrix.container_runtime }}.json
+              pytest -vv -n 4 --reruns 3 tests/integration/buildcmd/test_build_terraform_applications.py tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-terraform-${{ matrix.container_runtime }}.json
               ;;
             "package-delete-deploy")
               pytest -vv tests/integration/package tests/integration/delete tests/integration/deploy --dist=loadgroup -n 6 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package-delete-deploy-${{ matrix.container_runtime }}.json
               ;;
             "sync")
-              pytest -vv tests/integration/sync -n 6 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync-${{ matrix.container_runtime }}.json
+              pytest -vv tests/integration/sync -n 4 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync-${{ matrix.container_runtime }}.json
               ;;
             "local-invoke")
               pytest -vv --reruns 3 tests/integration/local/invoke tests/integration/local/generate_event --ignore tests/integration/local/invoke/test_invoke_durable.py --json-report --json-report-file=TEST_REPORT-integration-local-invoke-${{ matrix.container_runtime }}.json
@@ -389,8 +408,7 @@ jobs:
               pytest -vv --reruns 3 tests/integration/local/invoke/test_invoke_durable.py tests/integration/local/start_api/test_start_api_durable.py tests/integration/local/start_lambda/test_start_lambda_durable.py tests/integration/local/callback/test_callback.py tests/integration/local/execution/test_execution.py --json-report --json-report-file=TEST_REPORT-integration-durable-functions-${{ matrix.container_runtime }}.json
               ;;
             "other-and-e2e")
-              pytest -vv -n 4 --reruns 4 --dist loadgroup tests/integration tests/end_to_end --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-others-${{ matrix.container_runtime }}.json
-              pytest -vv --reruns 3 tests/regression --json-report --json-report-file=TEST_REPORT-regression-${{ matrix.container_runtime }}.json
+              pytest -vv -n 4 --reruns 3 --dist loadgroup tests/integration tests/end_to_end tests/regression --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-others-${{ matrix.container_runtime }}.json
               ;;
           esac
 

pytest.ini

@@ -16,7 +16,7 @@ filterwarnings =
 ;   Ignore dateparser deprecation warnings about parsing dates without years (Python 3.13+)
     ignore:.*Parsing dates involving a day of month without a year specified.*:DeprecationWarning
 ;   The following deprecation warnings are treated as failures unless we explicitly tell pytest not to
-;   Remove once we no longer support python3.7
+;   Remove once we no longer support python3.9
     ignore::boto3.exceptions.PythonDeprecationWarning
 markers =
     ruby

tests/integration/deploy/test_deploy_command.py

@@ -12,7 +12,13 @@
 from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, SamConfig
 from samcli.local.docker.utils import get_validated_container_client
 from tests.integration.deploy.deploy_integ_base import DeployIntegBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, UpdatableSARTemplate
+from tests.testing_utils import (
+    RUNNING_ON_CI,
+    RUNNING_TEST_FOR_MASTER_ON_CI,
+    RUN_BY_CANARY,
+    SKIP_LMI_TESTS,
+    UpdatableSARTemplate,
+)
 
 # Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
 # This is to restrict package tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary
@@ -1750,17 +1756,15 @@ def test_deploy_with_valid_config_capabilities_string(self, template_file, confi
         deploy_process_execute = self.run_command(deploy_command_list)
         self.assertEqual(deploy_process_execute.process.returncode, 0)
 
-    @skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
+    @skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
     def test_deploy_lmi_function(self):
         """Test deployment of LMI (Lambda Managed Infrastructure) functions with capacity providers."""
         # Validate LMI environment variables are set
         lmi_subnet_id = os.environ.get("LMI_SUBNET_ID")
         lmi_security_group_id = os.environ.get("LMI_SECURITY_GROUP_ID")
-        lmi_operator_role_arn = os.environ.get("LMI_OPERATOR_ROLE_ARN")
 
         self.assertTrue(lmi_subnet_id, "LMI_SUBNET_ID environment variable must be set")
         self.assertTrue(lmi_security_group_id, "LMI_SECURITY_GROUP_ID environment variable must be set")
-        self.assertTrue(lmi_operator_role_arn, "LMI_OPERATOR_ROLE_ARN environment variable must be set")
 
         template_path = self.test_data_path.joinpath("lmi_function", "template.yaml")
         stack_name = self._method_to_stack_name(self.id())

tests/integration/local/callback/test_callback.py

@@ -48,9 +48,14 @@ def test_callback_already_completed_execution(self, action, operation_name, call
         result = run_command(succeed_command)
         self.assertEqual(result.process.returncode, 0)
 
-        # Wait for process to complete
+        # Wait for process to complete and close file handles
         process.wait(timeout=30)
         thread.join(timeout=5)
+        # Close file handles to prevent ResourceWarning about unclosed files
+        if process.stdin:
+            process.stdin.close()
+        if process.stdout:
+            process.stdout.close()
 
         # Try to send another callback (should fail)
         second_command = self.get_callback_command_list(action, callback_id)

tests/integration/local/invoke/invoke_integ_base.py

@@ -111,6 +111,12 @@ def run_command(self, command_list, env=None, cwd=None):
             return stdout, stderr, process.returncode
         except TimeoutExpired:
             process.kill()
+            # Close file handles to prevent resource warnings
+            if process.stdout:
+                process.stdout.close()
+            if process.stderr:
+                process.stderr.close()
+            process.wait()
             raise
 
 

tests/integration/pipeline/test_bootstrap_command.py

@@ -356,9 +356,7 @@ def test_interactive_pipeline_user_only_created_once(self):
                 "y",  # Create resources confirmation
             ]
 
-            bootstrap_process_execute = run_command_with_input(
-                bootstrap_command_list, ("\n".join(inputs) + "\n").encode()
-            )
+            bootstrap_process_execute = run_command_with_inputs(bootstrap_command_list, inputs)
 
             self.assertEqual(bootstrap_process_execute.process.returncode, 0)
             stdout = bootstrap_process_execute.stdout.decode()

tests/integration/remote/invoke/test_lambda_invoke_response_stream.py

@@ -1,10 +1,10 @@
 import json
 import os
 import uuid
-from unittest import skipIf
+from unittest import skipIf, SkipTest
 
 from tests.integration.remote.invoke.remote_invoke_integ_base import RemoteInvokeIntegBase
-from tests.testing_utils import run_command, RUNNING_ON_CI
+from tests.testing_utils import SKIP_LMI_TESTS, run_command, RUNNING_ON_CI
 
 from pathlib import Path
 import pytest
@@ -105,18 +105,19 @@ def test_invoke_different_boto_options(self):
         self.assertEqual(response_event_stream, expected_output_result)
 
 
-@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
+@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
 class TestInvokeResponseStreamingCapacityProvider(RemoteInvokeIntegBase):
     template = Path("template-lambda-response-capacity-provider-stream-fn.yaml")
 
     @classmethod
     def setUpClass(cls):
+        if SKIP_LMI_TESTS:
+            raise SkipTest("Skip LMI tests when running on canary")
         super().setUpClass()
         cls.stack_name = f"{cls.__name__}-{uuid.uuid4().hex}"
         # LMI is Lambda Managed Instance
         assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
         assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
-        assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
 
         # Read LMI infrastructure from environment variables
         cls.parameter_overrides = {

tests/integration/remote/invoke/test_remote_invoke.py

@@ -6,29 +6,30 @@
 import os
 
 from parameterized import parameterized
-from unittest import skipIf
+from unittest import skipIf, SkipTest
 
 from tests.integration.remote.invoke.remote_invoke_integ_base import RemoteInvokeIntegBase
-from tests.testing_utils import run_command, RUNNING_ON_CI
+from tests.testing_utils import SKIP_LMI_TESTS, run_command
 
 from pathlib import Path
 import pytest
 
 SQS_WAIT_TIME_SECONDS = 20
 
 
-@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
+@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
 class TestingInvokeWithCapacityProvider(RemoteInvokeIntegBase):
     template = Path("template-single-lambda-capacity-provider.yaml")
 
     @classmethod
     def setUpClass(cls):
+        if SKIP_LMI_TESTS:
+            raise SkipTest("Skip LMI tests when running on canary")
         super().setUpClass()
         cls.stack_name = f"{cls.__name__}-{uuid.uuid4().hex}"
         # LMI is Lambda Managed Instance
         assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
         assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
-        assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
 
         # Read LMI infrastructure from environment variables
         cls.parameter_overrides = {

tests/integration/sync/test_sync_code.py

@@ -9,7 +9,7 @@
 import uuid
 from pathlib import Path
 from typing import Dict
-from unittest import skipIf
+from unittest import skipIf, SkipTest
 
 
 import pytest
@@ -24,7 +24,7 @@
 )
 from tests.integration.sync.sync_integ_base import SyncIntegBase
 
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, SKIP_LMI_TESTS
 from tests.testing_utils import run_command_with_input
 
 # Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
@@ -796,14 +796,14 @@ def test_skip_build(self):
                 self.assertEqual(lambda_response.get("message"), "hello mars")
 
 
-@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
 @parameterized_class(
     [
         {"dependency_layer": True},
         {"dependency_layer": False},
     ]
 )
 @pytest.mark.timeout(300)
+@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
 class TestSyncCodeLMI(TestSyncCodeBase):
     """Test sync code operations with Lambda Managed Instance functions"""
 
@@ -812,11 +812,12 @@ class TestSyncCodeLMI(TestSyncCodeBase):
 
     @classmethod
     def setUpClass(cls) -> None:
+        if SKIP_LMI_TESTS:
+            raise SkipTest("Skip LMI tests when running on canary")
         super().setUpClass()
         # Validate LMI environment variables are set
         assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
         assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
-        assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
 
         # Read LMI infrastructure from environment variables
         cls.parameter_overrides = {

tests/integration/sync/test_sync_infra.py

@@ -7,7 +7,7 @@
 import uuid
 from pathlib import Path
 from typing import Dict
-from unittest import skipIf
+from unittest import skipIf, SkipTest
 
 import pytest
 from parameterized import parameterized, parameterized_class
@@ -18,7 +18,7 @@
     AWS_STEPFUNCTIONS_STATEMACHINE,
 )
 from tests.integration.sync.sync_integ_base import SyncIntegBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, SKIP_LMI_TESTS
 from tests.testing_utils import run_command_with_input
 
 # Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
@@ -587,7 +587,7 @@ def test_sync_infra_esbuild(self, template_file):
             self.assertEqual(lambda_response.get("message"), "hello world")
 
 
-@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
+@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
 @parameterized_class(
     [
         {"dependency_layer": True},
@@ -603,11 +603,12 @@ class TestSyncInfraLMI(SyncIntegBase):
 
     @classmethod
     def setUpClass(cls) -> None:
+        if SKIP_LMI_TESTS:
+            raise SkipTest("Skip LMI tests when running on canary")
         super().setUpClass()
         # Validate LMI environment variables are set
         assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
         assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
-        assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
 
         # Read LMI infrastructure from environment variables
         cls.parameter_overrides = {

tests/integration/sync/test_sync_watch.py

@@ -8,7 +8,7 @@
 import json
 from pathlib import Path
 from typing import Dict, List
-from unittest import skipIf
+from unittest import skipIf, SkipTest
 
 import pytest
 import boto3
@@ -28,6 +28,7 @@
     RUNNING_ON_CI,
     RUNNING_TEST_FOR_MASTER_ON_CI,
     RUN_BY_CANARY,
+    SKIP_LMI_TESTS,
     kill_process,
     read_until_string,
     start_persistent_process,
@@ -57,6 +58,11 @@ class TestSyncWatchBase(SyncIntegBase):
     parameter_overrides: Dict[str, str] = {}
     watch_exclude: List[str] = []
 
+    @classmethod
+    def setUpClass(cls):
+        if not SKIP_SYNC_TESTS:
+            super().setUpClass()
+
     def setUp(self):
         # set up clean testing folder
         self.test_data_path = Path(tempfile.mkdtemp())
@@ -863,7 +869,7 @@ def test_sync_watch_code_excludes(self):
             self.assertEqual(lambda_response.get("message"), "hello world")
 
 
-@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
+@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
 @parameterized_class([{"dependency_layer": True}, {"dependency_layer": False}])
 @pytest.mark.timeout(600)  # 10 minutes timeout for LMI operations
 class TestSyncWatchCodeLMI(TestSyncWatchBase):
@@ -874,10 +880,13 @@ class TestSyncWatchCodeLMI(TestSyncWatchBase):
 
     @classmethod
     def setUpClass(cls) -> None:
+        # Skip the entire class if LMI tests should be skipped
+        if SKIP_LMI_TESTS:
+            raise SkipTest("Skip LMI tests when running on canary")
+
         # Validate LMI environment variables are set
         assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
         assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
-        assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
 
         cls.template_before = str(Path("code", "before", "template-lmi-with-publish.yaml"))
         cls.parameter_overrides = {