ci: improve patching and fix erroneous script paths for samcli-vm (runfinch#341)

This PR adds more patching for assisting with teardown issues in image deletion in SAMcli tests. These were new issues that arose after the previous merge. It also fixes the paths to scripts that were added for improved error handling.

Signed-off-by: ayush-panta <[email protected]>

Author: ayush-panta

.github/workflows/samcli-direct.yaml

@@ -2,6 +2,8 @@ name: samcli-direct
 
 on:
   pull_request:
+    branches:
+      - main
     types:
       - closed
   schedule:
@@ -58,8 +60,13 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Checkout finch-daemon repo
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Checkout finch-daemon
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+          persist-credentials: false
+          submodules: recursive
 
       - name: Stop pre-existing services
         run: |

.github/workflows/samcli-vm.yaml

@@ -159,8 +159,8 @@ jobs:
 
       - name: Patch SAM CLI for Docker image cleanup
         run: |
-          # Apply git patch to handle ImageNotFound exceptions for all Docker tests
-          su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && git apply ${{ github.workspace }}/finch-daemon-pr/scripts/samcli-vm/invoke-teardown.patch'
+          # Apply patch to handle ImageNotFound exceptions for all Docker tests
+          su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && patch -p1 tests/integration/local/invoke/test_integrations_cli.py < ${{ github.workspace }}/finch-daemon-pr/scripts/samcli-vm/invoke-teardown.patch'
         shell: bash
 
       - name: Run invoke tests

scripts/samcli-direct/run-invoke-tests.sh

@@ -19,4 +19,4 @@ test_building_new_rapid_image_removes_old_rapid_images
 EOF
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh invoke_output.txt expected_invoke_failures.txt "Invoke tests"
+$GITHUB_WORKSPACE/scripts/validate-test-results.sh invoke_output.txt expected_invoke_failures.txt "Invoke tests"

scripts/samcli-direct/run-package-tests.sh

@@ -16,4 +16,4 @@ test_package_with_loadable_image_archive_0_template_image_load_yaml
 EOF
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh package_output.txt expected_package_failures.txt "Package tests"
+$GITHUB_WORKSPACE/scripts/validate-test-results.sh package_output.txt expected_package_failures.txt "Package tests"

scripts/samcli-direct/run-start-api-tests.sh

@@ -13,4 +13,4 @@ test_can_invoke_lambda_layer_successfully
 EOF
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh start_api_output.txt expected_start_api_failures.txt "Start-API tests"
+$GITHUB_WORKSPACE/scripts/validate-test-results.sh start_api_output.txt expected_start_api_failures.txt "Start-API tests"

scripts/samcli-direct/run-start-lambda-tests.sh

@@ -9,4 +9,4 @@ python -m pytest tests/integration/local/start_lambda -k 'not Terraform' -v --tb
 touch expected_start_lambda_failures.txt
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh start_lambda_output.txt expected_start_lambda_failures.txt "Start-Lambda tests"
+$GITHUB_WORKSPACE/scripts/validate-test-results.sh start_lambda_output.txt expected_start_lambda_failures.txt "Start-Lambda tests"

scripts/samcli-vm/invoke-teardown.patch

@@ -1,6 +1,6 @@
 --- a/tests/integration/local/invoke/test_integrations_cli.py
 +++ b/tests/integration/local/invoke/test_integrations_cli.py
-@@ -754,6 +754,11 @@
+@@ -755,7 +755,12 @@
          docker_client = docker.from_env()
          samcli_images = docker_client.images.list(name="samcli/lambda")
          for image in samcli_images:
@@ -13,4 +13,31 @@
 +                print(f"An error occurred while trying to remove image {image.id}: {str(e)}")
 
          shutil.rmtree(str(self.layer_cache), ignore_errors=True)
-+
+ 
+@@ -772,7 +777,12 @@
+         docker_client = docker.from_env()
+         samcli_images = docker_client.images.list(name="samcli/lambda")
+         for image in samcli_images:
+-            docker_client.images.remove(image.id)
++            try:
++                docker_client.images.remove(image.id, force=True)
++            except docker.errors.ImageNotFound:
++                print(f"Image {image.id} was not found. It may have been removed already.")
++            except Exception as e:
++                print(f"An error occurred while trying to remove image {image.id}: {str(e)}")
+         integ_layer_cache_dir = Path().home().joinpath("integ_layer_cache")
+         if integ_layer_cache_dir.exists():
+             shutil.rmtree(str(integ_layer_cache_dir))
+@@ -1012,7 +1022,12 @@
+         docker_client = docker.from_env()
+         samcli_images = docker_client.images.list(name="samcli/lambda")
+         for image in samcli_images:
+-            docker_client.images.remove(image.id)
++            try:
++                docker_client.images.remove(image.id, force=True)
++            except docker.errors.ImageNotFound:
++                print(f"Image {image.id} was not found. It may have been removed already.")
++            except Exception as e:
++                print(f"An error occurred while trying to remove image {image.id}: {str(e)}")
+ 
+     def test_layer_does_not_exist(self):

scripts/samcli-vm/run-invoke-tests.sh

@@ -36,4 +36,4 @@ test_successful_invoke
 EOF
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh /tmp/invoke_test_output.txt expected_invoke_failures.txt "Invoke tests"
+$GITHUB_WORKSPACE/finch-daemon-pr/scripts/validate-test-results.sh /tmp/invoke_test_output.txt expected_invoke_failures.txt "Invoke tests"

scripts/samcli-vm/run-package-tests.sh

@@ -27,4 +27,4 @@ test_package_with_loadable_image_archive_0_template_image_load_yaml
 EOF
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh /tmp/package_test_output.txt expected_package_failures.txt "Package tests"
+$GITHUB_WORKSPACE/finch-daemon-pr/scripts/validate-test-results.sh /tmp/package_test_output.txt expected_package_failures.txt "Package tests"

scripts/samcli-vm/run-start-api-tests.sh

@@ -35,9 +35,12 @@ echo "=== START-API TESTS - Finished at $(date) ==="
 
 # test_can_invoke_lambda_layer_successfully: Uses random port, fails occasionally.
 #         Only 1 test of 386 total, acceptable failure rate.
+# test_changed_code_got_observed_and_loaded: Fails due to race conditions with containers.
+# Fails infrequently, but is of note. 
 cat > expected_start_api_failures.txt << 'EOF'
 test_can_invoke_lambda_layer_successfully
+test_changed_code_got_observed_and_loaded
 EOF
 
 # Validate test results
-$(dirname "$0")/../validate-test-results.sh /tmp/start_api_test_output.txt expected_start_api_failures.txt "Start-API tests"
+$GITHUB_WORKSPACE/finch-daemon-pr/scripts/validate-test-results.sh /tmp/start_api_test_output.txt expected_start_api_failures.txt "Start-API tests"