Merge branch 'feature/fix-sdlc-pipeline-polling' into 'develop'

Fix SDLC Pipeline Polling and Security Compliance Issues

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!323

Author: rstrahan

iam-roles/cloudformation-management/IDP-Cloudformation-Service-Role.yaml

@@ -13,7 +13,7 @@ Resources:
   CloudFormationServiceRole:
     Type: AWS::IAM::Role
     Properties:
-      RoleName: IDPAcceleratorCloudFormationServiceRole
+      RoleName: !Sub '${AWS::StackName}-CFServiceRole'
       AssumeRolePolicyDocument:
         Version: '2012-10-17'
         Statement:
@@ -109,7 +109,7 @@ Resources:
   PassRolePolicy:
     Type: AWS::IAM::ManagedPolicy
     Properties:
-      ManagedPolicyName: IDP-PassRolePolicy
+      ManagedPolicyName: !Sub '${AWS::StackName}-PassRolePolicy'
       Description: Policy to allow passing the IDP CloudFormation service role
       PolicyDocument:
         Version: '2012-10-17'

patterns/pattern-1/template.yaml

@@ -714,6 +714,7 @@ Resources:
           - id: W92
             reason: "Function does not require concurrent execution limits as it is designed to scale based on demand"
     Properties:
+      PermissionsBoundary: !If [HasPermissionsBoundary, !Ref PermissionsBoundaryArn, !Ref AWS::NoValue]
       CodeUri: src/bda_discovery_function/
       Handler: index.handler
       Runtime: python3.12

scripts/sdlc/cfn/codepipeline-s3.yml

@@ -155,7 +155,10 @@ Resources:
                 - export IDP_CFN_PREFIX=$(make cfn-prefix) || { echo "CFN prefix generation failed"; exit 1; }
                 - make install -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
                 - make smoketest -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
-                - make uninstall -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
+            post_build:
+              commands:
+                - echo "Running cleanup regardless of build result..."
+                - make uninstall -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX || echo "Cleanup failed but continuing"
 
   DeploymentPipeline:
     Type: 'AWS::CodePipeline::Pipeline'
@@ -177,6 +180,7 @@ Resources:
               Configuration:
                 S3Bucket: !Ref BucketName
                 S3ObjectKey: !Ref FileKey
+                PollForSourceChanges: true
               OutputArtifacts:
                 - Name: SourceOutput
               RunOrder: 1

scripts/sdlc/idp-cli/Makefile

@@ -1,4 +1,4 @@
-IDP_ACCOUNT_ID?=715841340161
+IDP_ACCOUNT_ID?=020432867916
 AWS_REGION?=us-east-1
 IDP_ADMIN_EMAIL?[email protected]
 IDP_CWD?=../../../

scripts/sdlc/idp-cli/poetry.lock

@@ -14,7 +14,8 @@ dependencies = [
     "loguru (>=0.7.3,<0.8.0)",
     "python-slugify (>=8.0.4,<9.0.0)",
     "pytest (>=8.3.4,<9.0.0)",
-    "urllib3 (>=2.5.0,<3.0.0)"
+    "urllib3 (>=2.5.0,<3.0.0)",
+    "setuptools"
 ]
 
 [tool.poetry]

scripts/sdlc/idp-cli/pyproject.toml

@@ -74,7 +74,7 @@ def uninstall(
 def smoketest(
     stack_name: str = typer.Option("idp-Stack", "--stack-name", help="Name of the deployed stack to test"),
     file_path: str = typer.Option("../../../samples/lending_package.pdf", "--file-path", help="Path to the test file"),
-    verify_string: str = typer.Option("BIGTOWN, MA, 02801", "--verify-string", help="String to verify in the processed output")
+    verify_string: str = typer.Option("ANYTOWN, USA 12345", "--verify-string", help="String to verify in the processed output")
 ):
     """
     Run a smoke test on the deployed IDP Accelerator

scripts/sdlc/idp-cli/src/idp_cli/cli/main.py

@@ -136,106 +136,6 @@ def publish(self):
             return False
 
 
-    def cleanup_failed_stack(self, stack_name):
-        """
-        Clean up failed stack if it exists in ROLLBACK_COMPLETE state.
-        
-        Args:
-            stack_name: Name of the stack to clean up
-            
-        Returns:
-            bool: True if cleanup successful or not needed, False if cleanup failed
-        """
-        try:
-            # Check stack status
-            cmd = [
-                'aws', 'cloudformation', 'describe-stacks',
-                '--region', self.region,
-                '--stack-name', stack_name,
-                '--query', 'Stacks[0].StackStatus',
-                '--output', 'text'
-            ]
-            
-            process = subprocess.run(
-                cmd,
-                check=True,
-                text=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE
-            )
-            
-            stack_status = process.stdout.strip()
-            
-            if stack_status in ['ROLLBACK_COMPLETE', 'CREATE_FAILED', 'DELETE_FAILED']:
-                logger.info(f"Cleaning up failed stack {stack_name} (status: {stack_status})")
-                
-                delete_cmd = [
-                    'aws', 'cloudformation', 'delete-stack',
-                    '--region', self.region,
-                    '--stack-name', stack_name
-                ]
-                
-                subprocess.run(delete_cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                
-                # Wait for deletion to complete
-                wait_cmd = [
-                    'aws', 'cloudformation', 'wait', 'stack-delete-complete',
-                    '--region', self.region,
-                    '--stack-name', stack_name
-                ]
-                
-                subprocess.run(wait_cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                logger.info(f"Successfully cleaned up failed stack {stack_name}")
-                
-            return True
-            
-        except subprocess.CalledProcessError:
-            # Stack doesn't exist - no cleanup needed
-            return True
-        except Exception as e:
-            logger.error(f"Failed to cleanup stack {stack_name}: {e}")
-            return False
-
-    def get_service_role_arn(self):
-        """
-        Check if CloudFormation service role stack exists and return its ARN.
-        
-        Returns:
-            str: The ARN of the existing service role, or None if not found
-        """
-        service_role_stack_name = f"{self.cfn_prefix}-cloudformation-service-role"
-        
-        try:
-            describe_cmd = [
-                'aws', 'cloudformation', 'describe-stacks',
-                '--region', self.region,
-                '--stack-name', service_role_stack_name,
-                '--query', 'Stacks[0].Outputs[?OutputKey==`ServiceRoleArn`].OutputValue',
-                '--output', 'text'
-            ]
-
-            process = subprocess.run(
-                describe_cmd,
-                check=True,
-                text=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE
-            )
-
-            service_role_arn = process.stdout.strip()
-            if service_role_arn and service_role_arn != "None":
-                logger.info(f"Found existing service role: {service_role_arn}")
-                return service_role_arn
-            else:
-                return None
-
-        except subprocess.CalledProcessError:
-            logger.debug(f"Service role stack {service_role_stack_name} not found")
-            return None
-        except Exception as e:
-            logger.error(f"Error checking for existing service role: {e}")
-            return None
-
     def deploy_service_role(self):
         """
         Deploy the CloudFormation service role stack.
@@ -278,8 +178,8 @@ def deploy_service_role(self):
             if process.stderr:
                 logger.debug(f"Service role deploy stderr: {process.stderr}")
 
-            # Get the service role ARN from stack outputs
-            service_role_arn = self.get_service_role_arn()
+            # Get the service role ARN from the deployed stack
+            service_role_arn = self._get_service_role_arn_from_stack(service_role_stack_name)
             if service_role_arn:
                 logger.info(f"Successfully deployed service role: {service_role_arn}")
                 return service_role_arn
@@ -296,62 +196,78 @@ def deploy_service_role(self):
                 logger.debug(f"Command stdout: {e.stdout}")
             if e.stderr:
                 logger.debug(f"Command stderr: {e.stderr}")
-            
-            # Cleanup failed service role deployment
-            logger.info("Cleaning up failed service role deployment...")
-            self.cleanup_failed_stack(service_role_stack_name)
             return None
         except Exception as e:
             logger.error(f"Unexpected error during service role deployment: {e}")
             return None
 
+    def _get_service_role_arn_from_stack(self, stack_name):
+        """
+        Get service role ARN from a specific stack.
+        
+        Returns:
+            str: The ARN of the service role, or None if not found
+        """
+        try:
+            describe_cmd = [
+                'aws', 'cloudformation', 'describe-stacks',
+                '--region', self.region,
+                '--stack-name', stack_name,
+                '--query', 'Stacks[0].Outputs[?OutputKey==`ServiceRoleArn`].OutputValue',
+                '--output', 'text'
+            ]
+            
+            process = subprocess.run(
+                describe_cmd,
+                check=True,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+            
+            service_role_arn = process.stdout.strip()
+            if service_role_arn and service_role_arn != "None":
+                return service_role_arn
+            else:
+                return None
+
+        except subprocess.CalledProcessError:
+            return None
+        except Exception as e:
+            logger.error(f"Error getting service role ARN from stack {stack_name}: {e}")
+            return None
+
     def create_permission_boundary_policy(self):
-        """Create an 'allow everything' permission boundary policy if it doesn't exist"""
+        """Create an 'allow everything' permission boundary policy"""
 
-        policy_name = "IDPPermissionBoundary"
+        policy_name = f"{self.cfn_prefix}-IDPPermissionBoundary"
         iam = boto3.client('iam')
 
         try:
-            # First, check if the policy already exists
-            account_id = boto3.client('sts').get_caller_identity()['Account']
-            policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}"
+            policy_document = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": "*",
+                        "Resource": "*"
+                    }
+                ]
+            }
 
-            # Try to get the existing policy
-            iam.get_policy(PolicyArn=policy_arn)
-            logger.info(f"Permission boundary policy already exists: {policy_arn}")
+            response = iam.create_policy(
+                PolicyName=policy_name,
+                PolicyDocument=json.dumps(policy_document),
+                Description="Permission boundary for IDP deployment - allows all actions"
+            )
+            
+            policy_arn = response['Policy']['Arn']
+            logger.info(f"Created permission boundary policy: {policy_arn}")
             return policy_arn
 
-        except ClientError as e:
-            if e.response['Error']['Code'] == 'NoSuchEntity':
-                # Policy doesn't exist, create it
-                policy_document = {
-                    "Version": "2012-10-17",
-                    "Statement": [
-                        {
-                            "Effect": "Allow",
-                            "Action": "*",
-                            "Resource": "*"
-                        }
-                    ]
-                }
-                
-                try:
-                    response = iam.create_policy(
-                        PolicyName=policy_name,
-                        PolicyDocument=json.dumps(policy_document),
-                        Description="Permission boundary for IDP deployment - allows all actions"
-                    )
-                    
-                    policy_arn = response['Policy']['Arn']
-                    logger.info(f"Created permission boundary policy: {policy_arn}")
-                    return policy_arn
-                    
-                except ClientError as create_error:
-                    logger.error(f"Error creating permission boundary policy: {create_error}")
-                    return None
-            else:
-                logger.error(f"Error checking for existing permission boundary policy: {e}")
-                return None
+        except ClientError as create_error:
+            logger.error(f"Error creating permission boundary policy: {create_error}")
+            return None
 
     def validate_permission_boundary(self, stack_name, boundary_arn):
         """Validate that all IAM roles in the stack and nested stacks have the permission boundary"""
@@ -511,6 +427,7 @@ def install(self, admin_email: str, idp_pattern: str):
             logger.info("Step 4: Validating permission boundary on all IAM roles...")
             if not self.validate_permission_boundary(self.stack_name, permission_boundary_arn):
                 logger.error("Permission boundary validation failed!")
+                logger.error("Deployment failed due to security policy violations.")
                 return False
 
             logger.info("Deployment and validation completed successfully!")
@@ -525,10 +442,6 @@ def install(self, admin_email: str, idp_pattern: str):
                 logger.debug(f"Command stdout: {e.stdout}")
             if e.stderr:
                 logger.debug(f"Command stderr: {e.stderr}")
-            
-            # Cleanup failed deployment for next attempt
-            logger.info("Cleaning up failed deployment for next attempt...")
-            self.cleanup_failed_stack(self.stack_name)
             return False
         except Exception as e:
             logger.error(f"Unexpected error during stack deployment: {e}")

scripts/sdlc/idp-cli/src/idp_cli/service/install_service.py

@@ -102,7 +102,7 @@ def verify_result(self, folder_key):
         _, output_bucket_name = self.get_bucket_names()
 
         # Define the path to the expected result.json file
-        object_path = f"{folder_key}/pages/1/result.json"
+        object_path = f"{folder_key}/pages/0/result.json"
         logger.debug(f"Looking for result file at: s3://{output_bucket_name}/{object_path}")
 
         try:
@@ -122,7 +122,7 @@ def verify_result(self, folder_key):
             # Check if the text content contains the expected verification string
             if self.verify_string not in result_json["pages"][0]["representation"]["markdown"]:
                 logger.error(f"Text content does not contain expected string: '{self.verify_string}'")
-                logger.debug(f"Actual text starts with: '{result_json['text'][:100]}...'")
+                logger.debug(f"Actual text starts with: '{result_json['pages'][0]['representation']['markdown'][:100]}...'")
                 raise ValueError("Text content does not contain expected verification string")
 
             logger.debug("Smoke test verification passed!")