Merge branch 'develop' of ssh.gitlab.aws.dev:genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator into develop

Author: Bob Strahan

CHANGELOG.md

@@ -6,7 +6,7 @@ SPDX-License-Identifier: MIT-0
 ## [Unreleased]
 
 ### Added
-- **Document Compression for Large Documents**
+- **Document Compression for Large Documents - all patterns**
   - Added automatic compression support to handle large documents and avoid exceeding Step Functions payload limits (256KB)
   - **Key Features**:
     - Automatic compression (default trigger threshold of 0KB enables compression by default)

config_library/pattern-3/default/config.yaml

@@ -3,9 +3,17 @@
 
 notes: Default settings
 ocr:
+  backend: "textract"  # Default to Textract for backward compatibility
+  model_id: "us.anthropic.claude-3-7-sonnet-20250219-v1:0"
+  system_prompt: "You are an expert OCR system. Extract all text from the provided image accurately, preserving layout where possible."
+  task_prompt: "Extract all text from this document image. Preserve the layout, including paragraphs, tables, and formatting."
   features:
     - name: LAYOUT
     - name: TABLES
+    - name: SIGNATURES
+  image:
+    target_width: '951'
+    target_height: '1268'
 classes:
   - name: letter
     description: A formal written correspondence with sender/recipient addresses, date, salutation, body, and closing signature

patterns/pattern-1/src/bda_invoke_function/index.py

@@ -129,11 +129,11 @@ def handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
     try:
         logger.info(f"Received event: {json.dumps(event)}")
 
-        # Get document from event
-        document = Document.from_dict(event["document"])
+        # Get document from event using new utility method
+        working_bucket = event['working_bucket']
+        document = Document.load_document(event["document"], working_bucket, logger)
         input_bucket = document.input_bucket
         object_key = document.input_key
-        working_bucket = event['working_bucket']
         data_project_arn = event['BDAProjectArn']
         task_token = event['taskToken']
 
@@ -159,4 +159,4 @@ def handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
 
     except Exception as e:
         logger.error(f"Error processing request: {str(e)}", exc_info=True)
-        raise
+        raise

patterns/pattern-1/src/processresults_function/index.py

@@ -1176,9 +1176,15 @@ def handler(event, context):
 
     appsync_service.update_document(document)
 
-    # Prepare response
+    # Prepare response using new serialization method
+    # Use working bucket for document compression
+    working_bucket = os.environ.get('WORKING_BUCKET')
+    if not working_bucket:
+        logger.warning("WORKING_BUCKET environment variable not set, using output_bucket for compression")
+        working_bucket = output_bucket
+    
     response = {
-        "document": document.to_dict(),
+        "document": document.serialize_document(working_bucket, "processresults", logger),
         "hitl_triggered": hitl_triggered,
         "bda_response_count": len(bda_responses)
     }

patterns/pattern-1/src/summarization_function/index.py

@@ -41,8 +41,13 @@ def handler(event, context):
         if not document_dict:
             raise ValueError("No document data provided")
 
-        # Convert dict to Document object
-        document = Document.from_dict(document_dict)
+        # Get working bucket and load document using new utility method
+        working_bucket = os.environ.get('WORKING_BUCKET')
+        if not working_bucket:
+            raise ValueError("WORKING_BUCKET environment variable not set")
+        
+        # Convert dict to Document object using new utility method
+        document = Document.load_document(document_dict, working_bucket, logger)
 
         # Update document status to SUMMARIZING
         document.status = Status.SUMMARIZING
@@ -71,9 +76,9 @@ def handler(event, context):
         else:
             logger.warning("Document summarization completed but no summary report URI was set")
 
-        # Return the processed document
+        # Return the processed document using new serialization method
         return {
-            'document': processed_document.to_dict(),
+            'document': processed_document.serialize_document(working_bucket, "summarization", logger),
         }
 
     except Exception as e:
@@ -90,4 +95,4 @@ def handler(event, context):
         except Exception as status_error:
             logger.error(f"Failed to update document status: {str(status_error)}", exc_info=True)
 
-        raise e
+        raise e

patterns/pattern-1/template.yaml

@@ -594,6 +594,7 @@ Resources:
           ENABLE_HITL: !Ref EnableHITL
           DB_NAME: !Ref BDAMetadataTable
           BDA_PROJECT_ARN: !Ref BDAProjectArn
+          WORKING_BUCKET: !Ref WorkingBucket
       LoggingConfig:
         LogGroup: !Ref ProcessResultsFunctionLogGroup
       Policies:
@@ -678,6 +679,7 @@ Resources:
           GUARDRAIL_ID_AND_VERSION: !If [HasGuardrailConfig, !Sub "${BedrockGuardrailId}:${BedrockGuardrailVersion}", ""]
           LOG_LEVEL: !Ref LogLevel
           APPSYNC_API_URL: !Ref AppSyncApiUrl
+          WORKING_BUCKET: !Ref WorkingBucket
       LoggingConfig:
         LogGroup: !Ref SummarizationFunctionLogGroup
       Policies:
@@ -686,6 +688,8 @@ Resources:
             BucketName: !Ref InputBucket
         - S3CrudPolicy:
             BucketName: !Ref OutputBucket
+        - S3CrudPolicy:
+            BucketName: !Ref WorkingBucket
         - DynamoDBCrudPolicy:
             TableName: !Ref ConfigurationTable
         - Statement:
@@ -966,6 +970,19 @@ Resources:
 
   HITLWaitFunction:
     Type: AWS::Serverless::Function
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W11
+            reason: "Cloudwatch does not support resource-level permissions"
+          - id: W89
+            reason: "This Lambda function does not require VPC access as it only interacts with AWS services via AWS APIs"
+          - id: W92
+            reason: "Function does not require concurrent execution limits as it is designed to scale based on demand"
+    # checkov:skip=CKV_AWS_116: "DLQ not required for this function as StepFunctions will handle retries"
+    # checkov:skip=CKV_AWS_173: "Environment variables do not contain sensitive data - only configuration values like feature flags and non-sensitive settings"
+    # checkov:skip=CKV_AWS_117: "Function does not require VPC access as it only interacts with AWS services via APIs"
+    # checkov:skip=CKV_AWS_115: "Function does not require reserved concurrency as it scales based on demand"
     Properties:
       CodeUri: src/hitl-wait-function/
       Handler: index.lambda_handler

template.yaml

@@ -3549,6 +3549,11 @@ Resources:
   A2IHumanTaskUILambdaRole:
     Type: AWS::IAM::Role
     Condition: IsPattern1HITLEnabled
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W11
+            reason: "Role requires * resource access for SageMaker A2I operations as resource name is not available"
     Properties:
       AssumeRolePolicyDocument:
         Version: '2012-10-17'
@@ -3671,6 +3676,16 @@ Resources:
   GetWorkforceURLFunction:
     Type: AWS::Serverless::Function
     Condition: IsPattern1HITLEnabled
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W89
+            reason: "Function does not require VPC access as it only interacts with AWS services via APIs"
+          - id: W92
+            reason: "Function does not require reserved concurrency as it scales based on demand"
+    # checkov:skip=CKV_AWS_116: "DLQ not required for Cfn Custom Resource function"
+    # checkov:skip=CKV_AWS_117: "Function does not require VPC access as it only interacts with AWS services via APIs"
+    # checkov:skip=CKV_AWS_115: "Function does not require reserved concurrency as it scales based on demand"
     Properties:
       Handler: index.handler
       Runtime: python3.12