fix: Handle missing blueprintVersion in BDA blueprint service and bump to v0.4.0-rc6

Author: Bob Strahan

VERSION

@@ -1 +1 @@
-0.4.0-rc5
+0.4.0-rc6

lib/idp_common_pkg/idp_common/bda/bda_blueprint_service.py

@@ -67,7 +67,10 @@ def _retrieve_all_blueprints(self, project_arn: str):
                             blueprint_arn=blueprint_arn, stage="LIVE"
                         )
                         _blueprint = response.get("blueprint")
-                        _blueprint["blueprintVersion"] = blueprint["blueprintVersion"]
+                        # Add blueprintVersion with default if missing
+                        _blueprint["blueprintVersion"] = blueprint.get(
+                            "blueprintVersion", "1"
+                        )
                         all_blueprints.append(_blueprint)
                     logger.info(
                         f"{len(all_blueprints)} blueprints retrieved for {project_arn}"
@@ -88,20 +91,24 @@ def _retrieve_all_blueprints(self, project_arn: str):
 
     def _transform_json_schema_to_bedrock_blueprint(self, json_schema: dict) -> dict:
         """
-        Transform a standard JSON Schema to Bedrock Document Analysis blueprint format.
+        Transform JSON Schema (draft 2020-12) to BDA blueprint format (draft-07).
 
-        Bedrock expects:
-        - "class" and "description" at top level (not $id)
-        - "instruction" and "inferenceType" for each field property
-        - Sections in definitions with references in properties
+        BDA requirements based on working schemas:
+        - Uses "definitions" (not "$defs") - JSON Schema draft-07
+        - References use "#/definitions/" (not "#/$defs/")
+        - Only LEAF properties get "inferenceType" and "instruction"
+        - Object/array types do NOT get these fields
 
         Args:
-            json_schema: Standard JSON Schema from migration
+            json_schema: JSON Schema from configuration
 
         Returns:
-            Blueprint schema in Bedrock format
+            Blueprint schema in BDA-compatible draft-07 format
         """
-        # Start with the basic structure Bedrock expects
+        # Extract $defs and convert to definitions
+        defs = json_schema.get(DEFS_FIELD, {})
+
+        # Start with BDA-expected structure
         blueprint = {
             "$schema": "http://json-schema.org/draft-07/schema#",
             "class": json_schema.get(
@@ -111,17 +118,86 @@ def _transform_json_schema_to_bedrock_blueprint(self, json_schema: dict) -> dict
                 SCHEMA_DESCRIPTION, "Document schema for data extraction"
             ),
             "type": TYPE_OBJECT,
-            "definitions": deepcopy(json_schema.get(DEFS_FIELD, {})),
-            "properties": {},
         }
 
-        # Transform each property to add Bedrock-specific fields
+        # Convert definitions and add BDA fields to leaf properties only
+        if defs:
+            blueprint["definitions"] = {}
+            for def_name, def_value in defs.items():
+                blueprint["definitions"][def_name] = self._add_bda_fields_to_schema(
+                    def_value
+                )
+
+        # Transform properties and update $ref paths
+        blueprint["properties"] = {}
         for prop_name, prop_value in json_schema.get(SCHEMA_PROPERTIES, {}).items():
-            transformed_prop = self._add_bedrock_fields_to_property(prop_value)
-            blueprint["properties"][prop_name] = transformed_prop
+            transformed = self._add_bda_fields_to_schema(prop_value)
+            # Update $ref paths from #/$defs/ to #/definitions/
+            if REF_FIELD in transformed:
+                transformed[REF_FIELD] = transformed[REF_FIELD].replace(
+                    "/$defs/", "/definitions/"
+                )
+            blueprint["properties"][prop_name] = transformed
 
         return blueprint
 
+    def _add_bda_fields_to_schema(self, schema: dict) -> dict:
+        """
+        Add BDA fields (inferenceType, instruction) ONLY to leaf properties.
+
+        Critical BDA requirements (based on working schemas):
+        - Pure $ref properties: ONLY the $ref field
+        - Object/array types: ONLY type and properties (NO description, inferenceType, instruction)
+        - Leaf types: type, inferenceType, instruction (NO description)
+
+        Args:
+            schema: Property or definition schema
+
+        Returns:
+            Schema with BDA fields, description removed
+        """
+        if not isinstance(schema, dict):
+            return schema
+
+        # If this has a $ref, return ONLY the $ref (strip all other fields)
+        if REF_FIELD in schema:
+            # Pure $ref should have nothing else - this is critical for BDA
+            return {REF_FIELD: schema[REF_FIELD].replace("/$defs/", "/definitions/")}
+
+        # Make deep copy to avoid mutation
+        result = deepcopy(schema)
+
+        # Remove description field - BDA doesn't use it (only instruction)
+        result.pop(SCHEMA_DESCRIPTION, None)
+
+        prop_type = result.get(SCHEMA_TYPE, "string")
+
+        # Add BDA fields ONLY for leaf/primitive types
+        if prop_type not in [TYPE_OBJECT, TYPE_ARRAY]:
+            # This is a leaf property - add BDA fields
+            if "inferenceType" not in result:
+                result["inferenceType"] = "inferred"
+
+            if "instruction" not in result:
+                # Use description if available before we removed it
+                if SCHEMA_DESCRIPTION in schema:
+                    result["instruction"] = schema[SCHEMA_DESCRIPTION]
+                else:
+                    result["instruction"] = "Extract this field from the document"
+
+        # Recursively process nested structures
+        if prop_type == TYPE_OBJECT and SCHEMA_PROPERTIES in result:
+            result[SCHEMA_PROPERTIES] = {
+                name: self._add_bda_fields_to_schema(value)
+                for name, value in result[SCHEMA_PROPERTIES].items()
+            }
+
+        # Handle array items (but don't add BDA fields to the array itself)
+        if prop_type == TYPE_ARRAY and SCHEMA_ITEMS in result:
+            result[SCHEMA_ITEMS] = self._add_bda_fields_to_schema(result[SCHEMA_ITEMS])
+
+        return result
+
     def _add_bedrock_fields_to_property(self, prop: dict) -> dict:
         """
         Add Bedrock-specific fields (instruction, inferenceType) to a property.

lib/idp_common_pkg/tests/unit/bda/test_bda_blueprint_service.py

@@ -478,8 +478,12 @@ def test_transform_does_not_mutate_input_schema(self, service):
         )
         assert blueprint["properties"]["invoiceNumber"]["inferenceType"] == "inferred"
 
-    def test_transform_preserves_ref_nodes(self, service):
-        """Ensure that $ref-only nodes are not augmented with Bedrock fields."""
+    def test_transform_converts_defs_to_definitions(self, service):
+        """Ensure that $defs is converted to definitions for BDA draft-07 compatibility.
+
+        BDA uses JSON Schema draft-07 which uses "definitions", not "$defs".
+        References should be preserved but updated to #/definitions/ path.
+        """
         schema = build_json_schema(
             doc_id="Document",
             description="Document schema",
@@ -491,17 +495,47 @@ def test_transform_preserves_ref_nodes(self, service):
             defs={
                 "Address": {
                     "type": "object",
+                    "description": "Address information",
                     "properties": {
                         "street": {
                             "type": "string",
                             "description": "Street line",
-                        }
+                        },
+                        "city": {
+                            "type": "string",
+                            "description": "City name",
+                        },
                     },
                 }
             },
         )
 
         blueprint = service._transform_json_schema_to_bedrock_blueprint(schema)
 
-        # address property should remain a pure $ref
-        assert blueprint["properties"]["address"] == {"$ref": "#/$defs/Address"}
+        # Verify $defs was converted to definitions
+        assert "definitions" in blueprint
+        assert "$defs" not in blueprint
+        assert "Address" in blueprint["definitions"]
+
+        # Verify $ref path was updated
+        address_prop = blueprint["properties"]["address"]
+        assert address_prop["$ref"] == "#/definitions/Address"
+
+        # Verify definition has proper structure (object should NOT have inferenceType/instruction)
+        address_def = blueprint["definitions"]["Address"]
+        assert address_def["type"] == "object"
+        assert "inferenceType" not in address_def  # Objects should not have this
+        assert "instruction" not in address_def  # Objects should not have this
+        assert "properties" in address_def
+
+        # Verify leaf properties DO have BDA fields
+        street_prop = address_def["properties"]["street"]
+        assert street_prop["type"] == "string"
+        assert "inferenceType" in street_prop
+        assert "instruction" in street_prop
+        assert street_prop["instruction"] == "Street line"
+
+        city_prop = address_def["properties"]["city"]
+        assert city_prop["type"] == "string"
+        assert "inferenceType" in city_prop
+        assert "instruction" in city_prop

patterns/pattern-1/TEST-BDA-DISCOVERY-README.md

@@ -0,0 +1,111 @@
+# BDA Discovery Lambda Local Testing
+
+This guide explains how to test the BDA Discovery Lambda function locally with the real AWS BDA service.
+
+## Prerequisites
+
+1. **AWS Credentials**: Ensure your AWS credentials are configured
+   ```bash
+   aws configure
+   # or
+   export AWS_PROFILE=your-profile
+   ```
+
+2. **Python Environment**: Python 3.12+ with dependencies installed
+   ```bash
+   cd lib/idp_common_pkg
+   pip install -e ".[extraction]"
+   ```
+
+3. **Required Environment Variables**: The test script sets defaults but you can override:
+   ```bash
+   export BDA_PROJECT_ARN="your-project-arn"
+   export STACK_NAME="your-stack-name"
+   export CONFIGURATION_TABLE_NAME="your-config-table"
+   export DISCOVERY_TRACKING_TABLE="your-tracking-table"
+   ```
+
+## Running the Test
+
+From the project root:
+
+```bash
+cd patterns/pattern-1
+python test-bda-discovery.py
+```
+
+Or make it executable and run directly:
+
+```bash
+./patterns/pattern-1/test-bda-discovery.py
+```
+
+## What the Test Does
+
+1. Loads the Lambda handler from `src/bda_discovery_function/index.py`
+2. Uses a real SQS event from CloudWatch logs
+3. Calls the actual AWS BDA service to create/update blueprints
+4. Reports success or failure with detailed output
+
+## Expected Output
+
+**Success:**
+```
+================================================================================
+Testing BDA Discovery Lambda Function
+================================================================================
+
+Environment:
+  BDA_PROJECT_ARN: arn:aws:bedrock:us-west-2:...:data-automation-project/...
+  STACK_NAME: IDP-BDA-3
+  CONFIGURATION_TABLE_NAME: IDP-BDA-3-ConfigurationTable-...
+  LOG_LEVEL: INFO
+
+================================================================================
+Invoking handler...
+================================================================================
+
+[INFO logs from the function...]
+
+================================================================================
+Result:
+================================================================================
+{
+  "batchItemFailures": []
+}
+
+================================================================================
+✅ Test completed successfully!
+================================================================================
+```
+
+**Failure:**
+The script will show detailed error messages and stack traces to help diagnose issues.
+
+## Troubleshooting
+
+### Missing deepdiff Module
+```bash
+pip install deepdiff>=6.0.0
+```
+
+### AWS Permissions
+Ensure your AWS credentials have permissions for:
+- `bedrock:GetDataAutomationProject`
+- `bedrock:ListBlueprints`
+- `bedrock:GetBlueprint`
+- `bedrock:CreateBlueprint`
+- `bedrock:UpdateBlueprint`
+- `bedrock:CreateBlueprintVersion`
+- `dynamodb:GetItem` (for configuration table)
+
+### Invalid Blueprint Schema Errors
+If you see validation errors, check the CloudWatch logs output in the test for the actual schema being sent to BDA. Compare with working blueprints in your project.
+
+## Modifying the Test
+
+You can modify `test-bda-discovery.py` to:
+- Change environment variables
+- Use different test events
+- Add more detailed logging
+- Test specific configuration scenarios