Merge branch 'main' into support-cases-guardrails

Author: iakov-gan

case-summarization/deploy/case-summarization.yaml

@@ -269,7 +269,7 @@ Resources:
                   description="Focus on the main elements of the conversation and highlight the proposed solutions. If no specific solution, skip this section."
               )
               actions: str = Field(
-                  description="Finish off with any action items or next steps highlighting ownership of actions or next steps."
+                  description="Finish off with any action items or next steps highlighting ownership of actions or next steps. If none, skip this section."
               )
               references: list[str] = Field(
                   description=(
@@ -360,6 +360,7 @@ Resources:
               model_id = get_model_id()
                             
               llm_program = get_llm_program(communications, model_id, GUARDRAIL_ID, GUARDRAIL_VERSION, GUARDRAIL_TRACE)
+
               try:
                   case_data_content['Summary'] = llm_program().model_dump_json()
               except Exception as exc:
@@ -493,7 +494,7 @@ Resources:
   QueueEventSourceMapping:
     Type: AWS::Lambda::EventSourceMapping
     Properties:
-      BatchSize: !Ref BatchSize  # Process N messages at a time to avoid Bedrock Tootling
+      BatchSize: !Ref BatchSize  # Process N messages at a time to avoid Bedrock Throttling
       Enabled: true
       EventSourceArn: !GetAtt SummarizationQueue.Arn
       FunctionName: !Ref SummarizationLambda

case-summarization/utils/release.sh

@@ -2,28 +2,9 @@
 # shellcheck disable=SC2016,SC2086,SC2162
 # This script can be used for release
 
-export AWS_REGION=us-east-1
-export STACK_SET_NAME=LayerBuckets
 export CENTRAL_BUCKET=aws-managed-cost-intelligence-dashboards
 
-code_path=$(git rev-parse --show-toplevel)/data-collection/deploy
+code_path=$(git rev-parse --show-toplevel)/case-summarization/deploy
 
 echo "sync to central bucket"
-aws s3 sync $code_path/       s3://$CENTRAL_BUCKET/cfn/data-collection/
-
-
-
-aws cloudformation list-stack-instances \
-  --stack-set-name $STACK_SET_NAME \
-  --query 'Summaries[].[StackId,Region]' \
-  --output text |
-  while read stack_id region; do
-    echo "sync to $region"
-    bucket=$(aws cloudformation list-stack-resources --stack-name $stack_id \
-      --query 'StackResourceSummaries[?LogicalResourceId == `LayerBucket`].PhysicalResourceId' \
-      --region $region --output text)
-
-    aws s3 sync $code_path/       s3://$bucket/cfn/data-collection/ --delete
-  done
-
-echo 'Done'
+aws s3 sync $code_path/       s3://$CENTRAL_BUCKET/cfn/case-summarization/

data-collection/deploy/account-collector.yaml

@@ -110,10 +110,10 @@ Resources:
 
             import boto3
 
-            ROLE_NAME = os.environ['ROLE_NAME']
-            RESOURCE_PREFIX = os.environ['RESOURCE_PREFIX']
-            MANAGEMENT_ACCOUNT_IDS = os.environ['MANAGEMENT_ACCOUNT_IDS']
-            BUCKET = os.environ['BUCKET_NAME']
+            ROLE_NAME = os.environ.get('ROLE_NAME')
+            RESOURCE_PREFIX = os.environ.get('RESOURCE_PREFIX')
+            MANAGEMENT_ACCOUNT_IDS = os.environ.get('MANAGEMENT_ACCOUNT_IDS')
+            BUCKET = os.environ.get('BUCKET_NAME')
             PREDEF_ACCOUNT_LIST_KEY = os.environ.get('PREDEF_ACCOUNT_LIST_KEY')
             LINKED_ACCOUNT_LIST_KEY = os.environ.get('LINKED_ACCOUNT_LIST_KEY')
             PAYER_ACCOUNT_LIST_KEY = os.environ.get('PAYER_ACCOUNT_LIST_KEY')
@@ -185,14 +185,14 @@ Resources:
                 defined_accounts, ext = get_defined_list(BUCKET, PREDEF_ACCOUNT_LIST_KEY)
                 try:
                     if defined_accounts:
-                        logger.info(f'Using defined account list found in s3://{BUCKET}/{PREDEF_ACCOUNT_LIST_KEY} instead of payer organization')
+                        logger.info(f'Using defined account list found in s3://{BUCKET}/{PREDEF_ACCOUNT_LIST_KEY}{ext} instead of payer organization')
                         for account_data in defined_accounts:
                             if ext == "json":
                                 account = json.loads(account_data)
                                 yield format_account(account['account_id'], account['account_name'], account['payer_id'])
                             else:
                                 account = account_data.split(',')
-                                yield format_account(account[0], account[1], account[2])
+                                yield format_account(account[0].strip(), account[1].strip(), account[2].strip())
                     else:
                         logger.info('Using payer organization for the account list')
                         excluded_accounts = get_from_bucket(BUCKET, EXCLUDED_ACCOUNT_LIST_KEY)
@@ -210,13 +210,13 @@ Resources:
                                     continue
                                 yield format_account(account.get('Id'), account.get('Name'), org_account['payer_id'])
                 except Exception as exc: #pylint: disable=broad-exception-caught
-                    logger.error(f'{org_account}: {exc}')
+                    logger.error( f'{type(exc).__name__}: When trying to build linked account list. {exc} ')
 
             def get_defined_list(bucket, key):
                 s3 = boto3.client("s3")
                 exts = [".json", ".csv"]
                 for ext in exts:
-                    accts = get_from_bucket(bucket, key, s3)
+                    accts = get_from_bucket(bucket, key+ext, s3)
                     if accts:
                         return accts, ext
                 logger.debug(f'Predefined account list not retrieved or not being used')

data-collection/deploy/deploy-in-linked-account.yaml

@@ -238,6 +238,8 @@ Resources:
               - "elasticache:DescribeCacheClusters"
               - "eks:ListClusters"
               - "eks:DescribeCluster"
+              - "eks:ListNodegroups"
+              - "eks:DescribeNodegroup"
               - "lambda:ListFunctions"
             Resource: "*" ## Policy is used for scanning of a wide range of resources
       Roles:
@@ -341,4 +343,4 @@ Resources:
       cfn_nag:
         rules_to_suppress:
           - id: W12
-            reason: "Policy is used for scanning of a wide range of resources"
+            reason: "Policy is used for scanning of a wide range of resources"

data-collection/deploy/module-inventory.yaml

@@ -1142,15 +1142,38 @@ Resources:
                       )
                       .search("clusters")
                   ):
+                      # Get cluster details
                       cluster = client.describe_cluster(name=cluster_name)
-                      yield {
+                      cluster_data = {
                           "Arn": cluster["cluster"]["arn"],
                           "Name": cluster["cluster"]["name"],
                           "CreatedAt": datetime.strftime(
                               cluster["cluster"]["createdAt"].astimezone(tz=timezone.utc), "%Y-%m-%dT%H:%M:%SZ"
                           ),
                           "Version": cluster["cluster"]["version"],
                       }
+                      # Get node groups for the cluster
+                      try:
+                          node_groups = client.list_nodegroups(clusterName=cluster_name).get("nodegroups", [])
+                          cluster_data["NodeGroups"] = []
+                          for node_group in node_groups:
+                              node_group_details = client.describe_nodegroup(clusterName=cluster_name, nodegroupName=node_group)
+                              cluster_data["NodeGroups"].append({
+                                  "NodeGroupName": node_group_details["nodegroup"].get("nodegroupName"),
+                                  "NodeRole": node_group_details["nodegroup"].get("nodeRole"),
+                                  "InstanceTypes": node_group_details["nodegroup"].get("instanceTypes",[]),
+                                  "ScalingConfig": node_group_details["nodegroup"].get("scalingConfig",{}),
+                                  "Subnets": node_group_details["nodegroup"].get("subnets",[]),
+                                  "Status": node_group_details["nodegroup"].get("status","Unknown"),
+                                  "AMIType": node_group_details["nodegroup"].get("amiType","Unknown"),
+                                  "Version": node_group_details["nodegroup"].get("version","Unknown"),
+                                  "CreatedAt": datetime.strftime(
+                                      node_group_details["nodegroup"]["createdAt"].astimezone(tz=timezone.utc), "%Y-%m-%dT%H:%M:%SZ"
+                                  ),
+                              })
+                      except Exception as ng_exc:  # Catch node group-specific issues
+                          logger.error(f"Error fetching node groups for {cluster_name} in {region}: {ng_exc}")
+                      yield cluster_data
               except Exception as exc: #pylint: disable=W0718
                   logger.error(f"Cannot get info from {account_id}/{region}: {type(exc)}-{exc}")
               return []