[sdlf-utils] fix legislators example script (#541)

Author: dlpzx

sdlf-utils/workshop-examples/legislators/README.md

@@ -0,0 +1,55 @@
+# Dataset Example
+
+This example demonstrates how to hydrate the data lake with a sample dataset for a specific team.
+
+## Usage
+
+```bash
+./deploy.sh -t <team-name> -d <dataset-name> -r <region> [-p <aws-profile>]
+```
+
+### Parameters
+
+- `-t <team-name>` (required): Team name (2-12 characters, lowercase letters and numbers only)
+- `-d <dataset-name>` (required): Dataset name (2-12 characters, lowercase letters and numbers only)
+- `-r <region>` (required): AWS region (e.g., us-east-1, eu-west-1)
+- `-p <aws-profile>` (optional): AWS profile to use
+- `-h`: Show help message
+
+### Examples
+
+```bash
+# Deploy legislators dataset for team "engineering" in us-east-1
+./deploy.sh -t engineering -d legislators -r us-east-1
+
+# Deploy customers dataset for team "analytics" in eu-west-1 with specific AWS profile
+./deploy.sh -t analytics -d customers -r eu-west-1 -p my-profile
+```
+
+## What it creates
+
+1. **Glue Job**: `sdlf-<team-name>-<dataset-name>-glue-job`
+2. **IAM Role**: `sdlf-<team-name>-<dataset-name>-glue-role`
+3. **CloudFormation Stack**: `sdlf-<team-name>-<dataset-name>-glue-job`
+4. **S3 Data**: Uploads sample data to `s3://<raw-bucket>/<team-name>/<dataset-name>/`
+
+## Data Processing
+
+The Glue job processes three JSON files:
+- `persons.json` → `persons/` (Parquet)
+- `memberships.json` → `memberships/` (Parquet)
+- `organizations.json` → `organizations/` (Parquet)
+- Creates a joined `history/` dataset partitioned by organization name
+
+## Prerequisites
+
+- AWS CLI configured
+- SDLF framework deployed in the specified region
+- Appropriate IAM permissions
+
+## Example Resource Names
+
+For team "engineering" and dataset "legislators":
+- Glue Job: `sdlf-engineering-legislators-glue-job`
+- IAM Role: `sdlf-engineering-legislators-glue-role`
+- S3 Path: `s3://<raw-bucket>/engineering/legislators/`

sdlf-utils/workshop-examples/legislators/deploy.sh

@@ -1,48 +1,81 @@
 #!/bin/bash
 pflag=false
+tflag=false
+dflag=false
+rflag=false
 
 DIRNAME=$(dirname "$0")
 
 usage () { echo "
     -h -- Opens up this help message
     -p -- Name of the AWS profile to use
+    -t -- Team name (required)
+    -d -- Dataset name (required)
+    -r -- AWS region (required)
 "; }
-options=':p:h'
+options=':p:t:d:r:h'
 while getopts "$options" option
 do
     case "$option" in
         p  ) pflag=true; PROFILE=$OPTARG;;
+        t  ) tflag=true; TEAM=$OPTARG;;
+        d  ) dflag=true; DATASET=$OPTARG;;
+        r  ) rflag=true; REGION=$OPTARG;;
         h  ) usage; exit;;
         \? ) echo "Unknown option: -$OPTARG" >&2; exit 1;;
         :  ) echo "Missing option argument for -$OPTARG" >&2; exit 1;;
         *  ) echo "Unimplemented option: -$OPTARG" >&2; exit 1;;
     esac
 done
 
+if ! "$tflag"
+then
+    echo "Team name is required. Use -t <team-name>" >&2
+    usage
+    exit 1
+fi
+
+if ! "$dflag"
+then
+    echo "Dataset name is required. Use -d <dataset-name>" >&2
+    usage
+    exit 1
+fi
+
+if ! "$rflag"
+then
+    echo "AWS region is required. Use -r <region>" >&2
+    usage
+    exit 1
+fi
+
 if "$pflag"
 then
     echo "using AWS profile $PROFILE..." >&2
 fi
-REGION=$(aws configure get region ${PROFILE:+--profile "$PROFILE"})
 
-ARTIFACTS_BUCKET=$(aws --region "$REGION" ssm get-parameter --name "/sdlf/storage/rArtifactsBucket/dev" --query "Parameter.Value" --output text ${PROFILE:+--profile "$PROFILE"})
+echo "using team: $TEAM" >&2
+echo "using dataset: $DATASET" >&2
+echo "using region: $REGION" >&2
+
+ARTIFACTS_BUCKET=$(aws --region "$REGION" ssm get-parameter --name "/SDLF2/S3/ArtifactsBucket" --query "Parameter.Value" --output text ${PROFILE:+--profile "$PROFILE"})
 aws s3 cp "$DIRNAME/scripts/legislators-glue-job.py" "s3://$ARTIFACTS_BUCKET/artifacts/" ${PROFILE:+--profile "$PROFILE"}
 
-mkdir "$DIRNAME"/output
+mkdir -p "$DIRNAME"/output
 
-function send_legislators() 
+function send_data() 
 {
   ORIGIN="$DIRNAME/data/"
 
-  RAW_BUCKET=$(aws --region "$REGION" ssm get-parameter --name "/sdlf/storage/rRawBucket/dev" --query "Parameter.Value" --output text ${PROFILE:+--profile "$PROFILE"})
-  KMS_KEY=$(aws --region "$REGION" ssm get-parameter --name "/sdlf/dataset/rKMSDataKey/dev" --query "Parameter.Value" --output text ${PROFILE:+--profile "$PROFILE"})
+  RAW_BUCKET=$(aws --region "$REGION" ssm get-parameter --name "/SDLF2/S3/RawBucket" --query "Parameter.Value" --output text ${PROFILE:+--profile "$PROFILE"})
+  KMS_KEY=$(aws --region "$REGION" ssm get-parameter --name "/SDLF/KMS/$TEAM/DataKeyId" --query "Parameter.Value" --output text ${PROFILE:+--profile "$PROFILE"})
 
   S3_DESTINATION=s3://$RAW_BUCKET/
   COUNT=0
   for FILE in "$ORIGIN"/*.json;
   do
     (( COUNT++ )) || true
-    aws s3 cp "$FILE" "${S3_DESTINATION}legislators/" --sse aws:kms --sse-kms-key-id "$KMS_KEY" ${PROFILE:+--profile "$PROFILE"}
+    aws s3 cp "$FILE" "${S3_DESTINATION}${TEAM}/${DATASET}/" --sse aws:kms --sse-kms-key-id "$KMS_KEY" ${PROFILE:+--profile "$PROFILE"}
     echo "transferred $COUNT files"
   done
 }
@@ -58,14 +91,15 @@ aws cloudformation package --template-file "$DIRNAME"/scripts/legislators-glue-j
   ${PROFILE:+--profile "$PROFILE"} \
   --output-template-file "$DIRNAME"/output/packaged-template.yaml
 
-STACK_NAME="sdlf-legislators-glue-job"
+STACK_NAME="sdlf-${TEAM}-${DATASET}-glue-job"
 aws cloudformation deploy \
     --s3-bucket "$ARTIFACTS_BUCKET" --s3-prefix sdlf-utils \
     --stack-name "$STACK_NAME" \
     --template-file "$DIRNAME"/output/packaged-template.yaml \
-    --tags Framework=sdlf \
+    --parameter-overrides pTeamName="$TEAM" pDatasetName="$DATASET" \
+    --tags Framework=sdlf Team="$TEAM" Dataset="$DATASET" \
     --capabilities "CAPABILITY_NAMED_IAM" "CAPABILITY_AUTO_EXPAND" \
     --region "$REGION" \
     ${PROFILE:+--profile "$PROFILE"} || exit 1
 
-send_legislators
+send_data

sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml

@@ -2,14 +2,24 @@ AWSTemplateFormatVersion: 2010-09-09
 Description: Glue Job Sample
 
 Parameters:
+  pTeamName:
+    Type: String
+    Description: Team name for resource naming
+    AllowedPattern: "[a-z0-9]{2,12}"
+    ConstraintDescription: Must be between 2 and 12 characters, lowercase letters and numbers only
+  pDatasetName:
+    Type: String
+    Description: Dataset name for resource naming
+    AllowedPattern: "[a-z0-9]{2,12}"
+    ConstraintDescription: Must be between 2 and 12 characters, lowercase letters and numbers only
   pPipelineDeploymentInstance:
     Type: String
     Description: specific pipeline stage deployment instance this job is for
     Default: mainB
   pArtifactsBucket:
     Description: S3 bucket used to store artifacts (from CICD or generated by data pipelines)
     Type: AWS::SSM::Parameter::Value<String>
-    Default: /sdlf/storage/rArtifactsBucket/dev
+    Default: /SDLF2/S3/ArtifactsBucket
   pEnableVpc:
     Description: Deploy SDLF resources in a VPC
     Type: AWS::SSM::Parameter::Value<String>
@@ -22,6 +32,7 @@ Resources:
   rGlueRole:
     Type: AWS::IAM::Role
     Properties:
+      RoleName: !Sub sdlf-${pTeamName}-${pDatasetName}-glue-role
       Path: /service-role/
       AssumeRolePolicyDocument:
         Version: 2012-10-17
@@ -37,7 +48,7 @@ Resources:
         - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess
         - !Sub arn:${AWS::Partition}:iam::aws:policy/CloudWatchLogsFullAccess
       Policies:
-        - PolicyName: !Sub sdlf-${pPipelineDeploymentInstance}-glue-job
+        - PolicyName: !Sub sdlf-${pTeamName}-${pDatasetName}-glue-policy
           PolicyDocument:
             Version: 2012-10-17
             Statement:
@@ -50,17 +61,17 @@ Resources:
                   - kms:GenerateDataKey*
                   - kms:ReEncrypt*
                 Resource:
-                  - "{{resolve:ssm:/sdlf/dataset/rKMSInfraKey/dev:1}}"
-                  - "{{resolve:ssm:/sdlf/dataset/rKMSDataKey/dev:1}}"
-                  - "{{resolve:ssm:/sdlf/storage/rKMSKey/dev:1}}"
+                  - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId:1}}"
+                  - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/DataKeyId:1}}"
+                  - !Sub "{{resolve:ssm:/SDLF2/KMS/KeyArn:1}}"
 
   rGlueJob:
     Type: AWS::Glue::Job
     Properties:
       Command:
         Name: glueetl
         PythonVersion: "3"
-        ScriptLocation: !Sub s3://${pArtifactsBucket}/artifacts/${pPipelineDeploymentInstance}-glue-job.py
+        ScriptLocation: !Sub s3://${pArtifactsBucket}/artifacts/legislators-glue-job.py
       DefaultArguments: !If
         - RunInVpc
         -
@@ -75,6 +86,19 @@ Resources:
       MaxRetries: 0
       MaxCapacity: 2.0
       GlueVersion: "4.0"
-      Name: !Sub sdlf-${pPipelineDeploymentInstance}-glue-job
-      SecurityConfiguration: "{{resolve:ssm:/sdlf/dataset/rGlueSecurityConfiguration/dev:1}}"
+      Name: !Sub sdlf-${pTeamName}-${pDatasetName}-glue-job
+      SecurityConfiguration: !Sub "{{resolve:ssm:/SDLF/Glue/${pTeamName}/SecurityConfigurationId:1}}"
       Role: !Ref rGlueRole
+
+Outputs:
+  oGlueJobName:
+    Description: Name of the Glue job
+    Value: !Ref rGlueJob
+    Export:
+      Name: !Sub ${AWS::StackName}-glue-job-name
+  
+  oGlueRoleArn:
+    Description: ARN of the Glue job role
+    Value: !GetAtt rGlueRole.Arn
+    Export:
+      Name: !Sub ${AWS::StackName}-glue-role-arn