[sdlf-utils] update parameters in examples

Author: cnfait

sdlf-utils/workshop-examples/10-demo/sdlf-workshop/dataset-legislators.yaml

@@ -11,7 +11,6 @@ Resources:
         Type: awslabs::sdlf::dataset::MODULE
         Properties:
             pPipelineReference: !Ref pPipelineReference
-            pDatasetName: legislators
             pS3Prefix: legislators
             pDeploymentInstance: dev
             pStorageDeploymentInstance: dev

sdlf-utils/workshop-examples/10-demo/sdlf-workshop/pipeline-main.yaml

@@ -10,10 +10,7 @@ Resources:
     rMainA:
         Type: awslabs::sdlf::stagelambda::MODULE
         Properties:
-            pStageName: A
-            pPipeline: main
-            pDataset: legislators
-            pDeploymentInstance: maina
+            pDeploymentInstance: mainA
             pStorageDeploymentInstance: dev
             pDatasetDeploymentInstance: dev
             pTriggerType: event
@@ -35,13 +32,10 @@ Resources:
     rMainB:
         Type: awslabs::sdlf::stageglue::MODULE
         Properties:
-            pStageName: B
-            pPipeline: main
-            pDataset: legislators
-            pDeploymentInstance: mainb
+            pDeploymentInstance: mainB
             pStorageDeploymentInstance: dev
             pDatasetDeploymentInstance: dev
-            pGlueJobName: sdlf-legislators-glue-job
+            pGlueJobName: sdlf-mainB-glue-job
             pGlueNumberOfWorkers: 10
             pGlueWorkerType: G.1X
             pTriggerType: schedule
@@ -54,7 +48,7 @@ Resources:
                         "name": ["{{resolve:ssm:/sdlf/storage/rStageBucket/dev}}"]
                     },
                     "object": {
-                        "key": [{ "prefix": "legislators/main/A/" }]
+                        "key": [{ "prefix": "legislators/mainA/" }]
                     }
                 }
                 }
@@ -65,6 +59,6 @@ Resources:
                 "--job-bookmark-option": "job-bookmark-enable",
                 "--enable-metrics": "",
                 "--enable-auto-scaling": "true",
-                "--SOURCE_LOCATION": !Sub "s3://{{resolve:ssm:/sdlf/storage/rStageBucket/dev}}/legislators/main/A",
-                "--OUTPUT_LOCATION": !Sub "s3://{{resolve:ssm:/sdlf/storage/rAnalyticsBucket/dev}}/legislators/main/B"
+                "--SOURCE_LOCATION": !Sub "s3://{{resolve:ssm:/sdlf/storage/rStageBucket/dev}}/legislators/mainA",
+                "--OUTPUT_LOCATION": !Sub "s3://{{resolve:ssm:/sdlf/storage/rAnalyticsBucket/dev}}/legislators/mainB"
               }

sdlf-utils/workshop-examples/legislators/scripts/legislators-glue-job.yaml

@@ -2,10 +2,10 @@ AWSTemplateFormatVersion: 2010-09-09
 Description: Glue Job Sample
 
 Parameters:
-  pDatasetName:
-    Description: The dataset name
+  pPipelineDeploymentInstance:
     Type: String
-    Default: legislators
+    Description: specific pipeline stage deployment instance this job is for
+    Default: mainB
   pArtifactsBucket:
     Description: S3 bucket used to store artifacts (from CICD or generated by data pipelines)
     Type: AWS::SSM::Parameter::Value<String>
@@ -37,7 +37,7 @@ Resources:
         - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess
         - !Sub arn:${AWS::Partition}:iam::aws:policy/CloudWatchLogsFullAccess
       Policies:
-        - PolicyName: !Sub sdlf-${pDatasetName}-glue-job
+        - PolicyName: !Sub sdlf-${pPipelineDeploymentInstance}-glue-job
           PolicyDocument:
             Version: 2012-10-17
             Statement:
@@ -60,7 +60,7 @@ Resources:
       Command:
         Name: glueetl
         PythonVersion: "3"
-        ScriptLocation: !Sub s3://${pArtifactsBucket}/artifacts/${pDatasetName}-glue-job.py
+        ScriptLocation: !Sub s3://${pArtifactsBucket}/artifacts/${pPipelineDeploymentInstance}-glue-job.py
       DefaultArguments: !If
         - RunInVpc
         -
@@ -75,6 +75,6 @@ Resources:
       MaxRetries: 0
       MaxCapacity: 2.0
       GlueVersion: "4.0"
-      Name: !Sub sdlf-${pDatasetName}-glue-job
+      Name: !Sub sdlf-${pPipelineDeploymentInstance}-glue-job
       SecurityConfiguration: "{{resolve:ssm:/sdlf/dataset/rGlueSecurityConfiguration/dev:1}}"
       Role: !Ref rGlueRole