[sdlf-stage-glue] glue arguments configurable through cfn parameter

Author: cnfait

sdlf-stage-glue/src/glue.yaml

@@ -130,6 +130,9 @@ Parameters:
     Type: String
     Default: G.1X
     AllowedValues: [G.1X, G.2X, G.4X, G.8X, G.025X]
+  pGlueArguments:
+    Description: Glue job arguments (JSON)
+    Type: String
   # the ideal would be to fetch ssm:/SDLF/VPC/Enabled and not ask the user to set this variable to true manually.
   # however between AWS::SSM::Parameter::Value<String> not working in CloudFormation modules,
   # Fn::ImportValue not being accepted in CloudFormation modules template fragments,
@@ -531,6 +534,7 @@ Resources:
         lTransform: !Ref pGlueJobName
         lWorkerType: !Ref pGlueWorkerType
         lNumberOfWorkers: !Ref pGlueNumberOfWorkers
+        lGlueArguments: !Ref pGlueArguments
         # lArguments:
         lCrawlerName: !If [FetchFromDatasetSsm, !Sub "{{resolve:ssm:/sdlf/dataset/rStageGlueCrawler/${pDatasetDeploymentInstance}}}", !Ref pGlueCrawler]
         lWaitTime: 75

sdlf-stage-glue/src/state-machine/stage-glue.asl.json

@@ -31,7 +31,8 @@
                     "Parameters": {
                       "JobName": "${lTransform}",
                       "WorkerType": "${lWorkerType}",
-                      "NumberOfWorkers": ${lNumberOfWorkers}
+                      "NumberOfWorkers": ${lNumberOfWorkers},
+                      "Arguments": ${lGlueArguments}
                     },
                     "End": true
                   }