[sdlf-stage-glue] max items per batch configurable through cfn parameter

Author: cnfait

sdlf-stage-glue/src/glue.yaml

@@ -112,6 +112,10 @@ Parameters:
   pEnableTracing:
     Description: Flag for whether XRay tracing is enabled
     Type: String
+  pMaxItemsPerBatch:
+    Description: Maximum number of items to process per batch - each Glue job execution will process that many items, if available
+    Type: Number
+    Default: 5
   pGlueJobName:
     Description: Glue job name
     Type: String
@@ -523,6 +527,7 @@ Resources:
       DefinitionSubstitutions:
         lPostMetadata: !GetAtt rLambdaPostMetadataStep.Arn
         lError: !GetAtt rLambdaErrorStep.Arn
+        lMaxItemsPerBatch: !Ref pMaxItemsPerBatch
         lTransform: !Ref pGlueJobName
         lWorkerType: !Ref pGlueWorkerType
         lNumberOfWorkers: !Ref pGlueNumberOfWorkers

sdlf-stage-glue/src/state-machine/stage-glue.asl.json

@@ -41,7 +41,7 @@
               "MaxConcurrency": 50,
               "ToleratedFailurePercentage": 100,
               "ItemBatcher": {
-                "MaxItemsPerBatch": 1
+                "MaxItemsPerBatch": ${lMaxItemsPerBatch}
               },
               "InputPath": "$.Items",
               "Next": "Run Glue Crawler"