[sdlf-dataset][cdk] ability to provide parameter values without CfnParameter

Author: cnfait

sdlf-dataset/deployspec.yaml

@@ -12,10 +12,7 @@ deploy:
     build:
       commands:
         - |-
-          cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json \
-            --no-previous-parameters \
-            --parameters DatasetStack:pDatasetName="$SEEDFARMER_PARAMETER_DATASET" \
-            --parameters DatasetStack:pS3Prefix="$SEEDFARMER_PARAMETER_S3_PREFIX"
+          cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json
         - seedfarmer metadata convert -f cdk-exports.json || true
 destroy:
   phases:

sdlf-dataset/src/app.py

@@ -12,8 +12,15 @@
 
 if project_name:
     stack_name = f"{project_name}-{deployment_name}-{module_name}"
+    param_prefix = "SEEDFARMER_PARAMETER_"
 else:  # app.py not used in a seedfarmer context somehow
     stack_name = "sdlf-dataset"
+    param_prefix = ""
+
+
+def _param(name: str, default: str = None) -> str:
+    return os.getenv(f"{param_prefix}{name}", default)
+
 
 app = cdk.App()
 lakeformation_cicd_stack = cdk.Stack(app, f"{stack_name}-lakeformation")
@@ -33,7 +40,17 @@
 
 stack = cdk.Stack(app, stack_name)
 stack.add_dependency(lakeformation_cicd_stack)
-stack_dataset = Dataset(stack, "dataset")
+stack_dataset = Dataset(
+    stack,
+    "dataset",
+    raw_bucket=_param("RAW_BUCKET"),
+    stage_bucket=_param("STAGE_BUCKET"),
+    analytics_bucket=_param("ANALYTICS_BUCKET"),
+    artifacts_bucket=_param("ARTIFACTS_BUCKET"),
+    lakeformation_dataaccess_role=_param("LAKEFORMATION_DATAACCESS_ROLE"),
+    dataset=_param("DATASET"),
+    s3_prefix=_param("S3_PREFIX"),
+)
 
 cdk.CfnOutput(
     scope=stack,

sdlf-dataset/src/dataset.py

@@ -28,11 +28,37 @@
 class Dataset(Construct):
     external_interface = {}
 
-    def __init__(self, scope: Construct, id: str, **kwargs) -> None:
+    def __init__(
+        self,
+        scope: Construct,
+        id: str,
+        dataset: str,
+        s3_prefix: str,
+        org: str = None,
+        data_domain: str = None,
+        raw_bucket: str = None,
+        stage_bucket: str = None,
+        analytics_bucket: str = None,
+        artifacts_bucket: str = None,
+        lakeformation_dataaccess_role: str = None,
+        **kwargs,
+    ) -> None:
         super().__init__(scope, id)
 
-        # using context values would be better(?) for CDK but we haven't decided yet what the story is around ServiceCatalog and CloudFormation modules
-        # perhaps both (context values feeding into CfnParameter) would be a nice-enough solution. Not sure though. TODO
+        # if arguments are passed to the constructor, their values are fed to CfnParameter() below as default
+        # if arguments aren't specified, standard SDLF SSM parameters are checked instead
+        # the jury is still out on the usage of CfnParameter(),
+        # and there is still work to get the latest SSM parameter values as well as using a prefix to allow multiple deployments TODO
+        raw_bucket = raw_bucket or "{{resolve:ssm:/sdlf/storage/rRawBucket:1}}"
+        stage_bucket = stage_bucket or "{{resolve:ssm:/sdlf/storage/rStageBucket:1}}"
+        analytics_bucket = analytics_bucket or "{{resolve:ssm:/sdlf/storage/rAnalyticsBucket:1}}"
+        artifacts_bucket = artifacts_bucket or "{{resolve:ssm:/sdlf/storage/rArtifactsBucket:1}}"
+        lakeformation_dataaccess_role = (
+            lakeformation_dataaccess_role or "{{resolve:ssm:/sdlf/storage/rLakeFormationDataAccessRoleArn:1}}"
+        )
+        org = org or "{{resolve:ssm:/sdlf/storage/rOrganization:1}}"
+        data_domain = data_domain or "{{resolve:ssm:/sdlf/storage/rDomain:1}}"
+
         p_pipelinereference = CfnParameter(
             self,
             "pPipelineReference",
@@ -45,54 +71,54 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             "pOrg",
             description="Name of the organization owning the datalake",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/pOrg:1}}",
+            default="{{resolve:ssm:/sdlf/storage/rOrganization:1}}",
         )
         p_org.override_logical_id("pOrg")
         p_domain = CfnParameter(
             self,
             "pDomain",
             description="Data domain name",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/pDomain:1}}",
+            default="{{resolve:ssm:/sdlf/storage/rDomain:1}}",
         )
         p_domain.override_logical_id("pDomain")
         p_rawbucket = CfnParameter(
             self,
             "pRawBucket",
             description="The raw bucket for the solution",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/rRawBucket:1}}",
+            default=raw_bucket,
         )
         p_rawbucket.override_logical_id("pRawBucket")
         p_stagebucket = CfnParameter(
             self,
             "pStageBucket",
             description="The stage bucket for the solution",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/rStageBucket:1}}",
+            default=stage_bucket,
         )
         p_stagebucket.override_logical_id("pStageBucket")
         p_analyticsbucket = CfnParameter(
             self,
             "pAnalyticsBucket",
             description="The analytics bucket for the solution",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/rAnalyticsBucket:1}}",
+            default=analytics_bucket,
         )
         p_analyticsbucket.override_logical_id("pAnalyticsBucket")
         p_artifactsbucket = CfnParameter(
             self,
             "pArtifactsBucket",
             description="The artifacts bucket  used by CodeBuild and CodePipeline",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/rArtifactsBucket:1}}",
+            default=artifacts_bucket,
         )
         p_artifactsbucket.override_logical_id("pArtifactsBucket")
         p_lakeformationdataaccessrole = CfnParameter(
             self,
             "pLakeFormationDataAccessRole",
             type="String",
-            default="{{resolve:ssm:/sdlf/storage/rLakeFormationDataAccessRoleArn:1}}",
+            default=lakeformation_dataaccess_role,
         )
         p_lakeformationdataaccessrole.override_logical_id("pLakeFormationDataAccessRole")
         p_datasetname = CfnParameter(
@@ -101,6 +127,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             description="The name of the dataset (all lowercase, no symbols or spaces)",
             type="String",
             allowed_pattern="[a-z0-9]{2,14}",
+            default=dataset,
         )
         p_datasetname.override_logical_id("pDatasetName")
         p_s3prefix = CfnParameter(
@@ -109,6 +136,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             description="S3 prefix or full bucket if empty/not provided",
             type="String",
             allowed_pattern="[a-z0-9]*",
+            default=s3_prefix,
         )
         p_s3prefix.override_logical_id("pS3Prefix")
         p_cicdrole = CfnParameter(