Feat: emr on eks data platform (#187)

* initial version or emr on eks construct

* added correct dependency for js-yaml

* upgrading projen and changing dependencies specification in .projenrc.js

* refactoring emr on eks

* refactor EKS cluster creation

* updated constructs (not finished)

* changes to make original version jsii-compliant

* fix to make cdk deploy compile successfully

* cdk deploys successfully

* stack is in deployable state: cluster autoscaler needs fixing

* cluster autoscaler is fixed

* comments & cleanup

* first draft for managed endpoint based on lambda custom resource, not functional at the moment

* changed client name variable to avoid scope confusion (raised by Lotfi)

* fix: managed endpoint now works, todo: sort out generation of ssl certificate

* fix: add timeout, format changes

* added security groups needs for emrstudio

* fix: support multiple managed endpoints

* added executionRoleArn as props

* fix: nodegroup taints, added configurationOverride

* upgrade cdk

* feature: nodegroup improvements and code refactoring

* fix: nodegroup taints and unit tests

* fix: tooling nodegroup labels for cluster autoscaler

* fix: added check on managedendpoint name lenght to be less than or equal to 64

* fix: add waiting for the endpoint termination status

* add license

* add managed endpoint dependency on eksCluster

* fix: simplify code in emr virtual cluster

* change execution role creation

* refactoring managed endpoint

* adding default EMR config and pod templates for default nodegroups

* refactoring managed endpoint custom resource

* fix: change in addManagedEndpoint in emr-eks-cluster.ts to support multistack, function now has one more parameter to take the scope of the a stack.

* fix: check constraint on namespace before its creation, only allowing lowercase alphanumeric

* fix: fix in emr-eks-cluster.ts to pass the test case and scoped the CR permissions

* fix: issues with scoped policy for CR in emr-eks-cluster.ts

* added a workaround for pod templates in notebook

* added unique service token to the managed endpoint CR lambda

Co-authored-by: Alex Tarasov <alexvt@amazon.com>
Co-authored-by: Lotfi Mouhib <mouhib@amazon.com>
Co-authored-by: Vincent Gromakowski <gromav@amazon.com>

Author: 3 people

CONTRIB_FAQ.md

@@ -138,6 +138,12 @@ See the `Dataset` class using static variables call the class constructor [here]
 
 ### How to implement Singleton pattern for an AWS CDK resource
 
+A singleton resource can have different scope:
+ * Across stacks: if you don't need to version it and it's the same resource across all AWS CDK Applications
+ * Within stacks: if the resource can evolve across versions, it's preferable to scope it to a stack and assign a unique ID
+
+#### Within stacks
+
 The singleton pattern can be implemented using the unique ID of the AWS CDK node. Instead of creating a new resource from the AWS CDK Construct, a static `getOrCreate` method is used to retrieve the resource by search for the unique ID in the AWS CDK Scope. If no resource exists, the method creates a new one.
 
 See the `SingletonBucket` Construct [here](./core/src/singleton-bucket.ts).

core/.projen/deps.json

@@ -1,10 +1,12 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: MIT-0
+
 const { basename, join, dirname, relative } = require('path');
 const glob = require('glob');
 
 
-const { AwsCdkConstructLibrary, DependenciesUpgradeMechanism } = require('projen');
+const { AwsCdkConstructLibrary, DependenciesUpgradeMechanism, Semver } = require('projen');
+
 const project = new AwsCdkConstructLibrary({
 
   authorName: 'Amazon Web Services',
@@ -23,7 +25,7 @@ const project = new AwsCdkConstructLibrary({
 
   cdkVersion: '1.130',
   defaultReleaseBranch: 'main',
-  license: 'MIT',
+  license: 'MIT-0',
   name: 'aws-analytics-reference-architecture',
   repositoryUrl: 'https://github.com/aws-samples/aws-analytics-reference-architecture.git',
   repositoryDirectory: 'core',
@@ -36,46 +38,59 @@ const project = new AwsCdkConstructLibrary({
   cdkVersionPinning: true,
 
   cdkDependencies: [
-    '@aws-cdk/core',
-    '@aws-cdk/custom-resources',
-    '@aws-cdk/aws-logs',
-    '@aws-cdk/aws-lambda',
-    '@aws-cdk/aws-lambda-python',
-    '@aws-cdk/aws-s3',
-    '@aws-cdk/aws-kinesis',
+    '@aws-cdk/assertions',
+    '@aws-cdk/aws-athena',
+    '@aws-cdk/aws-autoscaling',
+    '@aws-cdk/aws-ec2',
+    '@aws-cdk/aws-emrcontainers',
+    '@aws-cdk/aws-eks',
+    '@aws-cdk/aws-events',
+    '@aws-cdk/aws-events-targets',
+    '@aws-cdk/aws-glue',
     '@aws-cdk/aws-iam',
+    '@aws-cdk/aws-kinesis',
     '@aws-cdk/aws-kinesisfirehose',
     '@aws-cdk/aws-kinesisfirehose-destinations',
-    '@aws-cdk/aws-kinesis',
+    '@aws-cdk/aws-lambda',
+    '@aws-cdk/aws-lambda-python',
     '@aws-cdk/aws-logs',
-    '@aws-cdk/aws-glue',
-    '@aws-cdk/aws-athena',
-    '@aws-cdk/aws-glue',
-    '@aws-cdk/aws-stepfunctions',
-    '@aws-cdk/aws-stepfunctions-tasks',
-    '@aws-cdk/aws-events',
-    '@aws-cdk/aws-events-targets',
-    '@aws-cdk/aws-s3-deployment',
-    '@aws-cdk/aws-ec2',
     '@aws-cdk/aws-redshift',
-    '@aws-cdk/aws-secretsmanager',
+    '@aws-cdk/aws-s3',
     '@aws-cdk/aws-s3-assets',
-    '@aws-cdk/assertions',
-  ],
-  bundledDeps: [
-    'xmldom@github:xmldom/xmldom#0.7.0',
-    'aws-sdk',
+    '@aws-cdk/aws-s3-deployment',
+    '@aws-cdk/aws-secretsmanager',
+    '@aws-cdk/aws-stepfunctions',
+    '@aws-cdk/aws-stepfunctions-tasks',
+    '@aws-cdk/core',
+    '@aws-cdk/custom-resources',
+    '@aws-cdk/lambda-layer-awscli',
   ],
 
   devDeps: [
+    '@types/js-yaml',
+    '@types/jest',
     'esbuild',
   ],
 
+  bundledDeps: [
+    'js-yaml',
+    'uuid',
+    'aws-sdk',
+  ],
+
   python: {
     distName: 'aws_analytics_reference_architecture',
     module: 'aws_analytics_reference_architecture',
   },
 
+  tsconfig: {
+    compilerOptions: {
+      resolveJsonModule: true,
+      esModuleInterop: true,
+    },
+    include: ['src/**/*.json', 'src/**/*.ts'],
+  },
+
   stability: 'experimental',
 
 });