New serverless application schemas support (#897)

This integrates Schema Code Download with new serverless application model (SAM) templates that expect the schema - essentially making dynamic changes

This introduces support for 2 new EventBridge SAM templates (one static, one dynamic).
The static one can be considered as "hello world" template - showing how to react to EC2 instance state change notifications with a minimal EventBridge template and Lambda as a Target.

The dynamic one is interesting. The Lambda functions in these templates expect to find code that is generated by the EventBridge Schema service, so the flow is to run the standard SAM application creation flow, then download the code as a post-creation action.
These templates can also work from SAM CLI interactive mode where users are prompted for a schema selection.

Author: ejafarli

.changes/next-release/Breaking Change-b3dae61d-5af8-4f09-bbbf-e84fdd2771fb.json

@@ -0,0 +1,4 @@
+{
+    "type": "Breaking Change",
+    "description": "Minimum version of SAM CLI has been adjusted from 0.32.0 to 0.38.0 to accommodate new SAM application support for EventBridge Schemas"
+}

.changes/next-release/Feature-5dbc40fa-fc6f-4c81-b5c9-f2f3a2402b04.json

@@ -0,0 +1,4 @@
+{
+    "type": "Feature",
+    "description": "Added the ability to create new Serverless Applications with EventBridge Schemas support."
+}

package.nls.json

@@ -199,6 +199,7 @@
     "AWS.samcli.configured.location": "Configured SAM CLI Location: {0}",
     "AWS.samcli.error.notFound": "Unable to find the SAM CLI, which is required to create new Serverless Applications and debug them locally. If you have already installed the SAM CLI, update your User Settings by locating it.",
     "AWS.samcli.error.notFound.brief": "Could not get SAM CLI location",
+    "AWS.samcli.error.invalid_schema_support_version": "Installed SAM executable does not support templates that require Event Schema selection. Required minimum version {0}, but found {1}",
     "AWS.samcli.local.invoke.ended": "Local invoke of SAM Application has ended.",
     "AWS.samcli.local.invoke.error": "Error encountered running local SAM Application",
     "AWS.samcli.local.invoke.port.not.open": "The debug port doesn't appear to be open. The debugger might not succeed when attaching to your SAM Application.",
@@ -224,7 +225,21 @@
     "AWS.initWizard.name.browse.openLabel": "Open",
     "AWS.samcli.initWizard.name.error.empty": "Application name cannot be empty",
     "AWS.samcli.initWizard.name.error.pathSep": "The path separator ({0}) is not allowed in application names",
+    "AWS.samcli.initWizard.schemas.aws_credentials_missing": "You need to be connected to AWS to select {0}.",
     "AWS.samcli.initWizard.runtime.prompt": "Select a SAM Application Runtime",
+    "AWS.samcli.initWizard.template.prompt": "Select a SAM Application Template",
+    "AWS.samcli.initWizard.template.helloWorld.name": "AWS SAM Hello World",
+    "AWS.samcli.initWizard.template.eventBridge_helloWorld.name": "AWS SAM EventBridge Hello World",
+    "AWS.samcli.initWizard.template.eventBridge_starterApp.name": "AWS SAM EventBridge App from Scratch",
+    "AWS.samcli.initWizard.template.helloWorld.description": "A basic SAM app",
+    "AWS.samcli.initWizard.template.eventBridge_helloWorld.description": "A Hello World app for Amazon EventBridge that invokes a Lambda for every EC2 instance state change in your account",
+    "AWS.samcli.initWizard.template.eventBridge_starterApp.description": "A starter app for Amazon EventBridge that invokes a Lambda based on a dynamic event trigger for an EventBridge Schema of your choice",
+    "AWS.samcli.initWizard.schemas.region.prompt": "Select an EventBridge Schemas Region",
+    "AWS.samcli.initWizard.schemas.registry.prompt": "Select a Registry",
+    "AWS.samcli.initWizard.schemas.registry.failed_to_load_resources": "Error loading registries.",
+    "AWS.samcli.initWizard.schemas.schema.prompt": "Select a Schema",
+    "AWS.samcli.initWizard.schemas.notFound": "No schemas found in registry {0}.",
+    "AWS.samcli.initWizard.schemas.failed_to_load_resources": "Error loading schemas in registry {0}.",
     "AWS.samcli.initWizard.source.error.notFound": "Project created successfully, but main source code file not found: {0}",
     "AWS.samcli.initWizard.source.error.notInWorkspace": "Could not open file '{0}'. If this file exists on disk, try adding it to your workspace.",
     "AWS.wizard.selectedPreviously": "Selected Previously",

src/eventSchemas/commands/downloadSchemaItemCode.ts

@@ -100,7 +100,7 @@ function getCoreFileName(schemaName: string, fileExtension: string) {
     return parsedName[parsedName.length - 1].concat(fileExtension)
 }
 
-function createSchemaCodeDownloaderObject(client: SchemaClient): SchemaCodeDownloader {
+export function createSchemaCodeDownloaderObject(client: SchemaClient): SchemaCodeDownloader {
     const downloader = new CodeDownloader(client)
     const generator = new CodeGenerator(client)
     const poller = new CodeGenerationStatusPoller(client)
@@ -115,7 +115,7 @@ export interface SchemaCodeDownloadRequestDetails {
     language: string
     schemaVersion: string
     destinationDirectory: vscode.Uri
-    schemaCoreCodeFileName: string
+    schemaCoreCodeFileName?: string
 }
 
 export class SchemaCodeDownloader {
@@ -342,16 +342,18 @@ export class CodeExtractor {
         })
     }
 
-    public getCoreCodeFilePath(codeZipFile: string, coreFileName: string): string | undefined {
-        const zip = new admZip(codeZipFile)
-        const zipEntries = zip.getEntries()
-
-        for (const zipEntry of zipEntries) {
-            if (zipEntry.isDirectory) {
-                // Ignore directories
-            } else {
-                if (zipEntry.name === coreFileName) {
-                    return zipEntry.entryName
+    public getCoreCodeFilePath(codeZipFile: string, coreFileName: string | undefined): string | undefined {
+        if (coreFileName) {
+            const zip = new admZip(codeZipFile)
+            const zipEntries = zip.getEntries()
+
+            for (const zipEntry of zipEntries) {
+                if (zipEntry.isDirectory) {
+                    // Ignore directories
+                } else {
+                    if (zipEntry.name === coreFileName) {
+                        return zipEntry.entryName
+                    }
                 }
             }
         }

src/eventSchemas/models/schemaCodeGenUtils.ts

@@ -0,0 +1,91 @@
+/*!
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// TODO: This is fragile. Very fragile. But it is necessary to get Schemas service launched, and we've evaluated all other tradeoffs
+// This will be done on the server-side as soon as we can, but for now the client needs to do this
+export class SchemaCodeGenUtils {
+    private readonly SCHEMA_PACKAGE_PREFIX = 'schema'
+    private readonly AWS = 'aws'
+    private readonly PARTNER = 'partner'
+    private readonly AWS_PARTNER_PREFIX = `${this.AWS}.${this.PARTNER}-` // dash suffix because of 3p partner registry name format
+    private readonly AWS_EVENTS_PREFIX = `${this.AWS}.` // . suffix because of 1p event registry schema format
+
+    public buildSchemaPackageName(schemaName: string): string {
+        const builder = new CodeGenPackageBuilder()
+        builder.append(this.SCHEMA_PACKAGE_PREFIX)
+        this.buildPackageName(builder, schemaName)
+
+        return builder.build()
+    }
+
+    private buildPackageName(builder: CodeGenPackageBuilder, schemaName: string): void {
+        // do not modify the order of conditional checks
+        if (this.isAwsPartnerEvent(schemaName)) {
+            this.buildPartnerEventPackageName(builder, schemaName)
+        } else if (this.isAwsEvent(schemaName)) {
+            this.buildAwsEventPackageName(builder, schemaName)
+        } else {
+            this.buildCustomPackageName(builder, schemaName)
+        }
+    }
+
+    private isAwsPartnerEvent(schemaName: string): boolean {
+        return schemaName.startsWith(this.AWS_PARTNER_PREFIX)
+    }
+
+    private buildPartnerEventPackageName(builder: CodeGenPackageBuilder, schemaName: string): void {
+        const partnerSchemaString = schemaName.substring(this.AWS_PARTNER_PREFIX.length)
+
+        builder
+            .append(this.AWS)
+            .append(this.PARTNER)
+            .append(partnerSchemaString)
+    }
+
+    private isAwsEvent(name: string): boolean {
+        return name.startsWith(this.AWS_EVENTS_PREFIX)
+    }
+
+    private buildAwsEventPackageName(builder: CodeGenPackageBuilder, schemaName: string): void {
+        const awsEventSchemaParts = schemaName.split('.')
+        for (const part of awsEventSchemaParts) {
+            builder.append(part)
+        }
+    }
+
+    private buildCustomPackageName(builder: CodeGenPackageBuilder, schemaName: string): void {
+        builder.append(schemaName)
+    }
+}
+
+class CodeGenPackageBuilder {
+    private builder = ''
+    public build(): string {
+        return this.builder
+    }
+
+    public append(segment: String): CodeGenPackageBuilder {
+        if (this.builder.length > 0) {
+            this.builder = this.builder.concat(IdentifierFormatter.PACKAGE_SEPARATOR)
+        }
+        this.builder = this.builder.concat(IdentifierFormatter.toValidIdentifier(segment.toLowerCase()))
+
+        return this
+    }
+}
+
+export namespace IdentifierFormatter {
+    export const PACKAGE_SEPARATOR = '.'
+    const POTENTIAL_PACKAGE_SEPARATOR = '@'
+    const NOT_VALID_IDENTIFIER_REGEX = new RegExp(`[^a-zA-Z0-9_${POTENTIAL_PACKAGE_SEPARATOR}]`, 'g')
+    const POTENTIAL_PACKAGE_SEPARATOR_REGEX = new RegExp(POTENTIAL_PACKAGE_SEPARATOR, 'g')
+    const UNDERSCORE = '_'
+
+    export function toValidIdentifier(name: string): string {
+        return name
+            .replace(NOT_VALID_IDENTIFIER_REGEX, UNDERSCORE)
+            .replace(POTENTIAL_PACKAGE_SEPARATOR_REGEX, PACKAGE_SEPARATOR)
+    }
+}

src/eventSchemas/models/schemaCodeLangs.ts

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { Runtime } from 'aws-sdk/clients/lambda'
 import { Set } from 'immutable'
 
 export const JAVA = 'Java 8+'
@@ -44,3 +45,15 @@ export function getLanguageDetails(
             throw new Error(`Language ${language} is not supported as Schema Code Language`)
     }
 }
+
+export function supportsEventBridgeTemplates(runtime: Runtime): boolean {
+    return runtime === 'python3.7' || runtime === 'python3.6' || runtime === 'python3.8'
+}
+
+export function getApiValueForSchemasDownload(runtime: Runtime): string {
+    if (supportsEventBridgeTemplates(runtime)) {
+        return 'Python36'
+    }
+
+    throw new Error(`Runtime ${runtime} is not supported by eventBridge application`)
+}

src/eventSchemas/providers/schemasDataProvider.ts

@@ -0,0 +1,128 @@
+/*!
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { Credentials, Schemas } from 'aws-sdk'
+import { SchemaClient } from '../../shared/clients/schemaClient'
+import { getLogger, Logger } from '../../shared/logger'
+import { toArrayAsync } from '../../shared/utilities/collectionUtils'
+
+export class Cache {
+    public constructor(public readonly credentialsRegionDataList: credentialsRegionDataListMap[]) {}
+}
+
+export interface credentialsRegionDataListMap {
+    credentials: Credentials
+    regionDataList: regionRegistryMap[]
+}
+
+export interface regionRegistryMap {
+    region: string
+    registryNames: string[]
+    registrySchemasMapList: registrySchemasMap[]
+}
+
+export interface registrySchemasMap {
+    registryName: string
+    schemaList: Schemas.SchemaSummary[]
+}
+
+/**
+ * Responsible for retaining registry && schema list per region for Create-New-SAM-Application wizard
+ */
+export class SchemasDataProvider {
+    private static INSTANCE: SchemasDataProvider | undefined
+    private readonly logger: Logger = getLogger()
+
+    public constructor(private readonly cache: Cache) {}
+
+    public async getRegistries(region: string, client: SchemaClient, credentials: Credentials) {
+        const cachedRegion = this.cache.credentialsRegionDataList
+            .filter(x => x.credentials === credentials)
+            .shift()
+            ?.regionDataList.filter(x => x.region === region)
+            .shift()
+
+        try {
+            // if region is not cached, make api query and retain results
+            if (!cachedRegion || cachedRegion.registryNames.length === 0) {
+                const registrySummary = await toArrayAsync(client.listRegistries())
+                const registryNames = registrySummary.map(x => x.RegistryName!)
+                this.pushRegionDataIntoCache(region, registryNames, [], credentials)
+
+                return registryNames
+            }
+        } catch (err) {
+            const error = err as Error
+            this.logger.error('Error retrieving registries', error)
+
+            return undefined
+        }
+
+        return cachedRegion!.registryNames
+    }
+
+    public async getSchemas(region: string, registryName: string, client: SchemaClient, credentials: Credentials) {
+        const registrySchemasMapList = this.cache.credentialsRegionDataList
+            .filter(x => x.credentials === credentials)
+            .shift()
+            ?.regionDataList.filter(x => x.region === region)
+            .shift()?.registrySchemasMapList
+        let schemas = registrySchemasMapList?.filter(x => x.registryName === registryName).shift()?.schemaList
+        try {
+            // if no schemas found, make api query and retain results given that registryName && region already cached
+            if (!schemas || schemas.length === 0) {
+                schemas = await toArrayAsync(client.listSchemas(registryName))
+                const singleItem: registrySchemasMap = { registryName: registryName, schemaList: schemas }
+                //wizard setup always calls getRegistries method prior to getSchemas, so this shouldn't be undefined
+                if (!registrySchemasMapList) {
+                    this.pushRegionDataIntoCache(region, [], [singleItem], credentials)
+                }
+
+                if (registrySchemasMapList) {
+                    registrySchemasMapList.push(singleItem)
+                }
+            }
+        } catch (err) {
+            const error = err as Error
+            this.logger.error('Error retrieving schemas', error)
+
+            return undefined
+        }
+
+        return schemas
+    }
+
+    private pushRegionDataIntoCache(
+        region: string,
+        registryNames: string[],
+        registrySchemasMapList: registrySchemasMap[],
+        credentials?: Credentials
+    ): void {
+        const regionData: regionRegistryMap = {
+            region: region,
+            registryNames: registryNames,
+            registrySchemasMapList: registrySchemasMapList
+        }
+
+        const cachedCredential = this.cache.credentialsRegionDataList.filter(x => x.credentials === credentials).shift()
+        cachedCredential?.regionDataList.push(regionData)
+
+        if (!cachedCredential) {
+            const regionDataWithCredentials: credentialsRegionDataListMap = {
+                credentials: credentials!,
+                regionDataList: [regionData]
+            }
+            this.cache.credentialsRegionDataList.push(regionDataWithCredentials)
+        }
+    }
+
+    public static getInstance(): SchemasDataProvider {
+        if (!SchemasDataProvider.INSTANCE) {
+            SchemasDataProvider.INSTANCE = new SchemasDataProvider(new Cache([]))
+        }
+
+        return SchemasDataProvider.INSTANCE
+    }
+}