Fixed deployment of aws resources if a pipeline name is not specified (aws#117)

Fixed deployment of aws resources which would fail if option `--create-update-aws-pipeline-name` was not specified:
* corrected mismatch of resolver schema file name when creating the lambda zip file by passing the generated schema file name to the function which is creating the zip file
* changed the output file prefix to only have the `.output` suffix if there is no discovered graph name
* changed App Sync Function and DataSource resource names to replace any dashes `-` with underscores as dashes are not allowed (but can be present in Neptune db/graph names)
* added integration tests to validate that correct output files and zip file content are created if no pipeline name is provided

Other minor changes:
* changed file error logging to include the `error.message` as the error message was output to console but missing from the file
* added validation of output folder files to CDK integration tests
* modified affected test cases to have descriptive file names and output folders

Author: andreachild

CHANGELOG.md

@@ -108,4 +108,7 @@ permissions and limitations under the License.
   can be referenced further in the
   query ([#114](https://github.com/aws/amazon-neptune-for-graphql/pull/114))
 * Fixed Apollo template to reference `graphql` `parse` instead of `graphql-tag`
-  `gql` ([#116](https://github.com/aws/amazon-neptune-for-graphql/pull/116))
+  `gql` ([#116](https://github.com/aws/amazon-neptune-for-graphql/pull/116))
+* Fixed deployment of AWS resources if a pipeline name is not provided as an
+  input
+  option ([#117](https://github.com/aws/amazon-neptune-for-graphql/pull/117))

src/CDKPipelineApp.js

@@ -54,10 +54,9 @@ async function getSchemaFields(typeName) {
     return r;
 }
 
-async function createDeploymentFile(templateFolderPath, resolverFilePath) {
+async function createDeploymentFile({templateFolderPath, resolverFilePath, resolverSchemaFilePath}) {
     try {
         const zipFilePath = path.join(RELATIVE_OUTPUT_PATH, `${NAME}.zip`);
-        const resolverSchemaFilePath = path.join(RELATIVE_OUTPUT_PATH, `${NAME}.resolver.schema.json.gz`)
         await createLambdaDeploymentPackage({
             outputZipFilePath: zipFilePath,
             templateFolderPath: templateFolderPath,
@@ -85,6 +84,7 @@ async function createAWSpipelineCDK({
                                         neptunePort,
                                         outputFolderPath,
                                         resolverFilePath,
+                                        resolverSchemaFilePath,
                                         neptuneType
                                     }) {
 
@@ -153,7 +153,11 @@ async function createAWSpipelineCDK({
     }
 
     if (!quiet) spinner = ora('Creating ZIP ...').start();
-    await createDeploymentFile(lambdaFilesPath, resolverFilePath);
+    await createDeploymentFile({
+        templateFolderPath: lambdaFilesPath,
+        resolverFilePath: resolverFilePath,
+        resolverSchemaFilePath: resolverSchemaFilePath
+    });
     if (!quiet) spinner.succeed('Created ZIP File: ' + yellow(LAMBDA_ZIP_FILE));
 
     APPSYNC_ATTACH_QUERY = await getSchemaFields('Query');

src/logger.js

@@ -72,7 +72,7 @@ function loggerError(errorMessage, error) {
     let toLog = removeYellow(errorMessage);
     if (error) {
         toConsole = toConsole + ': ' + error.message + ' - Please see ' + logFileDestination + ' for more details';
-        toLog = toLog + '\n' + JSON.stringify(error, null, 4);
+        toLog = toLog + ': ' + error.message + '\n' + JSON.stringify(error, null, 4);
     }
     console.error(toConsole);
     fileLogger.error(toLog);

src/main.js

@@ -401,7 +401,8 @@ async function main() {
         createUpdatePipelineNeptuneDatabaseName || 
         inputCDKpipelineName || 
         inputCDKpipelineDatabaseName ||
-        `${neptuneInfo?.graphName?.concat('.') || ''}output`;
+        neptuneInfo?.graphName ||
+        'output';
     // save the neptune schema early for troubleshooting purposes
     saveNeptuneSchema(outputFilePrefix);
 
@@ -591,8 +592,8 @@ async function main() {
         }
 
         // Output Lambda resolver
-        if (outputLambdaResolverFile == '') { 
-            outputLambdaResolverFile = outputFolderPath + '/output.resolver.graphql.js'; 
+        if (!outputLambdaResolverFile) { 
+            outputLambdaResolverFile = path.join(outputFolderPath, `${outputFilePrefix}.resolver.graphql.js`); 
         }
 
         try {
@@ -659,11 +660,10 @@ async function main() {
 
         if  ( !(createUpdatePipeline || inputCDKpipeline) && createLambdaZip) {
 
-            if (outputLambdaResolverZipFile == '' && outputLambdaResolverZipName == '')  
-                outputLambdaResolverZipFile = outputFolderPath + '/output.lambda.zip';
-            
-            if (outputLambdaResolverZipFile == '' && outputLambdaResolverZipName != '')  
-                outputLambdaResolverZipFile = outputFolderPath + '/' + outputLambdaResolverZipName + '.zip';
+            if (!outputLambdaResolverZipFile) {
+                const zipName = outputLambdaResolverZipName || `${outputFilePrefix}.lambda`;
+                outputLambdaResolverZipFile = path.join(outputFolderPath, `${zipName}.zip`);
+            }
 
             try {
                 if (!quiet) spinner = ora('Creating Lambda ZIP ...').start();
@@ -693,21 +693,24 @@ async function main() {
                 let neptuneHost = neptuneInfo.host;
                 let neptunePort = neptuneInfo.port;
 
-                await createUpdateAWSpipeline(  createUpdatePipelineName, 
-                                                createUpdatePipelineNeptuneDatabaseName, 
-                                                createUpdatePipelineRegion,
-                                                outputSchema,
-                                                schemaModel,
-                                                __dirname + outputLambdaPackagePath,
-                                                outputSchemaMutations,
-                                                quiet,
-                                                __dirname,
-                                                isNeptuneIAMAuth,
-                                                neptuneHost,
-                                                neptunePort,
-                                                outputFolderPath,
-                                                outputLambdaResolverFile,
-                                                neptuneType );
+                await createUpdateAWSpipeline({
+                    pipelineName: createUpdatePipelineName,
+                    neptuneDBName: createUpdatePipelineNeptuneDatabaseName,
+                    neptuneDBregion: createUpdatePipelineRegion,
+                    appSyncSchema: outputSchema,
+                    schemaModel: schemaModel,
+                    lambdaFilesPath: __dirname + outputLambdaPackagePath,
+                    addMutations: outputSchemaMutations,
+                    quietI: quiet,
+                    __dirname: __dirname,
+                    isNeptuneIAMAuth: isNeptuneIAMAuth,
+                    neptuneHost: neptuneHost,
+                    neptunePort: neptunePort,
+                    outputFolderPath: outputFolderPath,
+                    resolverFilePath: outputLambdaResolverFile,
+                    resolverSchemaFilePath: resolverSchemaFile,
+                    neptuneType: neptuneType
+                });
             } catch (err) {
                 loggerError('Error creating AWS pipeline', err);
             }
@@ -743,6 +746,7 @@ async function main() {
                     neptunePort: neptunePort,
                     outputFolderPath: outputFolderPath,
                     resolverFilePath: outputLambdaResolverFile,
+                    resolverSchemaFilePath: resolverSchemaFile,
                     neptuneType: neptuneType
                 });
             } catch (err) {

src/pipelineResources.js

@@ -392,11 +392,11 @@ async function createLambdaRole() {
  * Creates the lambda deployment ZIP package
  * @param templateFolderPath the path to the template folder that contains contents to add to the zip
  * @param resolverFilePath the path to the resolver file that should be added to the zip
+ * @param resolverSchemaFilePath the path to the resolver schema file that should be added to the zip
  * @returns {Promise<Buffer<ArrayBufferLike>>}
  */
-async function createDeploymentPackage(templateFolderPath, resolverFilePath) {
+async function createDeploymentPackage({templateFolderPath, resolverFilePath, resolverSchemaFilePath}) {
     const zipFilePath = path.join(thisOutputFolderPath, `${NAME}.zip`);
-    const resolverSchemaFilePath = path.join(thisOutputFolderPath, `${NAME}.resolver.schema.json.gz`)
     await createLambdaDeploymentPackage({
         outputZipFilePath: zipFilePath,
         templateFolderPath: templateFolderPath,
@@ -535,11 +535,15 @@ async function createAppSyncAPI() {
     const apiKey = response.apiKey.id;
     succeedSpinner('Created API key', {logLevel: 'info'});
 
+    // some resource names cannot have dashes
+    const sanitizedName = NAME.replaceAll('-', '_');
+    
     // create datasource
     startSpinner('Creating DataSource ...', true);
+    const dataSourceName = `${sanitizedName}DataSource`;
     params = {
         apiId: apiId,
-        name: NAME + 'DataSource',       
+        name: dataSourceName,       
         type: "AWS_LAMBDA",
         serviceRoleArn: LAMBDA_INVOCATION_ROLE,
         lambdaConfig: {
@@ -548,15 +552,16 @@ async function createAppSyncAPI() {
     };    
     command = new CreateDataSourceCommand(params);
     response = await appSyncClient.send(command);
-    succeedSpinner('Created DataSource: ' + yellow(NAME+'DataSource'), {logLevel: 'debug'});
+    succeedSpinner('Created DataSource: ' + yellow(dataSourceName), {logLevel: 'debug'});
     loggerInfo('Created datasource');
 
     // create function
     startSpinner('Creating Function ...', true);
+    const functionName = `${sanitizedName}Function`;
     params = {
         apiId: apiId,
-        name: NAME+'Function',       
-        dataSourceName: NAME+'DataSource',
+        name: functionName,       
+        dataSourceName: dataSourceName,
         runtime: {
             name: "APPSYNC_JS",
             runtimeVersion: "1.0.0",
@@ -591,7 +596,7 @@ export function response(ctx) {
     await sleep(5000);
     let functionId = response.functionConfiguration.functionId;    
     storeResource({AppSyncAPIFunction: functionId});
-    succeedSpinner('Created Function: ' + yellow(NAME+'Function'), {logLevel: 'debug'});
+    succeedSpinner('Created Function: ' + yellow(functionName), {logLevel: 'debug'});
     loggerInfo('Created function');
 
     // Upload schema
@@ -917,21 +922,24 @@ async function updateAppSyncAPI(resources) {
 }
 
 
-async function createUpdateAWSpipeline (    pipelineName,
-                                            neptuneDBName,
-                                            neptuneDBregion,
-                                            appSyncSchema,
-                                            schemaModel,
-                                            lambdaFilesPath,
-                                            addMutations,
-                                            quietI,
-                                            __dirname,
-                                            isNeptuneIAMAuth,
-                                            neptuneHost,
-                                            neptunePort,
-                                            outputFolderPath,
-                                            resolverFilePath,
-                                            neptuneType) {
+async function createUpdateAWSpipeline({
+                                           pipelineName,
+                                           neptuneDBName,
+                                           neptuneDBregion,
+                                           appSyncSchema,
+                                           schemaModel,
+                                           lambdaFilesPath,
+                                           addMutations,
+                                           quietI,
+                                           __dirname,
+                                           isNeptuneIAMAuth,
+                                           neptuneHost,
+                                           neptunePort,
+                                           outputFolderPath,
+                                           resolverFilePath,
+                                           resolverSchemaFilePath,
+                                           neptuneType
+                                       }) {
 
     NAME = pipelineName;
     REGION = neptuneDBregion;
@@ -1002,7 +1010,11 @@ async function createUpdateAWSpipeline (    pipelineName,
             }
 
             startSpinner('Creating ZIP ...', true);
-            ZIP = await createDeploymentPackage(LAMBDA_FILES_PATH, resolverFilePath);
+            ZIP = await createDeploymentPackage({
+                templateFolderPath: LAMBDA_FILES_PATH,
+                resolverFilePath: resolverFilePath,
+                resolverSchemaFilePath: resolverSchemaFilePath
+            });
             succeedSpinner('Created ZIP File: ' + yellow(LAMBDA_FILES_PATH), {logLevel: 'info'});
 
             await createLambdaRole();
@@ -1034,7 +1046,11 @@ async function createUpdateAWSpipeline (    pipelineName,
         }  
 
         startSpinner('Creating ZIP ...', true);
-        ZIP = await createDeploymentPackage(LAMBDA_FILES_PATH, resolverFilePath);
+        ZIP = await createDeploymentPackage({
+            templateFolderPath: LAMBDA_FILES_PATH,
+            resolverFilePath: resolverFilePath,
+            resolverSchemaFilePath: resolverSchemaFilePath
+        });
         succeedSpinner('Created ZIP File: ' + yellow(LAMBDA_FILES_PATH), {logLevel: 'info'});
 
         loggerInfo('Updating Lambda function', {toConsole: true});

src/zipPackage.js

@@ -84,19 +84,20 @@ async function createZip({targetZipFilePath, includePaths = [], includeContent =
  * @returns {Promise<Buffer<ArrayBufferLike>>}
  */
 export async function createLambdaDeploymentPackage({outputZipFilePath, templateFolderPath, resolverFilePath, resolverSchemaFilePath}) {
-    const filePaths = [{source: templateFolderPath}, {source: resolverFilePath, target: 'output.resolver.graphql.js'}];
     const modulePath = getModulePath();
+    const filePaths = [
+        {source: templateFolderPath},
+        {source: resolverFilePath, target: 'output.resolver.graphql.js'},
+        {source: resolverSchemaFilePath, target: 'output.resolver.schema.json.gz'},
+        {source: path.join(modulePath, '/../templates/util.mjs')}
+    ];
+
     if (templateFolderPath.includes('HTTP')) {
         filePaths.push({
             source: path.join(modulePath, '/../templates/queryHttpNeptune.mjs')
         })
     }
 
-    filePaths.push({
-        source: resolverSchemaFilePath,
-        target: 'output.resolver.schema.json.gz'
-    }, {source: path.join(modulePath, '/../templates/util.mjs')});
-
     await createZip({
         targetZipFilePath: outputZipFilePath,
         includePaths: filePaths

templates/CDKTemplate.js

@@ -163,11 +163,15 @@ class AppSyncNeptuneStack extends Stack {
                 ]       
             })
         );
-                
-        // AppSync: DataSource
-        const dataSource = new CfnDataSource(this, NAME + 'DataSource', {
+
+       // some resources names cannot have dashes
+       const sanitizedName = NAME.replaceAll('-', '_');
+       
+       // AppSync: DataSource
+       const dataSourceName = `${sanitizedName}DataSource`;
+       const dataSource = new CfnDataSource(this, dataSourceName, {
             apiId: itemsGraphQLApi.attrApiId,
-            name: NAME + 'DataSource',
+            name: dataSourceName,
             type: 'AWS_LAMBDA',
             lambdaConfig: {
                 lambdaFunctionArn: LAMBDA_ARN,
@@ -181,10 +185,11 @@ class AppSyncNeptuneStack extends Stack {
 
 
         // AppSync: Function        
-        const functionappSync = new CfnFunctionConfiguration(this, NAME + 'Function', {
+       const functionName = `${sanitizedName}Function`;
+       const functionappSync = new CfnFunctionConfiguration(this, functionName, {
             apiId: itemsGraphQLApi.attrApiId,
-            dataSourceName: NAME + 'DataSource',
-            name: NAME + 'Function',
+            dataSourceName: dataSourceName,
+            name: functionName,
             runtime: {
                 name: "APPSYNC_JS",
                 runtimeVersion: "1.0.0",

test/TestCases/Case04/Case04.01.test.js

@@ -3,7 +3,7 @@ import { jest } from '@jest/globals';
 import { readJSONFile } from '../../testLib';
 import { main } from "../../../src/main";
 
-const casetest = readJSONFile('./test/TestCases/Case04/case.json');
+const casetest = readJSONFile('./test/TestCases/Case04/get-db-schema.json');
 
 async function executeUtility() {    
     process.argv = casetest.argv;

test/TestCases/Case04/Case04.02.test.js

@@ -2,13 +2,14 @@ import { readJSONFile, checkOutputFileContent, checkFolderContainsFiles } from '
 import { sortNeptuneSchema } from './util';
 import fs from "fs";
 import { parseNeptuneEndpoint } from "../../../src/util.js";
+import path from "path";
 
-const casetest = readJSONFile('./test/TestCases/Case04/case.json');
+const casetest = readJSONFile('./test/TestCases/Case04/get-db-schema.json');
 const testDbInfo = parseNeptuneEndpoint(casetest.host + ':' + casetest.port);
-const outputFolderPath = './test/TestCases/Case04/output';
+const outputFolderPath = './test/TestCases/Case04/get-db-schema-output';
 
-const schemaFile = `${testDbInfo.graphName}.output.neptune.schema.json`;
-const neptuneSchema = readJSONFile(`./test/TestCases/Case04/output/${schemaFile}`);
+const schemaFile = `${testDbInfo.graphName}.neptune.schema.json`;
+const neptuneSchema = readJSONFile(path.join(outputFolderPath, schemaFile));
 const refSchemaFile = `output.neptune.${testDbInfo.neptuneType.replace('neptune-', '')}.schema.json`;
 const refNeptuneSchema = readJSONFile(`./test/TestCases/Case04/outputReference/${refSchemaFile}`);
 
@@ -18,10 +19,10 @@ describe('Validate output content', () => {
     });
 
     checkFolderContainsFiles(outputFolderPath, [
-        `${testDbInfo.graphName}.output.resolver.graphql.js`,
-        `${testDbInfo.graphName}.output.resolver.schema.json.gz`,
-        `${testDbInfo.graphName}.output.schema.graphql`,
-        `${testDbInfo.graphName}.output.source.schema.graphql`
+        `${testDbInfo.graphName}.resolver.graphql.js`,
+        `${testDbInfo.graphName}.resolver.schema.json.gz`,
+        `${testDbInfo.graphName}.schema.graphql`,
+        `${testDbInfo.graphName}.source.schema.graphql`
     ]);
 
     // note that this test can be flaky depending on how the air routes sample data was loaded into neptune

test/TestCases/Case04/case.json test/TestCases/Case04/get-db-schema.jsontest/TestCases/Case04/case.json renamed to test/TestCases/Case04/get-db-schema.json

@@ -3,7 +3,7 @@
     "description":"",
     "argv":["--quiet",
             "--input-graphdb-schema-neptune-endpoint", "<AIR_ROUTES_DB_HOST>:<AIR_ROUTES_DB_PORT>",
-            "--output-folder-path", "./test/TestCases/Case04/output",
+            "--output-folder-path", "./test/TestCases/Case04/get-db-schema-output",
             "--output-no-lambda-zip"],
     "host": "<AIR_ROUTES_DB_HOST>",
     "port": "<AIR_ROUTES_DB_PORT>",