[fix] Update to node 18 in straggling references (#815)

* [fix] Update to node 18 in straggling references

* Revert "[DOP-3638]: Transpile API handlers from TS to JS (#800)"

This reverts commit 08bb923.

* [fix] Prune lambdas before deployment

* fixup

* fixup

* Update deploy-stg-ecs.yml

* 1:1 prd and stg deploy action

Author: casthewiz

.github/workflows/deploy-prd-ecs.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - name: Install Serverless Framework
         run: npm install -g serverless
       - name: Serverless AWS authentication
@@ -68,3 +68,4 @@ jobs:
           npm ci
           sls deploy --stage prd
           sls deploy --stage dotcomprd
+          sls prune -n 5

.github/workflows/deploy-stg-ecs.yml

@@ -66,6 +66,7 @@ jobs:
           sls deploy --stage dotcomstg
       - name: Deploy Lambdas	
         run: |
-          npm ci          
+          npm ci         
           sls deploy --stage stg
-          sls deploy --stage dotcomstg
+          sls deploy --stage dotcomstg 
+          sls prune -n 5

api/controllers/v1/dochub.ts

@@ -1,48 +1,17 @@
 import { FastlyConnector } from '../../../src/services/cdn';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { CDNCreds } from '../../../src/entities/creds';
-import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
-
-export const UpsertEdgeDictionaryItem = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
-  if (!event.body) {
-    return {
-      statusCode: 400,
-      body: 'event.body is null',
-    };
-  }
 
+export const UpsertEdgeDictionaryItem = async (event: any = {}): Promise<any> => {
   const body = JSON.parse(event.body);
-
   const pair = {
     key: body.source,
     value: body.target,
   };
-
-  const { FASTLY_DOCHUB_SERVICE_ID, FASTLY_DOCHUB_TOKEN, FASTLY_DOCHUB_MAP } = process.env;
-
-  if (!FASTLY_DOCHUB_SERVICE_ID) {
-    return {
-      statusCode: 500,
-      body: 'Fastly dochub service ID is not defined',
-    };
-  }
-
-  if (!FASTLY_DOCHUB_TOKEN) {
-    return {
-      statusCode: 500,
-      body: 'Fastly dochub token is not defined',
-    };
-  }
-
-  if (!FASTLY_DOCHUB_MAP) {
-    return {
-      statusCode: 500,
-      body: 'Fastly dochub map is not defined',
-    };
-  }
-
-  const creds = new CDNCreds(FASTLY_DOCHUB_SERVICE_ID, FASTLY_DOCHUB_TOKEN);
-  await new FastlyConnector(new ConsoleLogger()).upsertEdgeDictionaryItem(pair, FASTLY_DOCHUB_MAP, creds);
+  // TODO: Argument of type 'string | undefined' is not assignable to parameter of type 'string'.
+  // Above applies for all 'process.env' variables, they should be validated
+  const creds = new CDNCreds(process.env.FASTLY_DOCHUB_SERVICE_ID, process.env.FASTLY_DOCHUB_TOKEN);
+  await new FastlyConnector(new ConsoleLogger()).upsertEdgeDictionaryItem(pair, process.env.FASTLY_DOCHUB_MAP, creds);
   return {
     statusCode: 202,
     headers: { 'Content-Type': 'text/plain' },

api/controllers/v1/github.ts

@@ -1,35 +1,22 @@
 import * as c from 'config';
 import * as crypto from 'crypto';
 import * as mongodb from 'mongodb';
-import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
-
 import { JobRepository } from '../../../src/repositories/jobRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { BranchRepository } from '../../../src/repositories/branchRepository';
-import { Job, JobStatus } from '../../../src/entities/job';
-import { PushEvent } from '@octokit/webhooks-types';
 
 // This function will validate your payload from GitHub
 // See docs at https://developer.github.com/webhooks/securing/#validating-payloads-from-github
-function validateJsonWebhook(request: APIGatewayEvent, secret: string) {
-  const expectedSignature =
-    'sha256=' +
-    crypto
-      .createHmac('sha256', secret)
-      .update(request.body ?? '')
-      .digest('hex');
+function validateJsonWebhook(request: any, secret: string) {
+  const expectedSignature = 'sha256=' + crypto.createHmac('sha256', secret).update(request.body).digest('hex');
   const signature = request.headers['X-Hub-Signature-256'];
   if (signature !== expectedSignature) {
     return false;
   }
   return true;
 }
 
-async function prepGithubPushPayload(
-  githubEvent: PushEvent,
-  branchRepository: BranchRepository,
-  prefix: string
-): Promise<Omit<Job, '_id'>> {
+async function prepGithubPushPayload(githubEvent: any, branchRepository: BranchRepository, prefix: string) {
   const branch_name = githubEvent.ref.split('/')[2];
   const branch_info = await branchRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
@@ -40,7 +27,7 @@ async function prepGithubPushPayload(
     title: githubEvent.repository.full_name,
     user: githubEvent.pusher.name,
     email: githubEvent.pusher.email,
-    status: JobStatus.inQueue,
+    status: 'inQueue',
     createdTime: new Date(),
     startTime: null,
     endTime: null,
@@ -55,6 +42,7 @@ async function prepGithubPushPayload(
       branchName: githubEvent.ref.split('/')[2],
       isFork: githubEvent.repository.fork,
       private: githubEvent.repository.private,
+      isXlarge: true,
       repoOwner: githubEvent.repository.owner.login,
       url: githubEvent.repository.clone_url,
       newHead: githubEvent.after,
@@ -66,24 +54,14 @@ async function prepGithubPushPayload(
   };
 }
 
-export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
+export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<any> => {
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
   const branchRepository = new BranchRepository(db, c, consoleLogger);
-
-  if (!event.body) {
-    const err = 'Trigger build does not have a body in event payload';
-    consoleLogger.error('TriggerBuildError', err);
-    return {
-      statusCode: 400,
-      headers: { 'Content-Type': 'text/plain' },
-      body: err,
-    };
-  }
-
+  const sig = event.headers['X-Hub-Signature-256'];
   if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
     const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
     return {
@@ -92,7 +70,6 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
       body: errMsg,
     };
   }
-
   const body = JSON.parse(event.body);
 
   if (body.deleted) {
@@ -106,9 +83,8 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   const env = c.get<string>('env');
   const repoInfo = await branchRepository.getRepo(body.repository.name);
   const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
-
+  // TODO: Make job be of type Job
   const job = await prepGithubPushPayload(body, branchRepository, jobPrefix);
-
   try {
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

api/controllers/v1/jobs.ts

@@ -11,22 +11,13 @@ import { Job, JobStatus } from '../../../src/entities/job';
 import { ECSContainer } from '../../../src/services/containerServices';
 import { SQSConnector } from '../../../src/services/queue';
 import { Batch } from '../../../src/services/batch';
-import { APIGatewayEvent, APIGatewayProxyResult, SQSEvent, SQSRecord } from 'aws-lambda';
 
-export const TriggerLocalBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
+export const TriggerLocalBuild = async (event: any = {}, context: any = {}): Promise<any> => {
+  const client = new mongodb.MongoClient(c.get('dbUrl'));
+  await client.connect();
+  const db = client.db(c.get('dbName'));
   const consoleLogger = new ConsoleLogger();
   const sqs = new SQSConnector(consoleLogger, c);
-
-  if (!event.body) {
-    const err = 'Trigger local build does not have a body in event payload';
-    consoleLogger.error('TriggerLocalBuildError', err);
-    return {
-      statusCode: 400,
-      headers: { 'Content-Type': 'text/plain' },
-      body: err,
-    };
-  }
-
   const body = JSON.parse(event.body);
   try {
     consoleLogger.info(body.jobId, 'enqueuing Job');
@@ -47,23 +38,26 @@ export const TriggerLocalBuild = async (event: APIGatewayEvent): Promise<APIGate
   }
 };
 
-export const HandleJobs = async (event: SQSEvent): Promise<void> => {
+// TODO: use @types/aws-lambda
+export const HandleJobs = async (event: any = {}): Promise<any> => {
   /**
    * Check the status of the incoming jobs
    * if it is inqueue start a task
    * if it is inprogress call NotifyBuildProgress
    * if it is completed call NotifyBuildSummary
    */
-  const messages = event.Records;
+  const messages: JobQueueMessage[] = event.Records;
   await Promise.all(
-    messages.map(async (message: SQSRecord) => {
+    messages.map(async (message: any) => {
       const consoleLogger = new ConsoleLogger();
       const body = JSON.parse(message.body);
+      let queueUrl = '';
       const jobId = body['jobId'];
       const jobStatus = body['jobStatus'];
       try {
         switch (jobStatus) {
           case JobStatus[JobStatus.inQueue]:
+            queueUrl = c.get('jobsQueueUrl');
             await NotifyBuildProgress(jobId);
             // start the task , don't start the process before processing the notification
             const ecsServices = new ECSContainer(c, consoleLogger);
@@ -74,6 +68,7 @@ export const HandleJobs = async (event: SQSEvent): Promise<void> => {
             consoleLogger.info(jobId, JSON.stringify(res));
             break;
           case JobStatus[JobStatus.inProgress]:
+            queueUrl = c.get('jobUpdatesQueueUrl');
             await NotifyBuildProgress(jobId);
             break;
           case JobStatus[JobStatus.timedOut]:
@@ -85,7 +80,9 @@ export const HandleJobs = async (event: SQSEvent): Promise<void> => {
             break;
           case JobStatus[JobStatus.failed]:
           case JobStatus[JobStatus.completed]:
-            await Promise.all([NotifyBuildSummary(jobId), SubmitArchiveJob(jobId)]);
+            queueUrl = c.get('jobUpdatesQueueUrl');
+            await NotifyBuildSummary(jobId);
+            await SubmitArchiveJob(jobId);
             break;
           default:
             consoleLogger.error(jobId, 'Invalid status');
@@ -137,35 +134,42 @@ async function stopECSTask(taskId: string, consoleLogger: ConsoleLogger) {
   await ecs.stopZombieECSTask(taskId);
 }
 
-async function NotifyBuildSummary(jobId: string): Promise<void> {
+async function retry(message: JobQueueMessage, consoleLogger: ConsoleLogger, url: string): Promise<any> {
+  try {
+    const tries = message.tries;
+    // TODO: c.get('maxRetries') is of type 'Unknown', needs validation
+    if (tries < c.get('maxRetries')) {
+      const sqs = new SQSConnector(consoleLogger, c);
+      message['tries'] += 1;
+      let retryDelay = 10;
+      if (c.get('retryDelay')) {
+        retryDelay = c.get('retryDelay');
+      }
+      await sqs.sendMessage(message, url, retryDelay * tries);
+    }
+  } catch (err) {
+    consoleLogger.error(message['jobId'], err);
+  }
+}
+async function NotifyBuildSummary(jobId: string): Promise<any> {
   const consoleLogger = new ConsoleLogger();
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const env = c.get<string>('env');
 
   const jobRepository = new JobRepository(db, c, consoleLogger);
+  // TODO: Make fullDocument be of type Job, validate existence
   const fullDocument = await jobRepository.getJobById(jobId);
-
-  if (!fullDocument) {
-    consoleLogger.error(
-      `NotifyBuildSummary_${jobId}`,
-      `Notify build summary failed. Job does not exist for Job ID: ${jobId}`
-    );
-    return;
-  }
-
   const repoName = fullDocument.payload.repoName;
   const username = fullDocument.user;
   const slackConnector = new SlackConnector(consoleLogger, c);
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
   const entitlement = await repoEntitlementRepository.getSlackUserIdByGithubUsername(username);
-
   if (!entitlement?.['slack_user_id']) {
     consoleLogger.error(username, 'Entitlement failed');
     return;
   }
-
   await slackConnector.sendMessage(
     await prepSummaryMessage(
       env,
@@ -177,6 +181,9 @@ async function NotifyBuildSummary(jobId: string): Promise<void> {
     ),
     entitlement['slack_user_id']
   );
+  return {
+    statusCode: 200,
+  };
 }
 
 export const extractUrlFromMessage = (fullDocument): string[] => {
@@ -194,11 +201,12 @@ async function prepSummaryMessage(
   failed = false
 ): Promise<string> {
   const urls = extractUrlFromMessage(fullDocument);
-  let mms_urls: Array<string | null> = [null, null];
+  let mms_urls = [null, null];
   // mms-docs needs special handling as it builds two sites (cloudmanager & ops manager)
   // so we need to extract both URLs
   if (repoName === 'mms-docs') {
     if (urls.length >= 2) {
+      // TODO: Type 'string[]' is not assignable to type 'null[]'.
       mms_urls = urls.slice(-2);
     }
   }
@@ -249,24 +257,15 @@ function prepProgressMessage(
   }
 }
 
-async function NotifyBuildProgress(jobId: string): Promise<void> {
+async function NotifyBuildProgress(jobId: string): Promise<any> {
   const consoleLogger = new ConsoleLogger();
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const slackConnector = new SlackConnector(consoleLogger, c);
   const jobRepository = new JobRepository(db, c, consoleLogger);
-
+  // TODO: Make fullDocument be of type Job, validate existence
   const fullDocument = await jobRepository.getJobById(jobId);
-
-  if (!fullDocument) {
-    consoleLogger.error(
-      `NotifyBuildProgress_${jobId}`,
-      `Notify build progress failed. Job does not exist for Job ID: ${jobId}`
-    );
-    return;
-  }
-
   const jobTitle = fullDocument.title;
   const username = fullDocument.user;
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
@@ -275,8 +274,7 @@ async function NotifyBuildProgress(jobId: string): Promise<void> {
     consoleLogger.error(username, 'Entitlement Failed');
     return;
   }
-
-  await slackConnector.sendMessage(
+  const resp = await slackConnector.sendMessage(
     prepProgressMessage(
       c.get('dashboardUrl'),
       jobId,
@@ -286,6 +284,9 @@ async function NotifyBuildProgress(jobId: string): Promise<void> {
     ),
     entitlement['slack_user_id']
   );
+  return {
+    statusCode: 200,
+  };
 }
 
 function getMongoClient(config: IConfig): mongodb.MongoClient {
@@ -316,15 +317,6 @@ async function SubmitArchiveJob(jobId: string) {
     branches: new BranchRepository(db, c, consoleLogger),
   };
   const job = await models.jobs.getJobById(jobId);
-
-  if (!job) {
-    consoleLogger.error(
-      `SubmitArchiveJob_${jobId}`,
-      `Submit archive job failed. Job does not exist for Job ID: ${jobId}`
-    );
-    return;
-  }
-
   const repo = await models.branches.getRepo(job.payload.repoName);
 
   /* NOTE