[DOP-3638]: Transpile API handlers from TS to JS (#800)

* [DOP-3638]: Update tsconfig.json to include api directory and add shell script

* [DOP-3638]: Finish shell script to archive lambdas

* [DOP-3638]: Finish shell script to archive lambdas

* [DOP-3638]: Update typing and runtime validations for dochub

* [DOP-3638]: Update typing for PrepGithubPushPayload

* [DOP-3638]: Remove empty if statement

* [DOP-3638]: Resolve typescript errors

* [DOP-3638]: Remove unnecessary dependency

* [DOP-3638]: Add additional data validation and update more types

* [DOP-3638]: Add additional data validation and update more types

* [DOP-3638]: Additional refactor

* [DOP-3638]: Refactor error handling to be more consistent

* [DOP-3638]: Return object instead of throwing an error for the api handler

* [DOP-3638]: Correct error message

* [DOP-3638]: Correct error message

* [DOP-3638]: Move check for event.body

* [DOP-3638]: Return instead of throw when validating dochub env vars

* [DOP-3638]: Fix isNumber logic

* [DOP-3638]: remove overly complicated typeguard

* [DOP-3638]: Remove unused code

* [DOP-3638]: Remove unused connection

Author: branberry

api/controllers/v1/dochub.ts

@@ -1,17 +1,48 @@
 import { FastlyConnector } from '../../../src/services/cdn';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { CDNCreds } from '../../../src/entities/creds';
+import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
+
+export const UpsertEdgeDictionaryItem = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
+  if (!event.body) {
+    return {
+      statusCode: 400,
+      body: 'event.body is null',
+    };
+  }
 
-export const UpsertEdgeDictionaryItem = async (event: any = {}): Promise<any> => {
   const body = JSON.parse(event.body);
+
   const pair = {
     key: body.source,
     value: body.target,
   };
-  // TODO: Argument of type 'string | undefined' is not assignable to parameter of type 'string'.
-  // Above applies for all 'process.env' variables, they should be validated
-  const creds = new CDNCreds(process.env.FASTLY_DOCHUB_SERVICE_ID, process.env.FASTLY_DOCHUB_TOKEN);
-  await new FastlyConnector(new ConsoleLogger()).upsertEdgeDictionaryItem(pair, process.env.FASTLY_DOCHUB_MAP, creds);
+
+  const { FASTLY_DOCHUB_SERVICE_ID, FASTLY_DOCHUB_TOKEN, FASTLY_DOCHUB_MAP } = process.env;
+
+  if (!FASTLY_DOCHUB_SERVICE_ID) {
+    return {
+      statusCode: 500,
+      body: 'Fastly dochub service ID is not defined',
+    };
+  }
+
+  if (!FASTLY_DOCHUB_TOKEN) {
+    return {
+      statusCode: 500,
+      body: 'Fastly dochub token is not defined',
+    };
+  }
+
+  if (!FASTLY_DOCHUB_MAP) {
+    return {
+      statusCode: 500,
+      body: 'Fastly dochub map is not defined',
+    };
+  }
+
+  const creds = new CDNCreds(FASTLY_DOCHUB_SERVICE_ID, FASTLY_DOCHUB_TOKEN);
+  await new FastlyConnector(new ConsoleLogger()).upsertEdgeDictionaryItem(pair, FASTLY_DOCHUB_MAP, creds);
   return {
     statusCode: 202,
     headers: { 'Content-Type': 'text/plain' },

api/controllers/v1/github.ts

@@ -1,22 +1,35 @@
 import * as c from 'config';
 import * as crypto from 'crypto';
 import * as mongodb from 'mongodb';
+import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
+
 import { JobRepository } from '../../../src/repositories/jobRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { BranchRepository } from '../../../src/repositories/branchRepository';
+import { Job, JobStatus } from '../../../src/entities/job';
+import { PushEvent } from '@octokit/webhooks-types';
 
 // This function will validate your payload from GitHub
 // See docs at https://developer.github.com/webhooks/securing/#validating-payloads-from-github
-function validateJsonWebhook(request: any, secret: string) {
-  const expectedSignature = 'sha256=' + crypto.createHmac('sha256', secret).update(request.body).digest('hex');
+function validateJsonWebhook(request: APIGatewayEvent, secret: string) {
+  const expectedSignature =
+    'sha256=' +
+    crypto
+      .createHmac('sha256', secret)
+      .update(request.body ?? '')
+      .digest('hex');
   const signature = request.headers['X-Hub-Signature-256'];
   if (signature !== expectedSignature) {
     return false;
   }
   return true;
 }
 
-async function prepGithubPushPayload(githubEvent: any, branchRepository: BranchRepository, prefix: string) {
+async function prepGithubPushPayload(
+  githubEvent: PushEvent,
+  branchRepository: BranchRepository,
+  prefix: string
+): Promise<Omit<Job, '_id'>> {
   const branch_name = githubEvent.ref.split('/')[2];
   const branch_info = await branchRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
@@ -27,7 +40,7 @@ async function prepGithubPushPayload(githubEvent: any, branchRepository: BranchR
     title: githubEvent.repository.full_name,
     user: githubEvent.pusher.name,
     email: githubEvent.pusher.email,
-    status: 'inQueue',
+    status: JobStatus.inQueue,
     createdTime: new Date(),
     startTime: null,
     endTime: null,
@@ -42,7 +55,6 @@ async function prepGithubPushPayload(githubEvent: any, branchRepository: BranchR
       branchName: githubEvent.ref.split('/')[2],
       isFork: githubEvent.repository.fork,
       private: githubEvent.repository.private,
-      isXlarge: true,
       repoOwner: githubEvent.repository.owner.login,
       url: githubEvent.repository.clone_url,
       newHead: githubEvent.after,
@@ -54,14 +66,24 @@ async function prepGithubPushPayload(githubEvent: any, branchRepository: BranchR
   };
 }
 
-export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<any> => {
+export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
   const branchRepository = new BranchRepository(db, c, consoleLogger);
-  const sig = event.headers['X-Hub-Signature-256'];
+
+  if (!event.body) {
+    const err = 'Trigger build does not have a body in event payload';
+    consoleLogger.error('TriggerBuildError', err);
+    return {
+      statusCode: 400,
+      headers: { 'Content-Type': 'text/plain' },
+      body: err,
+    };
+  }
+
   if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
     const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
     return {
@@ -70,6 +92,7 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
       body: errMsg,
     };
   }
+
   const body = JSON.parse(event.body);
 
   if (body.deleted) {
@@ -83,8 +106,9 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
   const env = c.get<string>('env');
   const repoInfo = await branchRepository.getRepo(body.repository.name);
   const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
-  // TODO: Make job be of type Job
+
   const job = await prepGithubPushPayload(body, branchRepository, jobPrefix);
+
   try {
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

api/controllers/v1/jobs.ts

@@ -11,13 +11,22 @@ import { Job, JobStatus } from '../../../src/entities/job';
 import { ECSContainer } from '../../../src/services/containerServices';
 import { SQSConnector } from '../../../src/services/queue';
 import { Batch } from '../../../src/services/batch';
+import { APIGatewayEvent, APIGatewayProxyResult, SQSEvent, SQSRecord } from 'aws-lambda';
 
-export const TriggerLocalBuild = async (event: any = {}, context: any = {}): Promise<any> => {
-  const client = new mongodb.MongoClient(c.get('dbUrl'));
-  await client.connect();
-  const db = client.db(c.get('dbName'));
+export const TriggerLocalBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
   const consoleLogger = new ConsoleLogger();
   const sqs = new SQSConnector(consoleLogger, c);
+
+  if (!event.body) {
+    const err = 'Trigger local build does not have a body in event payload';
+    consoleLogger.error('TriggerLocalBuildError', err);
+    return {
+      statusCode: 400,
+      headers: { 'Content-Type': 'text/plain' },
+      body: err,
+    };
+  }
+
   const body = JSON.parse(event.body);
   try {
     consoleLogger.info(body.jobId, 'enqueuing Job');
@@ -38,26 +47,23 @@ export const TriggerLocalBuild = async (event: any = {}, context: any = {}): Pro
   }
 };
 
-// TODO: use @types/aws-lambda
-export const HandleJobs = async (event: any = {}): Promise<any> => {
+export const HandleJobs = async (event: SQSEvent): Promise<void> => {
   /**
    * Check the status of the incoming jobs
    * if it is inqueue start a task
    * if it is inprogress call NotifyBuildProgress
    * if it is completed call NotifyBuildSummary
    */
-  const messages: JobQueueMessage[] = event.Records;
+  const messages = event.Records;
   await Promise.all(
-    messages.map(async (message: any) => {
+    messages.map(async (message: SQSRecord) => {
       const consoleLogger = new ConsoleLogger();
       const body = JSON.parse(message.body);
-      let queueUrl = '';
       const jobId = body['jobId'];
       const jobStatus = body['jobStatus'];
       try {
         switch (jobStatus) {
           case JobStatus[JobStatus.inQueue]:
-            queueUrl = c.get('jobsQueueUrl');
             await NotifyBuildProgress(jobId);
             // start the task , don't start the process before processing the notification
             const ecsServices = new ECSContainer(c, consoleLogger);
@@ -68,7 +74,6 @@ export const HandleJobs = async (event: any = {}): Promise<any> => {
             consoleLogger.info(jobId, JSON.stringify(res));
             break;
           case JobStatus[JobStatus.inProgress]:
-            queueUrl = c.get('jobUpdatesQueueUrl');
             await NotifyBuildProgress(jobId);
             break;
           case JobStatus[JobStatus.timedOut]:
@@ -80,9 +85,7 @@ export const HandleJobs = async (event: any = {}): Promise<any> => {
             break;
           case JobStatus[JobStatus.failed]:
           case JobStatus[JobStatus.completed]:
-            queueUrl = c.get('jobUpdatesQueueUrl');
-            await NotifyBuildSummary(jobId);
-            await SubmitArchiveJob(jobId);
+            await Promise.all([NotifyBuildSummary(jobId), SubmitArchiveJob(jobId)]);
             break;
           default:
             consoleLogger.error(jobId, 'Invalid status');
@@ -134,42 +137,35 @@ async function stopECSTask(taskId: string, consoleLogger: ConsoleLogger) {
   await ecs.stopZombieECSTask(taskId);
 }
 
-async function retry(message: JobQueueMessage, consoleLogger: ConsoleLogger, url: string): Promise<any> {
-  try {
-    const tries = message.tries;
-    // TODO: c.get('maxRetries') is of type 'Unknown', needs validation
-    if (tries < c.get('maxRetries')) {
-      const sqs = new SQSConnector(consoleLogger, c);
-      message['tries'] += 1;
-      let retryDelay = 10;
-      if (c.get('retryDelay')) {
-        retryDelay = c.get('retryDelay');
-      }
-      await sqs.sendMessage(message, url, retryDelay * tries);
-    }
-  } catch (err) {
-    consoleLogger.error(message['jobId'], err);
-  }
-}
-async function NotifyBuildSummary(jobId: string): Promise<any> {
+async function NotifyBuildSummary(jobId: string): Promise<void> {
   const consoleLogger = new ConsoleLogger();
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const env = c.get<string>('env');
 
   const jobRepository = new JobRepository(db, c, consoleLogger);
-  // TODO: Make fullDocument be of type Job, validate existence
   const fullDocument = await jobRepository.getJobById(jobId);
+
+  if (!fullDocument) {
+    consoleLogger.error(
+      `NotifyBuildSummary_${jobId}`,
+      `Notify build summary failed. Job does not exist for Job ID: ${jobId}`
+    );
+    return;
+  }
+
   const repoName = fullDocument.payload.repoName;
   const username = fullDocument.user;
   const slackConnector = new SlackConnector(consoleLogger, c);
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
   const entitlement = await repoEntitlementRepository.getSlackUserIdByGithubUsername(username);
+
   if (!entitlement?.['slack_user_id']) {
     consoleLogger.error(username, 'Entitlement failed');
     return;
   }
+
   await slackConnector.sendMessage(
     await prepSummaryMessage(
       env,
@@ -181,9 +177,6 @@ async function NotifyBuildSummary(jobId: string): Promise<any> {
     ),
     entitlement['slack_user_id']
   );
-  return {
-    statusCode: 200,
-  };
 }
 
 export const extractUrlFromMessage = (fullDocument): string[] => {
@@ -201,12 +194,11 @@ async function prepSummaryMessage(
   failed = false
 ): Promise<string> {
   const urls = extractUrlFromMessage(fullDocument);
-  let mms_urls = [null, null];
+  let mms_urls: Array<string | null> = [null, null];
   // mms-docs needs special handling as it builds two sites (cloudmanager & ops manager)
   // so we need to extract both URLs
   if (repoName === 'mms-docs') {
     if (urls.length >= 2) {
-      // TODO: Type 'string[]' is not assignable to type 'null[]'.
       mms_urls = urls.slice(-2);
     }
   }
@@ -257,15 +249,24 @@ function prepProgressMessage(
   }
 }
 
-async function NotifyBuildProgress(jobId: string): Promise<any> {
+async function NotifyBuildProgress(jobId: string): Promise<void> {
   const consoleLogger = new ConsoleLogger();
   const client = new mongodb.MongoClient(c.get('dbUrl'));
   await client.connect();
   const db = client.db(c.get('dbName'));
   const slackConnector = new SlackConnector(consoleLogger, c);
   const jobRepository = new JobRepository(db, c, consoleLogger);
-  // TODO: Make fullDocument be of type Job, validate existence
+
   const fullDocument = await jobRepository.getJobById(jobId);
+
+  if (!fullDocument) {
+    consoleLogger.error(
+      `NotifyBuildProgress_${jobId}`,
+      `Notify build progress failed. Job does not exist for Job ID: ${jobId}`
+    );
+    return;
+  }
+
   const jobTitle = fullDocument.title;
   const username = fullDocument.user;
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
@@ -274,7 +275,8 @@ async function NotifyBuildProgress(jobId: string): Promise<any> {
     consoleLogger.error(username, 'Entitlement Failed');
     return;
   }
-  const resp = await slackConnector.sendMessage(
+
+  await slackConnector.sendMessage(
     prepProgressMessage(
       c.get('dashboardUrl'),
       jobId,
@@ -284,9 +286,6 @@ async function NotifyBuildProgress(jobId: string): Promise<any> {
     ),
     entitlement['slack_user_id']
   );
-  return {
-    statusCode: 200,
-  };
 }
 
 function getMongoClient(config: IConfig): mongodb.MongoClient {
@@ -317,6 +316,15 @@ async function SubmitArchiveJob(jobId: string) {
     branches: new BranchRepository(db, c, consoleLogger),
   };
   const job = await models.jobs.getJobById(jobId);
+
+  if (!job) {
+    consoleLogger.error(
+      `SubmitArchiveJob_${jobId}`,
+      `Submit archive job failed. Job does not exist for Job ID: ${jobId}`
+    );
+    return;
+  }
+
   const repo = await models.branches.getRepo(job.payload.repoName);
 
   /* NOTE

api/controllers/v1/slack.ts

@@ -245,7 +245,6 @@ function createPayload(
     urlSlug,
     isFork: true,
     private: repoOwner === '10gen',
-    isXlarge: true,
     repoOwner,
     url: 'https://github.com/' + repoOwner + '/' + repoName,
     newHead,