refactor: restructure webhook lambda (#3618)

Thie PR prepare for changes to easier migrate reading config from SSM
instead of environment (#3594), add option to only accept messages of a
defined IP list, and to introduce option to connect runners via
EventBridg.

- Validate input and throw validation exceptions if event cannot be
accepted
- Structure the code that to allow the webhook to be split in acceptiong
an event and distribute to a runner (prepare for EventBridge).
- Remove deprecated jest functions.
- THE PR minimized changed on thest, only small structural things. This
to ensure the test still validating the implemention

Author: npalm

lambdas/functions/webhook/jest.config.ts

@@ -6,10 +6,10 @@ const config: Config = {
   ...defaultConfig,
   coverageThreshold: {
     global: {
-      statements: 99.07,
-      branches: 93.33,
+      statements: 99.13,
+      branches: 96.87,
       functions: 100,
-      lines: 99.02,
+      lines: 99.09,
     },
   },
 };

lambdas/functions/webhook/src/ConfigResolver.ts

@@ -0,0 +1,15 @@
+import { QueueConfig } from './sqs';
+
+export class Config {
+  public repositoryAllowList: Array<string>;
+  public queuesConfig: Array<QueueConfig>;
+  public workflowJobEventSecondaryQueue;
+
+  constructor() {
+    const repositoryAllowListEnv = process.env.REPOSITORY_ALLOW_LIST || '[]';
+    this.repositoryAllowList = JSON.parse(repositoryAllowListEnv) as Array<string>;
+    const queuesConfigEnv = process.env.RUNNER_CONFIG || '[]';
+    this.queuesConfig = JSON.parse(queuesConfigEnv) as Array<QueueConfig>;
+    this.workflowJobEventSecondaryQueue = process.env.WORKFLOW_JOB_EVENT_SECONDARY_QUEUE || undefined;
+  }
+}

lambdas/functions/webhook/src/ValidatonError.ts

@@ -0,0 +1,13 @@
+class ValidationError extends Error {
+  constructor(
+    public statusCode: number,
+    public message: string,
+    public error?: Error,
+  ) {
+    super(message);
+    this.name = 'ValidationError';
+    this.stack = error ? error.stack : new Error().stack;
+  }
+}
+
+export default ValidationError;

lambdas/functions/webhook/src/lambda.test.ts

@@ -3,7 +3,8 @@ import { APIGatewayEvent, Context } from 'aws-lambda';
 import { mocked } from 'jest-mock';
 
 import { githubWebhook } from './lambda';
-import { handle } from './webhook/handler';
+import { handle } from './webhook';
+import ValidationError from './ValidatonError';
 
 const event: APIGatewayEvent = {
   body: JSON.stringify(''),
@@ -71,7 +72,7 @@ const context: Context = {
   },
 };
 
-jest.mock('./webhook/handler');
+jest.mock('./webhook');
 
 describe('Test scale up lambda wrapper.', () => {
   it('Happy flow, resolve.', async () => {
@@ -88,14 +89,10 @@ describe('Test scale up lambda wrapper.', () => {
 
   it('An expected error, resolve.', async () => {
     const mock = mocked(handle);
-    mock.mockImplementation(() => {
-      return new Promise((resolve) => {
-        resolve({ statusCode: 400 });
-      });
-    });
+    mock.mockRejectedValue(new ValidationError(400, 'some error'));
 
     const result = await githubWebhook(event, context);
-    expect(result).toEqual({ statusCode: 400 });
+    expect(result).toMatchObject({ statusCode: 400 });
   });
 
   it('Errors are not thrown.', async () => {

lambdas/functions/webhook/src/lambda.ts

@@ -1,28 +1,50 @@
 import middy from '@middy/core';
-import { logger, setContext } from '@terraform-aws-github-runner/aws-powertools-util';
-import { captureLambdaHandler, tracer } from '@terraform-aws-github-runner/aws-powertools-util';
+import { logger, setContext, captureLambdaHandler, tracer } from '@terraform-aws-github-runner/aws-powertools-util';
 import { APIGatewayEvent, Context } from 'aws-lambda';
 
-import { handle } from './webhook/handler';
+import { handle } from './webhook';
+import { Config } from './ConfigResolver';
+import { IncomingHttpHeaders } from 'http';
+import ValidationError from './ValidatonError';
 
 export interface Response {
   statusCode: number;
   body?: string;
 }
+
 middy(githubWebhook).use(captureLambdaHandler(tracer));
+
 export async function githubWebhook(event: APIGatewayEvent, context: Context): Promise<Response> {
   setContext(context, 'lambda.ts');
+  const config = new Config();
+
   logger.logEventIfEnabled(event);
+  logger.debug('Loading config', { config });
 
   let result: Response;
   try {
-    result = await handle(event.headers, event.body as string);
+    result = await handle(headersToLowerCase(event.headers), event.body as string, config);
   } catch (e) {
     logger.error(`Failed to handle webhook event`, { error: e });
-    result = {
-      statusCode: 500,
-      body: 'Check the Lambda logs for the error details.',
-    };
+    if (e instanceof ValidationError) {
+      result = {
+        statusCode: e.statusCode,
+        body: e.message,
+      };
+    } else {
+      result = {
+        statusCode: 500,
+        body: 'Check the Lambda logs for the error details.',
+      };
+    }
   }
   return result;
 }
+
+// ensure header keys lower case since github headers can contain capitals.
+function headersToLowerCase(headers: IncomingHttpHeaders): IncomingHttpHeaders {
+  for (const key in headers) {
+    headers[key.toLowerCase()] = headers[key];
+  }
+  return headers;
+}

lambdas/functions/webhook/src/local.ts

@@ -1,14 +1,16 @@
 import bodyParser from 'body-parser';
 import express from 'express';
 
-import { handle } from './webhook/handler';
+import { handle } from './webhook';
+import { Config } from './ConfigResolver';
 
 const app = express();
+const config = new Config();
 
 app.use(bodyParser.json());
 
 app.post('/event_handler', (req, res) => {
-  handle(req.headers, JSON.stringify(req.body))
+  handle(req.headers, JSON.stringify(req.body), config)
     .then((c) => res.status(c.statusCode).end())
     .catch((e) => {
       console.log(e);

lambdas/functions/webhook/src/sqs/index.test.ts

@@ -2,12 +2,11 @@ import { SendMessageCommandInput } from '@aws-sdk/client-sqs';
 
 import { ActionRequestMessage, GithubWorkflowEvent, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '.';
 import workflowjob_event from '../../test/resources/github_workflowjob_event.json';
+import { Config } from '../ConfigResolver';
 
 const mockSQS = {
   sendMessage: jest.fn(() => {
-    {
-      return {};
-    }
+    return {};
   }),
 };
 jest.mock('@aws-sdk/client-sqs', () => ({
@@ -66,6 +65,7 @@ describe('Test sending message to SQS.', () => {
     expect(result).resolves;
   });
 });
+
 describe('Test sending message to SQS.', () => {
   const message: GithubWorkflowEvent = {
     workflowJobEvent: JSON.parse(JSON.stringify(workflowjob_event)),
@@ -77,29 +77,36 @@ describe('Test sending message to SQS.', () => {
   afterEach(() => {
     jest.clearAllMocks();
   });
+
   it('sends webhook events to workflow job queue', async () => {
     // Arrange
-    process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl;
+    process.env.WORKFLOW_JOB_EVENT_SECONDARY_QUEUE = sqsMessage.QueueUrl;
+    const config = new Config();
 
     // Act
-    const result = await sendWebhookEventToWorkflowJobQueue(message);
+    const result = await sendWebhookEventToWorkflowJobQueue(message, config);
 
     // Assert
-    expect(mockSQS.sendMessage).toBeCalledWith(sqsMessage);
+    expect(mockSQS.sendMessage).toHaveBeenCalledWith(sqsMessage);
     expect(result).resolves;
   });
+
   it('Does not send webhook events to workflow job event copy queue', async () => {
     // Arrange
-    process.env.SQS_WORKFLOW_JOB_QUEUE = '';
+    process.env.WORKFLOW_JOB_EVENT_SECONDARY_QUEUE = '';
+    const config = new Config();
     // Act
-    await sendWebhookEventToWorkflowJobQueue(message);
+    await sendWebhookEventToWorkflowJobQueue(message, config);
 
     // Assert
-    expect(mockSQS.sendMessage).not.toBeCalledWith(sqsMessage);
+    expect(mockSQS.sendMessage).not.toHaveBeenCalledWith(sqsMessage);
   });
+
   it('Catch the exception when even copy queue throws exception', async () => {
     // Arrange
-    process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl;
+    process.env.WORKFLOW_JOB_EVENT_SECONDARY_QUEUE = sqsMessage.QueueUrl;
+    const config = new Config();
+
     const mockSQS = {
       sendMessage: jest.fn(() => {
         throw new Error();
@@ -108,7 +115,6 @@ describe('Test sending message to SQS.', () => {
     jest.mock('aws-sdk', () => ({
       SQS: jest.fn().mockImplementation(() => mockSQS),
     }));
-    await expect(mockSQS.sendMessage).toThrowError();
-    await expect(sendWebhookEventToWorkflowJobQueue(message)).resolves.not.toThrowError();
+    await expect(sendWebhookEventToWorkflowJobQueue(message, config)).resolves.not.toThrow();
   });
 });

lambdas/functions/webhook/src/sqs/index.ts

@@ -1,7 +1,7 @@
 import { SQS, SendMessageCommandInput } from '@aws-sdk/client-sqs';
 import { WorkflowJobEvent } from '@octokit/webhooks-types';
-import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
-import { getTracedAWSV3Client } from '@terraform-aws-github-runner/aws-powertools-util';
+import { createChildLogger, getTracedAWSV3Client } from '@terraform-aws-github-runner/aws-powertools-util';
+import { Config } from '../ConfigResolver';
 
 const logger = createChildLogger('sqs');
 
@@ -20,12 +20,15 @@ export interface MatcherConfig {
   exactMatch: boolean;
 }
 
+export type RunnerConfig = QueueConfig[];
+
 export interface QueueConfig {
   matcherConfig: MatcherConfig;
   id: string;
   arn: string;
   fifo: boolean;
 }
+
 export interface GithubWorkflowEvent {
   workflowJobEvent: WorkflowJobEvent;
 }
@@ -46,20 +49,18 @@ export const sendActionRequest = async (message: ActionRequestMessage): Promise<
   await sqs.sendMessage(sqsMessage);
 };
 
-export const sendWebhookEventToWorkflowJobQueue = async (message: GithubWorkflowEvent): Promise<void> => {
-  const webhook_events_workflow_job_queue = process.env.SQS_WORKFLOW_JOB_QUEUE || undefined;
-
-  if (webhook_events_workflow_job_queue != undefined) {
+export async function sendWebhookEventToWorkflowJobQueue(message: GithubWorkflowEvent, config: Config): Promise<void> {
+  if (config.workflowJobEventSecondaryQueue != undefined) {
     const sqs = new SQS({ region: process.env.AWS_REGION });
     const sqsMessage: SendMessageCommandInput = {
-      QueueUrl: String(process.env.SQS_WORKFLOW_JOB_QUEUE),
+      QueueUrl: String(config.workflowJobEventSecondaryQueue),
       MessageBody: JSON.stringify(message),
     };
-    logger.debug(`Sending Webhook events to the workflow job queue: ${webhook_events_workflow_job_queue}`);
+    logger.debug(`Sending Webhook events to the workflow job queue: ${config.workflowJobEventSecondaryQueue}`);
     try {
       await sqs.sendMessage(sqsMessage);
     } catch (e) {
       logger.warn(`Error in sending webhook events to workflow job queue: ${(e as Error).message}`);
     }
   }
-};
+}