feat: remove depcrecated bata feature workflow job queue

Author: npalm

docs/configuration.md

@@ -332,23 +332,34 @@ resource "aws_iam_role_policy" "event_rule_firehose_role" {
 
 ### Queue to publish workflow job events
 
-!!! warning "Deprecated
+!!! warning "Removed
 
-    This fearure will be removed since we introducing the EventBridge. Same functinallity can be implemented by adding a rule to the EventBridge to forward `workflow_job` events to the SQS queue.
+    This feaTure will be removed since we introducing the EventBridge. Same functionality can be implemented by adding a rule to the EventBridge to forward `workflow_job` events to the SQS queue.
 
-This queue is an experimental feature to allow you to receive a copy of the wokflow_jobs events sent by the GitHub App. This can be used to calculate a matrix or monitor the system.
+Below an example how you can sent all `workflow_job` with action `in_progress` to a SQS queue.
 
-To enable the feature set `enable_workflow_job_events_queue = true`. Be aware though, this feature is experimental!
+```hcl
 
-Messages received on the queue are using the same format as published by GitHub wrapped in a property `workflowJobEvent`.
+resource "aws_cloudwatch_event_rule" "workflow_job_in_progress" {
+  name           = "workflow-job-in-progress"
+  event_bus_name = modules.runners.webhook.eventbridge.name # The name of the event bus output by the module
 
-```
-export interface GithubWorkflowEvent {
-  workflowJobEvent: WorkflowJobEvent;
+  event_pattern = <<EOF
+{
+  "detail-type": ["workflow_job"],
+  "detail": {
+    "action": ["in_progress"]
+  }
+}
+EOF
+}
+
+resource "aws_sqs_queue" "workflow_job_in_progress" {
+  name = "workflow_job_in_progress
 }
+
 ```
 
-This extensible format allows more fields to be added if needed.
-You can configure the queue by setting properties to `workflow_job_events_queue_config`
+
 
 NOTE: By default, a runner AMI update requires a re-apply of this terraform config (the runner AMI ID is looked up by a terraform data source). To avoid this, you can use `ami_id_ssm_parameter_name` to have the scale-up lambda dynamically lookup the runner AMI ID from an SSM parameter at instance launch time. Said SSM parameter is managed outside of this module (e.g. by a runner AMI build workflow).

examples/default/main.tf

@@ -89,8 +89,6 @@ module "runners" {
 
   # override scaling down
   scale_down_schedule_expression = "cron(* * * * ? *)"
-  # enable this flag to publish webhook events to workflow job queue
-  # enable_workflow_job_events_queue  = true
 
   enable_user_data_debug_logging_runner = true
 

examples/multi-runner/main.tf

@@ -97,9 +97,6 @@ module "runners" {
   # runner_binaries_syncer_lambda_zip = "../lambdas-download/runner-binaries-syncer.zip"
   # runners_lambda_zip                = "../lambdas-download/runners.zip"
 
-  # enable_workflow_job_events_queue = true
-  # override delay of events in seconds
-
   # Enable debug logging for the lambda functions
   # log_level = "debug"
 

lambdas/functions/webhook/jest.config.ts

@@ -6,10 +6,10 @@ const config: Config = {
   ...defaultConfig,
   coverageThreshold: {
     global: {
-      statements: 99.58,
+      statements: 100,
       branches: 100,
       functions: 100,
-      lines: 99.57,
+      lines: 100,
     },
   },
 };

lambdas/functions/webhook/src/ConfigLoader.test.ts

@@ -94,9 +94,8 @@ describe('ConfigLoader Tests', () => {
   describe('ConfigWebhook', () => {
     it('should load config successfully', async () => {
       process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]';
-      process.env.SQS_WORKFLOW_JOB_QUEUE = 'secondary-queue';
-      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
       process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
       const matcherConfig = [
         {
           id: '1',
@@ -121,7 +120,6 @@ describe('ConfigLoader Tests', () => {
       const config: ConfigWebhook = await ConfigWebhook.load();
 
       expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']);
-      expect(config.workflowJobEventSecondaryQueue).toBe('secondary-queue');
       expect(config.matcherConfig).toEqual(matcherConfig);
       expect(config.webhookSecret).toBe('secret');
     });

lambdas/functions/webhook/src/ConfigLoader.ts

@@ -95,7 +95,6 @@ export class ConfigWebhook extends BaseConfig {
 
   async loadConfig(): Promise<void> {
     this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []);
-    this.loadEnvVar(process.env.SQS_WORKFLOW_JOB_QUEUE, 'workflowJobEventSecondaryQueue', '');
 
     await Promise.all([
       this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'),
@@ -129,7 +128,6 @@ export class ConfigDispatcher extends BaseConfig {
 
   async loadConfig(): Promise<void> {
     this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []);
-    this.loadEnvVar(process.env.SQS_WORKFLOW_JOB_QUEUE, 'workflowJobEventSecondaryQueue', '');
     await this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig');
 
     validateRunnerMatcherConfig(this);

lambdas/functions/webhook/src/modules.d.ts

@@ -7,6 +7,5 @@ declare namespace NodeJS {
     REPOSITORY_ALLOW_LIST: string;
     RUNNER_LABELS: string;
     ACCEPT_EVENTS: string;
-    SQS_WORKFLOW_JOB_QUEUE: string;
   }
 }

lambdas/functions/webhook/src/runners/dispatch.test.ts

@@ -14,7 +14,6 @@ import { logger } from '@aws-github-runner/aws-powertools-util';
 jest.mock('../sqs');
 jest.mock('@aws-github-runner/aws-ssm-util');
 
-const sendWebhookEventToWorkflowJobQueueMock = jest.mocked(sendActionRequest);
 const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET';
 
 const cleanEnv = process.env;
@@ -56,7 +55,6 @@ describe('Dispatcher', () => {
         statusCode: 403,
       });
       expect(sendActionRequest).not.toHaveBeenCalled();
-      expect(sendWebhookEventToWorkflowJobQueueMock).not.toHaveBeenCalled();
     });
 
     it('should handle workflow_job events without installation id', async () => {
@@ -65,7 +63,6 @@ describe('Dispatcher', () => {
       const resp = await dispatch(event, 'workflow_job', config);
       expect(resp.statusCode).toBe(201);
       expect(sendActionRequest).toHaveBeenCalled();
-      expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled();
     });
 
     it('should handle workflow_job events from allow listed repositories', async () => {
@@ -74,7 +71,6 @@ describe('Dispatcher', () => {
       const resp = await dispatch(event, 'workflow_job', config);
       expect(resp.statusCode).toBe(201);
       expect(sendActionRequest).toHaveBeenCalled();
-      expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled();
     });
 
     it('should match labels', async () => {
@@ -108,7 +104,6 @@ describe('Dispatcher', () => {
         queueFifo: false,
         repoOwnerType: 'Organization',
       });
-      expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled();
     });
 
     it('should sort matcher with exact first.', async () => {
@@ -157,7 +152,6 @@ describe('Dispatcher', () => {
         queueFifo: false,
         repoOwnerType: 'Organization',
       });
-      expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled();
     });
 
     it('should not accept jobs where not all labels are supported (single matcher).', async () => {
@@ -181,7 +175,6 @@ describe('Dispatcher', () => {
       const resp = await dispatch(event, 'workflow_job', config);
       expect(resp.statusCode).toBe(202);
       expect(sendActionRequest).not.toHaveBeenCalled();
-      expect(sendWebhookEventToWorkflowJobQueueMock).not.toHaveBeenCalled();
     });
   });
 

lambdas/functions/webhook/src/runners/dispatch.ts

@@ -2,7 +2,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
 import { WorkflowJobEvent } from '@octokit/webhooks-types';
 
 import { Response } from '../lambda';
-import { RunnerMatcherConfig, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs';
+import { RunnerMatcherConfig, sendActionRequest } from '../sqs';
 import ValidationError from '../ValidationError';
 import { ConfigDispatcher, ConfigWebhook } from '../ConfigLoader';
 
@@ -15,10 +15,7 @@ export async function dispatch(
 ): Promise<Response> {
   validateRepoInAllowList(event, config);
 
-  const result = await handleWorkflowJob(event, eventType, config.matcherConfig!);
-  await sendWebhookEventToWorkflowJobQueue({ workflowJobEvent: event }, config);
-
-  return result;
+  return await handleWorkflowJob(event, eventType, config.matcherConfig!);
 }
 
 function validateRepoInAllowList(event: WorkflowJobEvent, config: ConfigDispatcher) {

lambdas/functions/webhook/src/sqs/index.test.ts

@@ -1,9 +1,5 @@
 import { SendMessageCommandInput } from '@aws-sdk/client-sqs';
-
-import { ActionRequestMessage, GithubWorkflowEvent, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '.';
-import workflowjob_event from '../../test/resources/github_workflowjob_event.json';
-import { getParameter } from '@aws-github-runner/aws-ssm-util';
-import { mocked } from 'jest-mock';
+import { ActionRequestMessage, sendActionRequest } from '.';
 
 const mockSQS = {
   sendMessage: jest.fn(() => {
@@ -15,9 +11,6 @@ jest.mock('@aws-sdk/client-sqs', () => ({
 }));
 jest.mock('@aws-github-runner/aws-ssm-util');
 
-import { SQS } from '@aws-sdk/client-sqs';
-import { ConfigDispatcher, ConfigWebhook } from '../ConfigLoader';
-
 describe('Test sending message to SQS.', () => {
   const queueUrl = 'https://sqs.eu-west-1.amazonaws.com/123456789/queued-builds';
   const message = {
@@ -72,62 +65,3 @@ describe('Test sending message to SQS.', () => {
     await expect(result).resolves.not.toThrow();
   });
 });
-
-describe('Test sending message to SQS.', () => {
-  const message: GithubWorkflowEvent = {
-    workflowJobEvent: JSON.parse(JSON.stringify(workflowjob_event)),
-  };
-  const sqsMessage: SendMessageCommandInput = {
-    QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue',
-    MessageBody: JSON.stringify(message),
-  };
-  beforeEach(() => {
-    ConfigDispatcher.reset();
-    const mockedGet = mocked(getParameter);
-    mockedGet.mockResolvedValue('["abc"]');
-  });
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
-  it('sends webhook events to workflow job queue', async () => {
-    // Arrange
-    process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl || '';
-    const config: ConfigWebhook = await ConfigWebhook.load();
-
-    // Act
-    const result = sendWebhookEventToWorkflowJobQueue(message, config);
-
-    // Assert
-    expect(mockSQS.sendMessage).toHaveBeenCalledWith(sqsMessage);
-    await expect(result).resolves.not.toThrow();
-  });
-
-  it('Does not send webhook events to workflow job event copy queue when job queue is not in environment', async () => {
-    // Arrange
-    process.env.SQS_WORKFLOW_JOB_QUEUE = '';
-    const config: ConfigDispatcher = await ConfigDispatcher.load();
-
-    // Act
-    await sendWebhookEventToWorkflowJobQueue(message, config);
-
-    // Assert
-    expect(SQS).not.toHaveBeenCalled();
-  });
-
-  it('Catch the exception when even copy queue throws exception', async () => {
-    // Arrange
-    process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl || '';
-    const config: ConfigDispatcher = await ConfigDispatcher.load();
-
-    const mockSQS = {
-      sendMessage: jest.fn(() => {
-        throw new Error();
-      }),
-    };
-    jest.mock('aws-sdk', () => ({
-      SQS: jest.fn().mockImplementation(() => mockSQS),
-    }));
-    await expect(sendWebhookEventToWorkflowJobQueue(message, config)).resolves.not.toThrow();
-  });
-});