fix(lambda): Prevent scale-up lambda from starting runner for user repo if org level runners is enabled (#3909)

We run a setup where we only have "Org Level Runners"
(`enable_organization_runners: true`).

When creating an action job in a repository that does not belong to an
Organization (User level repository), that ScaleUp lambda will happily
create a new VM for it.
Well, this new VM in turn, will never be able to register itself in
GitHub because it cannot register itself as a "Org Level Runner" as such
concept does not exist at the user level.
This VM will then become a "zombie" VM and will also cause issues with
the ScaleDown lambda (described in `ToBeCreatedIssue`).

In this PR, we will pass the "Repo Owner Type" all the way from the
Webhook lambda to the ScaleUp lambda (by adding a new field in the SQS
payload). Then, the ScaleUp lambda can decide to drop the request if the
"Repo Owner Type" is not of the type `Organization` and
`enable_organization_runners` is set to `true`.

---------

Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: Stuart Pearson <[email protected]>

Author: 3 people

lambdas/functions/control-plane/src/lambda.test.ts

@@ -15,6 +15,7 @@ const body: ActionRequestMessage = {
   installationId: 1,
   repositoryName: 'name',
   repositoryOwner: 'owner',
+  repoOwnerType: 'Organization',
 };
 
 const sqsRecord: SQSRecord = {

lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts

@@ -53,6 +53,7 @@ const TEST_DATA: scaleUpModule.ActionRequestMessage = {
   repositoryName: 'hello-world',
   repositoryOwner: 'Codertocat',
   installationId: 2,
+  repoOwnerType: 'Organization',
 };
 
 // installationId 0 means no installationId is set.
@@ -62,6 +63,7 @@ const TEST_DATA_WITH_ZERO_INSTALL_ID: scaleUpModule.ActionRequestMessage = {
   repositoryName: 'hello-world',
   repositoryOwner: 'Codertocat',
   installationId: 0,
+  repoOwnerType: 'Organization',
 };
 
 const cleanEnv = process.env;
@@ -305,6 +307,14 @@ describe('scaleUp with GHES', () => {
       expect(mockOctokit.paginate).toHaveBeenCalledTimes(1);
     });
 
+    it('Discards event if it is a User repo and org level runners is enabled', async () => {
+      process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+      const USER_REPO_TEST_DATA = { ...TEST_DATA };
+      USER_REPO_TEST_DATA.repoOwnerType = 'User';
+      await scaleUpModule.scaleUp('aws:sqs', USER_REPO_TEST_DATA);
+      expect(createRunner).not.toHaveBeenCalled();
+    });
+
     it('create SSM parameter for runner group id if it doesnt exist', async () => {
       mockSSMClient.on(GetParameterCommand).rejects();
       await scaleUpModule.scaleUp('aws:sqs', TEST_DATA);

lambdas/functions/control-plane/src/scale-runners/scale-up.ts

@@ -27,6 +27,7 @@ export interface ActionRequestMessage {
   repositoryName: string;
   repositoryOwner: string;
   installationId: number;
+  repoOwnerType: string;
 }
 
 interface CreateGitHubRunnerConfig {
@@ -250,6 +251,16 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
         `Please ensure you have enabled workflow_job events.`,
     );
   }
+
+  if (!isValidRepoOwnerTypeIfOrgLevelEnabled(payload, enableOrgLevel)) {
+    logger.warn(
+      `Repository ${payload.repositoryOwner}/${payload.repositoryName} does not belong to a GitHub` +
+        `organization and organization runners are enabled. This is not supported. Not scaling up for this event.` +
+        `Not throwing error to prevent re-queueing and just ignoring the event.`,
+    );
+    return;
+  }
+
   const ephemeral = ephemeralEnabled && payload.eventType === 'workflow_job';
   const runnerType = enableOrgLevel ? 'Org' : 'Repo';
   const runnerOwner = enableOrgLevel ? payload.repositoryOwner : `${payload.repositoryOwner}/${payload.repositoryName}`;
@@ -341,6 +352,10 @@ async function createStartRunnerConfig(
   }
 }
 
+function isValidRepoOwnerTypeIfOrgLevelEnabled(payload: ActionRequestMessage, enableOrgLevel: boolean): boolean {
+  return !(enableOrgLevel && payload.repoOwnerType !== 'Organization');
+}
+
 function addDelay(instances: string[]) {
   const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
   const ssmParameterStoreMaxThroughput = 40;

lambdas/functions/webhook/src/sqs/index.test.ts

@@ -28,6 +28,7 @@ describe('Test sending message to SQS.', () => {
     repositoryOwner: 'owner',
     queueId: queueUrl,
     queueFifo: false,
+    repoOwnerType: 'Organization',
   };
 
   afterEach(() => {

lambdas/functions/webhook/src/sqs/index.ts

@@ -13,6 +13,7 @@ export interface ActionRequestMessage {
   installationId: number;
   queueId: string;
   queueFifo: boolean;
+  repoOwnerType: string;
 }
 
 export interface MatcherConfig {

lambdas/functions/webhook/src/webhook/index.test.ts

@@ -450,6 +450,7 @@ describe('handler', () => {
         installationId: 0,
         queueId: 'ubuntu-queue-id',
         queueFifo: false,
+        repoOwnerType: 'Organization',
       });
     });
     it('Check webhook will accept jobs for latest labels if workflow labels are not specific', async () => {
@@ -492,6 +493,7 @@ describe('handler', () => {
         installationId: 0,
         queueId: 'ubuntu-queue-id',
         queueFifo: false,
+        repoOwnerType: 'Organization',
       });
     });
   });
@@ -531,6 +533,7 @@ describe('handler', () => {
       installationId: 0,
       queueId: 'ubuntu-queue-id',
       queueFifo: false,
+      repoOwnerType: 'Organization',
     });
   });
 

lambdas/functions/webhook/src/webhook/index.ts

@@ -53,6 +53,7 @@ async function handleWorkflowJob(
           installationId: installationId,
           queueId: queue.id,
           queueFifo: queue.fifo,
+          repoOwnerType: body.repository.owner.type,
         });
         logger.info(`Successfully queued job for ${body.repository.full_name} to the queue ${queue.id}`);
         return { statusCode: 201 };