feat: fetch prerelease binaries (#165)

* Update changelog

* Fetch latest runner pre-release to avoid runner self-update on start

* Update lambda to support prerelease binaries

* Add variable to allow prerelease binaries to terraform

* Update docs

* Update changelog

* Fix review comments

* Added PR number and contributor to changelog

Co-authored-by: Simon Jagoe <[email protected]>

Author: gertjanmaas

CHANGELOG.md

@@ -10,13 +10,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - feat: Manage log groups via module. When upgrading you have to import the log groups by AWS into your state. See below the example commands for the default example.
+
 ```bash
 terraform import module.runners.module.runner_binaries.aws_cloudwatch_log_group.syncer "/aws/lambda/default-syncer"
 terraform import module.runners.module.runners.aws_cloudwatch_log_group.scale_up "/aws/lambda/default-scale-up"
 terraform import module.runners.module.runners.aws_cloudwatch_log_group.scale_down "/aws/lambda/default-scale-down"
 terraform import module.runners.module.webhook.aws_cloudwatch_log_group.webhook "/aws/lambda/default-webhook"
 ```
 
+- feat: Added option to binaries syncer to upgrade to prereleases, preventing any auto-updating on startup. Option `runner_allow_prerelease_binaries` is disabled by default. (#141, #165) @sjagoe
+
 ## [0.4.0] - 2020-08-10
 
 ### Added

README.md

@@ -301,6 +301,7 @@ No requirements.
 | minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
 | role\_path | The path that will be added to role path for created roles, if not set the environment name will be used. | `string` | `null` | no |
 | role\_permissions\_boundary | Permissions boundary that will be added to the created roles. | `string` | `null` | no |
+| runner\_allow\_prerelease\_binaries | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
 | runner\_as\_root | Run the action runner under the root user. | `bool` | `false` | no |
 | runner\_binaries\_syncer\_lambda\_timeout | Time out of the binaries sync lambda in seconds. | `number` | `300` | no |
 | runner\_binaries\_syncer\_lambda\_zip | File location of the binaries sync lambda zip file. | `string` | `null` | no |

main.tf

@@ -100,7 +100,8 @@ module "runner_binaries" {
 
   distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
 
-  runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64"
+  runner_architecture              = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64"
+  runner_allow_prerelease_binaries = var.runner_allow_prerelease_binaries
 
   lambda_zip                = var.runner_binaries_syncer_lambda_zip
   lambda_timeout            = var.runner_binaries_syncer_lambda_timeout

modules/download-lambda/README.md

@@ -25,18 +25,27 @@ module "lambdas" {
 ```
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| null | n/a |
 
 ## Inputs
 
-| Name    | Description                           |  Type  | Default | Required |
-| ------- | ------------------------------------- | :----: | :-----: | :------: |
-| lambdas | Name and tag for lambdas to download. | object |   n/a   |   yes    |
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| lambdas | Name and tag for lambdas to download. | <pre>list(object({<br>    name = string<br>    tag  = string<br>  }))</pre> | n/a | yes |
 
 ## Outputs
 
-| Name  | Description |
-| ----- | ----------- |
-| files |             |
+| Name | Description |
+|------|-------------|
+| files | n/a |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

modules/runner-binaries-syncer/README.md

@@ -57,6 +57,7 @@ No requirements.
 | logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
 | role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
 | role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
+| runner\_allow\_prerelease\_binaries | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
 | runner\_architecture | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
 | tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
 

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

@@ -26,5 +26,8 @@
     "ts-jest": "^26.2.0",
     "ts-node-dev": "^1.0.0-pre.60",
     "typescript": "^3.9.6"
+  },
+  "dependencies": {
+    "yn": "^4.0.0"
   }
 }

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts

@@ -1,5 +1,6 @@
 import { handle } from './syncer/handler';
 
+// eslint-disable-next-line
 module.exports.handler = async (event: any, context: any, callback: any): Promise<any> => {
   await handle();
   return callback();

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/handler.test.ts

@@ -1,12 +1,12 @@
 import { handle } from './handler';
-import latestReleases from '../../test/resources/github-latest-releases.json';
-import latestReleasesEmpty from '../../test/resources/github-latest-releases-empty.json';
-import latestReleasesNoLinux from '../../test/resources/github-latest-releases-no-linux.json';
-import latestReleasesNoArm64 from '../../test/resources/github-latest-releases-no-arm64.json';
+import listReleases from '../../test/resources/github-list-releases.json';
+import listReleasesEmpty from '../../test/resources/github-list-releases-empty-assets.json';
+import listReleasesNoLinux from '../../test/resources/github-list-releases-no-linux.json';
+import listReleasesNoArm64 from '../../test/resources/github-list-releases-no-arm64.json';
 
 const mockOctokit = {
   repos: {
-    getLatestRelease: jest.fn(),
+    listReleases: jest.fn(),
   },
 };
 jest.mock('@octokit/rest', () => ({
@@ -31,31 +31,51 @@ describe('Synchronize action distribution.', () => {
   beforeEach(() => {
     process.env.S3_BUCKET_NAME = bucketName;
     process.env.S3_OBJECT_KEY = bucketObjectKey;
+    process.env.GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = 'false';
 
-    mockOctokit.repos.getLatestRelease.mockImplementation(() => ({
-      data: latestReleases.data,
+    mockOctokit.repos.listReleases.mockImplementation(() => ({
+      data: listReleases,
     }));
   });
 
-  it('Distribution is up-to-date.', async () => {
+  it('Distribution is up-to-date with latest release.', async () => {
     mockS3.getObjectTagging.mockImplementation(() => {
       return {
         promise() {
-          return Promise.resolve({ TagSet: [{ Key: 'name', Value: 'actions-runner-linux-x64-2.262.1.tar.gz' }] });
+          return Promise.resolve({ TagSet: [{ Key: 'name', Value: 'actions-runner-linux-x64-2.272.0.tar.gz' }] });
         },
       };
     });
 
     await handle();
-    expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
     expect(mockS3.getObjectTagging).toBeCalledWith({
       Bucket: bucketName,
       Key: bucketObjectKey,
     });
     expect(mockS3.upload).toBeCalledTimes(0);
   });
 
-  it('Distribution should update.', async () => {
+  it('Distribution is up-to-date with latest prerelease.', async () => {
+    process.env.GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = 'true';
+    mockS3.getObjectTagging.mockImplementation(() => {
+      return {
+        promise() {
+          return Promise.resolve({ TagSet: [{ Key: 'name', Value: 'actions-runner-linux-x64-2.273.0.tar.gz' }] });
+        },
+      };
+    });
+
+    await handle();
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
+    expect(mockS3.getObjectTagging).toBeCalledWith({
+      Bucket: bucketName,
+      Key: bucketObjectKey,
+    });
+    expect(mockS3.upload).toBeCalledTimes(0);
+  });
+
+  it('Distribution should update to release.', async () => {
     mockS3.getObjectTagging.mockImplementation(() => {
       return {
         promise() {
@@ -65,12 +85,63 @@ describe('Synchronize action distribution.', () => {
     });
 
     await handle();
-    expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
     expect(mockS3.getObjectTagging).toBeCalledWith({
       Bucket: bucketName,
       Key: bucketObjectKey,
     });
     expect(mockS3.upload).toBeCalledTimes(1);
+    const s3JsonBody = mockS3.upload.mock.calls[0][0];
+    expect(s3JsonBody['Tagging']).toEqual('name=actions-runner-linux-x64-2.272.0.tar.gz');
+  });
+
+  it('Distribution should update to prerelease.', async () => {
+    process.env.GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = 'true';
+    mockS3.getObjectTagging.mockImplementation(() => {
+      return {
+        promise() {
+          return Promise.resolve({ TagSet: [{ Key: 'name', Value: 'actions-runner-linux-x64-0.tar.gz' }] });
+        },
+      };
+    });
+
+    await handle();
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
+    expect(mockS3.getObjectTagging).toBeCalledWith({
+      Bucket: bucketName,
+      Key: bucketObjectKey,
+    });
+    expect(mockS3.upload).toBeCalledTimes(1);
+    const s3JsonBody = mockS3.upload.mock.calls[0][0];
+    expect(s3JsonBody['Tagging']).toEqual('name=actions-runner-linux-x64-2.273.0.tar.gz');
+  });
+
+  it('Distribution should not update to prerelease if there is a newer release.', async () => {
+    process.env.GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = 'true';
+    const releases = listReleases;
+    releases[0].prerelease = false;
+    releases[1].prerelease = true;
+
+    mockOctokit.repos.listReleases.mockImplementation(() => ({
+      data: releases,
+    }));
+    mockS3.getObjectTagging.mockImplementation(() => {
+      return {
+        promise() {
+          return Promise.resolve({ TagSet: [{ Key: 'name', Value: 'actions-runner-linux-x64-0.tar.gz' }] });
+        },
+      };
+    });
+
+    await handle();
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
+    expect(mockS3.getObjectTagging).toBeCalledWith({
+      Bucket: bucketName,
+      Key: bucketObjectKey,
+    });
+    expect(mockS3.upload).toBeCalledTimes(1);
+    const s3JsonBody = mockS3.upload.mock.calls[0][0];
+    expect(s3JsonBody['Tagging']).toEqual('name=actions-runner-linux-x64-2.273.0.tar.gz');
   });
 
   it('No tag in S3, distribution should update.', async () => {
@@ -83,7 +154,7 @@ describe('Synchronize action distribution.', () => {
     });
 
     await handle();
-    expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
     expect(mockS3.getObjectTagging).toBeCalledWith({
       Bucket: bucketName,
       Key: bucketObjectKey,
@@ -101,7 +172,7 @@ describe('Synchronize action distribution.', () => {
     });
 
     await handle();
-    expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
+    expect(mockOctokit.repos.listReleases).toBeCalledTimes(1);
     expect(mockS3.getObjectTagging).toBeCalledWith({
       Bucket: bucketName,
       Key: bucketObjectKey,
@@ -118,16 +189,16 @@ describe('No release assets found.', () => {
   });
 
   it('Empty list of assets.', async () => {
-    mockOctokit.repos.getLatestRelease.mockImplementation(() => ({
-      data: latestReleasesEmpty.data,
+    mockOctokit.repos.listReleases.mockImplementation(() => ({
+      data: listReleasesEmpty,
     }));
 
     await expect(handle()).rejects.toThrow(errorMessage);
   });
 
   it('No linux x64 asset.', async () => {
-    mockOctokit.repos.getLatestRelease.mockImplementation(() => ({
-      data: latestReleasesNoLinux.data,
+    mockOctokit.repos.listReleases.mockImplementation(() => ({
+      data: [listReleasesNoLinux],
     }));
 
     await expect(handle()).rejects.toThrow(errorMessage);
@@ -154,18 +225,18 @@ describe('Invalid config', () => {
 });
 
 describe('Synchronize action distribution for arm64.', () => {
- const errorMessage = 'Cannot find GitHub release asset.';
- beforeEach(() => {
-   process.env.S3_BUCKET_NAME = bucketName;
-   process.env.S3_OBJECT_KEY = bucketObjectKey;
-   process.env.GITHUB_RUNNER_ARCHITECTURE = 'arm64';
- });
-
- it('No linux arm64 asset.', async () => {
-  mockOctokit.repos.getLatestRelease.mockImplementation(() => ({
-    data: latestReleasesNoArm64.data,
-  }));
-
-  await expect(handle()).rejects.toThrow(errorMessage);
- });
+  const errorMessage = 'Cannot find GitHub release asset.';
+  beforeEach(() => {
+    process.env.S3_BUCKET_NAME = bucketName;
+    process.env.S3_OBJECT_KEY = bucketObjectKey;
+    process.env.GITHUB_RUNNER_ARCHITECTURE = 'arm64';
+  });
+
+  it('No linux arm64 asset.', async () => {
+    mockOctokit.repos.listReleases.mockImplementation(() => ({
+      data: [listReleasesNoArm64],
+    }));
+
+    await expect(handle()).rejects.toThrow(errorMessage);
+  });
 });

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/handler.ts

@@ -3,6 +3,7 @@ import { PassThrough } from 'stream';
 import request from 'request';
 import { S3 } from 'aws-sdk';
 import AWS from 'aws-sdk';
+import yn from 'yn';
 
 const versionKey = 'name';
 
@@ -31,13 +32,31 @@ interface ReleaseAsset {
   downloadUrl: string;
 }
 
-async function getLinuxReleaseAsset(runnerArch = 'x64'): Promise<ReleaseAsset | undefined> {
+async function getLinuxReleaseAsset(
+  runnerArch = 'x64',
+  fetchPrereleaseBinaries = false,
+): Promise<ReleaseAsset | undefined> {
   const githubClient = new Octokit();
-  const assets = await githubClient.repos.getLatestRelease({
+  const assetsList = await githubClient.repos.listReleases({
     owner: 'actions',
     repo: 'runner',
   });
-  const linuxAssets = assets.data.assets?.filter((a) => a.name?.includes(`actions-runner-linux-${runnerArch}-`));
+  if (assetsList.data?.length === 0) {
+    return undefined;
+  }
+
+  const latestPrereleaseIndex = assetsList.data.findIndex((a) => a.prerelease === true);
+  const latestReleaseIndex = assetsList.data.findIndex((a) => a.prerelease === false);
+
+  let asset = undefined;
+  if (fetchPrereleaseBinaries && latestPrereleaseIndex < latestReleaseIndex) {
+    asset = assetsList.data[latestPrereleaseIndex];
+  } else if (latestReleaseIndex != -1) {
+    asset = assetsList.data[latestReleaseIndex];
+  } else {
+    return undefined;
+  }
+  const linuxAssets = asset.assets?.filter((a) => a.name?.includes(`actions-runner-linux-${runnerArch}-`));
 
   return linuxAssets?.length === 1
     ? { name: linuxAssets[0].name, downloadUrl: linuxAssets[0].browser_download_url }
@@ -73,7 +92,8 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
 export const handle = async (): Promise<void> => {
   const s3 = new AWS.S3();
 
-  const runnerArch = process.env.GITHUB_RUNNER_ARCHITECTURE || 'x64'
+  const runnerArch = process.env.GITHUB_RUNNER_ARCHITECTURE || 'x64';
+  const fetchPrereleaseBinaries = yn(process.env.GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES, { default: false });
 
   const cacheObject: CacheObject = {
     bucket: process.env.S3_BUCKET_NAME as string,
@@ -83,7 +103,7 @@ export const handle = async (): Promise<void> => {
     throw Error('Please check all mandatory variables are set.');
   }
 
-  const actionRunnerReleaseAsset = await getLinuxReleaseAsset(runnerArch);
+  const actionRunnerReleaseAsset = await getLinuxReleaseAsset(runnerArch, fetchPrereleaseBinaries);
   if (actionRunnerReleaseAsset === undefined) {
     throw Error('Cannot find GitHub release asset.');
   }