Merge pull request #1376 from philips-labs/develop

release

Author: npalm

.github/workflows/release.yml

@@ -33,6 +33,7 @@ jobs:
           retention-days: 1
 
   release:
+    name: release
     runs-on: ubuntu-latest
     needs:
       prepare
@@ -80,4 +81,21 @@ jobs:
           cp .release/* .
           yarn
           yarn release --repositoryUrl https://x-access-token:[email protected]/$GITHUB_REPOSITORY.git
+          
+  provenance:
+    name: Generate provenance
+    runs-on: ubuntu-20.04
+    needs: 
+      release
+    if: startsWith(github.ref, 'refs/tags/')
+
+    steps:
+      - name: Generate provenance for release
+        uses: philips-labs/[email protected]
+        with:
+          artifact_path: release-assets
+          output_path: 'build.provenance'
+          tag_name: "${{ github.ref_name }}"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/terraform.yml

@@ -13,12 +13,11 @@ env:
   tf_working_dir: "."
   AWS_REGION: eu-west-1
 jobs:
-
   verify_module:
     name: Verify module
     strategy:
       matrix:
-        terraform: [1.0.8]    
+        terraform: [1.0.8]
     runs-on: ubuntu-latest
     container:
       image: hashicorp/terraform:${{ matrix.terraform }}
@@ -43,15 +42,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        terraform: [0.14.1, 0.15.0, 1.0.8]
+        terraform: [0.14.3, 0.15.5, 1.0.8]
         example: ["default", "ubuntu"]
     defaults:
       run:
         working-directory: examples/${{ matrix.example }}
     runs-on: ubuntu-latest
     container:
       image: hashicorp/terraform:${{ matrix.terraform }}
-    steps:   
+    steps:
       - uses: actions/checkout@v2
       - name: terraform init
         run: terraform init -get -backend=false -input=false
@@ -61,5 +60,3 @@ jobs:
         continue-on-error: true
       - name: validate terraform
         run: terraform validate
-
-

README.md

@@ -24,6 +24,8 @@ This [Terraform](https://www.terraform.io/) module creates the required infrastr
 - [Providers](#providers)
 - [Modules](#modules)
 - [Resources](#resources)
+- [Modules](#modules-1)
+- [Resources](#resources-1)
 - [Inputs](#inputs)
 - [Outputs](#outputs)
 - [Contribution](#contribution)
@@ -382,11 +384,14 @@ In case the setup does not work as intended follow the trace of events:
 | <a name="input_instance_profile_path"></a> [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no |
 | <a name="input_instance_type"></a> [instance\_type](#input\_instance\_type) | [DEPRECATED] See instance\_types. | `string` | `"m5.large"` | no |
 | <a name="input_instance_types"></a> [instance\_types](#input\_instance\_types) | List of instance types for the action runner. | `list(string)` | `null` | no |
+| <a name="input_job_queue_retention_in_seconds"></a> [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged | `number` | `86400` | no |
 | <a name="input_key_name"></a> [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no |
 | <a name="input_kms_key_arn"></a> [kms\_key\_arn](#input\_kms\_key\_arn) | Optional CMK Key ARN to be used for Parameter Store. This key must be in the current account. | `string` | `null` | no |
 | <a name="input_lambda_s3_bucket"></a> [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no |
 | <a name="input_lambda_security_group_ids"></a> [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
+| <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
+| <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
 | <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | <a name="input_market_options"></a> [market\_options](#input\_market\_options) | Market options for the action runner instances. Setting the value to `null` let the scaler create on-demand instances instead of spot instances. | `string` | `"spot"` | no |
 | <a name="input_minimum_running_time_in_minutes"></a> [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if not busy. | `number` | `5` | no |
@@ -396,9 +401,11 @@ In case the setup does not work as intended follow the trace of events:
 | <a name="input_runner_additional_security_group_ids"></a> [runner\_additional\_security\_group\_ids](#input\_runner\_additional\_security\_group\_ids) | (optional) List of additional security groups IDs to apply to the runner | `list(string)` | `[]` | no |
 | <a name="input_runner_allow_prerelease_binaries"></a> [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
 | <a name="input_runner_as_root"></a> [runner\_as\_root](#input\_runner\_as\_root) | Run the action runner under the root user. | `bool` | `false` | no |
+| <a name="input_runner_binaries_s3_sse_configuration"></a> [runner\_binaries\_s3\_sse\_configuration](#input\_runner\_binaries\_s3\_sse\_configuration) | Map containing server-side encryption configuration for runner-binaries S3 bucket. | `any` | `{}` | no |
 | <a name="input_runner_binaries_syncer_lambda_timeout"></a> [runner\_binaries\_syncer\_lambda\_timeout](#input\_runner\_binaries\_syncer\_lambda\_timeout) | Time out of the binaries sync lambda in seconds. | `number` | `300` | no |
 | <a name="input_runner_binaries_syncer_lambda_zip"></a> [runner\_binaries\_syncer\_lambda\_zip](#input\_runner\_binaries\_syncer\_lambda\_zip) | File location of the binaries sync lambda zip file. | `string` | `null` | no |
 | <a name="input_runner_boot_time_in_minutes"></a> [runner\_boot\_time\_in\_minutes](#input\_runner\_boot\_time\_in\_minutes) | The minimum time for an EC2 runner to boot and register as a runner. | `number` | `5` | no |
+| <a name="input_runner_ec2_tags"></a> [runner\_ec2\_tags](#input\_runner\_ec2\_tags) | Map of tags that will be added to the launch template instance tag specificatons. | `map(string)` | `{}` | no |
 | <a name="input_runner_egress_rules"></a> [runner\_egress\_rules](#input\_runner\_egress\_rules) | List of egress rules for the GitHub runner instances. | <pre>list(object({<br>    cidr_blocks      = list(string)<br>    ipv6_cidr_blocks = list(string)<br>    prefix_list_ids  = list(string)<br>    from_port        = number<br>    protocol         = string<br>    security_groups  = list(string)<br>    self             = bool<br>    to_port          = number<br>    description      = string<br>  }))</pre> | <pre>[<br>  {<br>    "cidr_blocks": [<br>      "0.0.0.0/0"<br>    ],<br>    "description": null,<br>    "from_port": 0,<br>    "ipv6_cidr_blocks": [<br>      "::/0"<br>    ],<br>    "prefix_list_ids": null,<br>    "protocol": "-1",<br>    "security_groups": null,<br>    "self": null,<br>    "to_port": 0<br>  }<br>]</pre> | no |
 | <a name="input_runner_extra_labels"></a> [runner\_extra\_labels](#input\_runner\_extra\_labels) | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
 | <a name="input_runner_group_name"></a> [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no |

examples/default/main.tf

@@ -39,6 +39,15 @@ module "runners" {
   # enable access to the runners via SSM
   enable_ssm_on_runners = true
 
+  # use S3 or KMS SSE to runners S3 bucket
+  # runner_binaries_s3_sse_configuration = {
+  #   rule = {
+  #     apply_server_side_encryption_by_default = {
+  #       sse_algorithm = "AES256"
+  #     }
+  #   }
+  # }
+
   # Uncommet idle config to have idle runners from 9 to 5 in time zone Amsterdam
   # idle_config = [{
   #   cron      = "* * 9-17 * * *"

main.tf

@@ -64,6 +64,9 @@ module "webhook" {
   role_path                 = var.role_path
   role_permissions_boundary = var.role_permissions_boundary
   repository_white_list     = var.repository_white_list
+
+  log_type  = var.log_type
+  log_level = var.log_level
 }
 
 module "runners" {
@@ -133,6 +136,9 @@ module "runners" {
   ghes_ssl_verify = var.ghes_ssl_verify
 
   kms_key_arn = var.kms_key_arn
+
+  log_type  = var.log_type
+  log_level = var.log_level
 }
 
 module "runner_binaries" {
@@ -154,8 +160,13 @@ module "runner_binaries" {
   lambda_timeout                  = var.runner_binaries_syncer_lambda_timeout
   logging_retention_in_days       = var.logging_retention_in_days
 
+  server_side_encryption_configuration = var.runner_binaries_s3_sse_configuration
+
   role_path                 = var.role_path
   role_permissions_boundary = var.role_permissions_boundary
+
+  log_type  = var.log_type
+  log_level = var.log_level
 }
 
 resource "aws_resourcegroups_group" "resourcegroups_group" {

modules/runner-binaries-syncer/README.md

@@ -84,11 +84,14 @@ No modules.
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
 | <a name="input_lambda_timeout"></a> [lambda\_timeout](#input\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `300` | no |
 | <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
+| <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
+| <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
 | <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
 | <a name="input_role_path"></a> [role\_path](#input\_role\_path) | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
 | <a name="input_role_permissions_boundary"></a> [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
 | <a name="input_runner_allow_prerelease_binaries"></a> [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
 | <a name="input_runner_architecture"></a> [runner\_architecture](#input\_runner\_architecture) | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
+| <a name="input_server_side_encryption_configuration"></a> [server\_side\_encryption\_configuration](#input\_server\_side\_encryption\_configuration) | Map containing server-side encryption configuration. | `any` | `{}` | no |
 | <a name="input_syncer_lambda_s3_key"></a> [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
 | <a name="input_syncer_lambda_s3_object_version"></a> [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

@@ -17,12 +17,12 @@
   "devDependencies": {
     "@octokit/rest": "^18.12.0",
     "@types/jest": "^27.0.1",
-    "@types/node": "^16.11.5",
+    "@types/node": "^16.11.6",
     "@types/request": "^2.48.4",
     "@typescript-eslint/eslint-plugin": "^4.33.0",
     "@typescript-eslint/parser": "^4.33.0",
     "@vercel/ncc": "^0.31.1",
-    "aws-sdk": "^2.1016.0",
+    "aws-sdk": "^2.1019.0",
     "eslint": "^7.32.0",
     "eslint-plugin-prettier": "4.0.0",
     "jest": "^27.3.1",
@@ -32,6 +32,7 @@
     "typescript": "^4.4.4"
   },
   "dependencies": {
+    "tslog": "^3.2.2",
     "axios": "^0.24.0"
   }
-}
+}

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts

@@ -1,7 +1,10 @@
 import { handle } from './syncer/handler';
+import { logger } from './syncer/logger';
 
 // eslint-disable-next-line
 export const handler = async (event: any, context: any, callback: any): Promise<void> => {
+  logger.setSettings({ requestId: context.awsRequestId });
+  logger.debug(JSON.stringify(event));
   try {
     await handle();
     callback(null);

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/handler.ts

@@ -3,6 +3,9 @@ import { PassThrough } from 'stream';
 import { S3 } from 'aws-sdk';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { logger as rootLogger } from './logger';
+
+const logger = rootLogger.getChildLogger();
 
 const versionKey = 'name';
 
@@ -22,7 +25,7 @@ async function getCachedVersion(s3: S3, cacheObject: CacheObject): Promise<strin
     const versions = objectTagging.TagSet?.filter((t: S3.Tag) => t.Key === versionKey);
     return versions.length === 1 ? versions[0].Value : undefined;
   } catch (e) {
-    console.debug('No tags found');
+    logger.debug('No tags found');
     return undefined;
   }
 }
@@ -73,7 +76,7 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
     })
     .promise();
 
-  console.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);
+  logger.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);
 
   const readPromise = new Promise<void>((resolve, reject) => {
     axios
@@ -93,9 +96,9 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
   });
 
   await Promise.all([readPromise, writePromise])
-    .then(() => console.info(`The new distribution is uploaded to S3.`))
+    .then(() => logger.info(`The new distribution is uploaded to S3.`))
     .catch((error) => {
-      console.error(`Uploading of the new distribution to S3 failed: ${error}`);
+      logger.error(`Uploading of the new distribution to S3 failed: ${error}`);
       throw error;
     });
 }
@@ -120,10 +123,10 @@ export const handle = async (): Promise<void> => {
   }
 
   const currentVersion = await getCachedVersion(s3, cacheObject);
-  console.debug('latest: ' + currentVersion);
+  logger.debug('latest: ' + currentVersion);
   if (currentVersion === undefined || currentVersion != actionRunnerReleaseAsset.name) {
     uploadToS3(s3, cacheObject, actionRunnerReleaseAsset);
   } else {
-    console.debug('Distribution is up-to-date, no action.');
+    logger.debug('Distribution is up-to-date, no action.');
   }
 };

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/logger.ts

@@ -0,0 +1,10 @@
+import { Logger } from 'tslog';
+
+export const logger = new Logger({
+  colorizePrettyLogs: false,
+  displayInstanceName: false,
+  minLevel: process.env.LOG_LEVEL || 'info',
+  name: 'runner-binaries-syncer',
+  overwriteConsole: true,
+  type: process.env.LOG_TYPE || 'pretty',
+});