From ddb231c20c0434180b023d1a9a17ad21c50c2cef Mon Sep 17 00:00:00 2001 From: "Connor Bell (Makeshift)" Date: Fri, 10 Jan 2025 19:57:03 +0000 Subject: [PATCH 1/6] fix(s3): support s3 auth via env --- lib/storage/drivers/s3.ts | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/storage/drivers/s3.ts b/lib/storage/drivers/s3.ts index a34a643..e50d5e6 100644 --- a/lib/storage/drivers/s3.ts +++ b/lib/storage/drivers/s3.ts @@ -17,24 +17,36 @@ import { streamToBuffer } from '~/lib/utils' export const s3Driver = defineStorageDriver({ envSchema: z.object({ STORAGE_S3_BUCKET: z.string().min(1), - STORAGE_S3_ENDPOINT: z.string().min(1), - STORAGE_S3_REGION: z.string().min(1).default('us-east-1'), - STORAGE_S3_PORT: z.coerce.number().positive(), - STORAGE_S3_USE_SSL: z.string().transform((v) => v === 'true'), - STORAGE_S3_ACCESS_KEY: z.string().min(1), - STORAGE_S3_SECRET_KEY: z.string().min(1), + STORAGE_S3_ENDPOINT: z.string().optional(), + STORAGE_S3_REGION: z.string().min(1).default('us-east-1').optional(), + STORAGE_S3_PORT: z.coerce.number().positive().optional(), + STORAGE_S3_USE_SSL: z + .string() + .transform((v) => v === 'true') + .optional(), + STORAGE_S3_ACCESS_KEY: z.string().optional(), + STORAGE_S3_SECRET_KEY: z.string().optional(), + AWS_REGION: z.string().optional(), + AWS_DEFAULT_REGION: z.string().optional(), }), async setup(options) { const protocol = options.STORAGE_S3_USE_SSL ? 'https' : 'http' const port = options.STORAGE_S3_PORT ? `:${options.STORAGE_S3_PORT}` : '' + let credentials - const s3 = new S3Client({ - credentials: { + if (options.STORAGE_S3_ACCESS_KEY && options.STORAGE_S3_SECRET_KEY) { + credentials = { secretAccessKey: options.STORAGE_S3_SECRET_KEY, accessKeyId: options.STORAGE_S3_ACCESS_KEY, - }, - endpoint: `${protocol}://${options.STORAGE_S3_ENDPOINT}${port}`, - region: options.STORAGE_S3_REGION, + } + } + + const s3 = new S3Client({ + credentials, + endpoint: options.STORAGE_S3_ENDPOINT + ? `${protocol}://${options.STORAGE_S3_ENDPOINT}${port}` + : undefined, + region: options.AWS_REGION ?? options.AWS_DEFAULT_REGION ?? options.STORAGE_S3_REGION, forcePathStyle: true, }) From 07e852b7b17f174177fb45d81972392ec541a4af Mon Sep 17 00:00:00 2001 From: "Connor Bell (Makeshift)" Date: Fri, 10 Jan 2025 20:00:22 +0000 Subject: [PATCH 2/6] chore(docs): remove uneeded env vars --- docs/content/2.storage-drivers/s3.md | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/docs/content/2.storage-drivers/s3.md b/docs/content/2.storage-drivers/s3.md index 7ab9c17..ddfb70f 100644 --- a/docs/content/2.storage-drivers/s3.md +++ b/docs/content/2.storage-drivers/s3.md @@ -62,12 +62,7 @@ services: STORAGE_DRIVER: s3 STORAGE_S3_BUCKET: gh-actions-cache - STORAGE_S3_ACCESS_KEY: access_key - STORAGE_S3_SECRET_KEY: secret_key - STORAGE_S3_ENDPOINT: s3.amazonaws.com - STORAGE_S3_PORT: '443' - STORAGE_S3_USE_SSL: 'true' volumes: - cache-data:/app/.data @@ -79,36 +74,20 @@ volumes: Don't forget to set the `STORAGE_DRIVER` environment variable to `s3` to use the S3 storage driver. +The AWS SDK will automatically use any AWS credentials available in the environment, e.g. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`. + #### `STORAGE_S3_BUCKET` Example: `gh-actions-cache` The name of the S3 bucket used for storage. -#### `STORAGE_S3_ACCESS_KEY` - -Example: `access_key` - -The access key for S3 storage. - -#### `STORAGE_S3_SECRET_KEY` - -Example: `secret_key` - -The secret key for S3 storage. - #### `STORAGE_S3_ENDPOINT` Example: `s3.amazonaws.com`, `minio` The endpoint hostname for S3 storage. -#### `STORAGE_S3_REGION` - -Example: `us-west-1` - -The region for AWS S3. Not needed with MinIO. - #### `STORAGE_S3_PORT` Example: `443`, `9000` From c005a8b501ef2c770d14c6c9d51a897e4f67d3dc Mon Sep 17 00:00:00 2001 From: "Connor Bell (Makeshift)" Date: Sat, 11 Jan 2025 13:01:35 +0000 Subject: [PATCH 3/6] chore(s3): remove unneeded optional --- lib/storage/drivers/s3.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/storage/drivers/s3.ts b/lib/storage/drivers/s3.ts index e50d5e6..f3b4d90 100644 --- a/lib/storage/drivers/s3.ts +++ b/lib/storage/drivers/s3.ts @@ -18,7 +18,7 @@ export const s3Driver = defineStorageDriver({ envSchema: z.object({ STORAGE_S3_BUCKET: z.string().min(1), STORAGE_S3_ENDPOINT: z.string().optional(), - STORAGE_S3_REGION: z.string().min(1).default('us-east-1').optional(), + STORAGE_S3_REGION: z.string().min(1).default('us-east-1'), STORAGE_S3_PORT: z.coerce.number().positive().optional(), STORAGE_S3_USE_SSL: z .string() From 166a5483c73ef909de6932d389aa746fd56837a8 Mon Sep 17 00:00:00 2001 From: "Connor Bell (Makeshift)" Date: Sun, 30 Mar 2025 11:55:22 +0100 Subject: [PATCH 4/6] feat: remove custom env var validation for s3 driver --- docker-compose.yml | 24 +++++++++- docs/content/2.storage-drivers/s3.md | 69 +++++++++++++++++++++------- lib/storage/drivers/s3.ts | 30 ++---------- nitro.config.ts | 1 - 4 files changed, 77 insertions(+), 47 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 73abf52..b20dd0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: postgres: image: postgres:15 @@ -24,3 +22,25 @@ services: command: -c 'mkdir -p /data/test && /usr/bin/minio server /data' ports: - 9000:9000 + environment: + MINIO_ROOT_USER: access_key + MINIO_ROOT_PASSWORD: secret_key + + cache-server: + build: + dockerfile: Dockerfile + context: . + ports: + - '3000:3000' + depends_on: + - minio + + environment: + API_BASE_URL: http://localhost:3000 + + STORAGE_DRIVER: s3 + STORAGE_S3_BUCKET: test + + AWS_ACCESS_KEY_ID: access_key + AWS_SECRET_ACCESS_KEY: secret_key + AWS_ENDPOINT_URL: http://minio:9000 diff --git a/docs/content/2.storage-drivers/s3.md b/docs/content/2.storage-drivers/s3.md index 6e51004..a3607ca 100644 --- a/docs/content/2.storage-drivers/s3.md +++ b/docs/content/2.storage-drivers/s3.md @@ -24,12 +24,10 @@ services: STORAGE_DRIVER: s3 STORAGE_S3_BUCKET: gh-actions-cache - STORAGE_S3_ACCESS_KEY: access_key - STORAGE_S3_SECRET_KEY: secret_key - STORAGE_S3_ENDPOINT: minio - STORAGE_S3_PORT: '9000' - STORAGE_S3_USE_SSL: 'false' + AWS_ACCESS_KEY_ID: access_key + AWS_SECRET_ACCESS_KEY: secret_key + AWS_ENDPOINT_URL: http://minio:9000 volumes: - cache-data:/app/.data @@ -47,6 +45,8 @@ volumes: ### `docker-compose` AWS S3 example +This example assumes that credentials are being provided by the environment, e.g. via an [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) or [EKS IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). + ```yaml [docker-compose.yml] version: '3.9' @@ -70,30 +70,65 @@ volumes: ### Environment Variables -Don't forget to set the `STORAGE_DRIVER` environment variable to `s3` to use the S3 storage driver. +The only required S3-related environment variables are `STORAGE_DRIVER: s3` and `STORAGE_S3_BUCKET`. The rest of the environment variables are optional and depend on your S3-compatible storage provider. + +The AWS SDK will automatically use any AWS credentials available in the environment, e.g. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`. Outside of AWS, these environment variables can still be used to authenticate with S3-compatible storage, as seen in the Minio example above. -The AWS SDK will automatically use any AWS credentials available in the environment, e.g. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`. +Common environment variables are listed below. For a full list of configuration options, see the [AWS SDK documentation](https://docs.aws.amazon.com/sdkref/latest/guide/settings-reference.html#EVarSettings). #### `STORAGE_S3_BUCKET` Example: `gh-actions-cache` -The name of the S3 bucket used for storage. +The name of the S3 bucket used for storage. This environment variable is always required. + +#### `AWS_REGION` + +Example: `us-east-1` + +The AWS SDK relies on this variable being set. In the cache server, it defaults to `us-east-1` if not provided. This has no effect if you are using a non-AWS S3-compatible storage provider, such as MinIO. + +#### `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + +Example: + `AWS_ACCESS_KEY_ID: access_key` + `AWS_SECRET_ACCESS_KEY: secret_key` -#### `STORAGE_S3_ENDPOINT` +This is the access key/secret key used to authenticate with S3-compatible storage. If required to authenticate with your provider, these should be provided by the provider. Alternatively, you can use the `AWS_PROFILE` environment variable to specify a profile from your AWS credentials file. -Example: `s3.amazonaws.com`, `minio` +#### `AWS_PROFILE` -The endpoint hostname for S3 storage. +Example: `my-profile` -#### `STORAGE_S3_PORT` +If you wish to run the cache server locally and utilize a profile from your AWS credentials file or local AWS CLI configuration, you can set the `AWS_PROFILE` environment variable to the name of the profile. Note that this will also require mounting the AWS credentials file into the container in order for the SDK to be able to find it. -Example: `443`, `9000` +```yaml [docker-compose.yml] +version: '3.9' -The port S3 storage is running on. +services: + cache-server: + image: ghcr.io/falcondev-oss/github-actions-cache-server:latest + ports: + - '3000:3000' + environment: + API_BASE_URL: http://localhost:3000 + + STORAGE_DRIVER: s3 + STORAGE_S3_BUCKET: gh-actions-cache + + AWS_PROFILE: my-profile + + volumes: + - cache-data:/app/.data + # Mount the AWS CLI credentials and config into the container + - ~/.aws:/root/.aws:ro + +volumes: + cache-data: +``` -#### `STORAGE_S3_USE_SSL` +#### `AWS_ENDPOINT_URL` -Example: `false` +Example: `http://minio:9000` -Whether to use SSL for S3 storage connections. +This is the endpoint URL for the S3-compatible storage. This is only required if you are using a non-AWS S3-compatible storage provider, such as MinIO. diff --git a/lib/storage/drivers/s3.ts b/lib/storage/drivers/s3.ts index f3b4d90..1421373 100644 --- a/lib/storage/drivers/s3.ts +++ b/lib/storage/drivers/s3.ts @@ -17,37 +17,13 @@ import { streamToBuffer } from '~/lib/utils' export const s3Driver = defineStorageDriver({ envSchema: z.object({ STORAGE_S3_BUCKET: z.string().min(1), - STORAGE_S3_ENDPOINT: z.string().optional(), - STORAGE_S3_REGION: z.string().min(1).default('us-east-1'), - STORAGE_S3_PORT: z.coerce.number().positive().optional(), - STORAGE_S3_USE_SSL: z - .string() - .transform((v) => v === 'true') - .optional(), - STORAGE_S3_ACCESS_KEY: z.string().optional(), - STORAGE_S3_SECRET_KEY: z.string().optional(), - AWS_REGION: z.string().optional(), - AWS_DEFAULT_REGION: z.string().optional(), + // AWS SDK requires an AWS_REGION to be set, even if you're using a custom endpoint + AWS_REGION: z.string().default('us-east-1'), }), async setup(options) { - const protocol = options.STORAGE_S3_USE_SSL ? 'https' : 'http' - const port = options.STORAGE_S3_PORT ? `:${options.STORAGE_S3_PORT}` : '' - let credentials - - if (options.STORAGE_S3_ACCESS_KEY && options.STORAGE_S3_SECRET_KEY) { - credentials = { - secretAccessKey: options.STORAGE_S3_SECRET_KEY, - accessKeyId: options.STORAGE_S3_ACCESS_KEY, - } - } - const s3 = new S3Client({ - credentials, - endpoint: options.STORAGE_S3_ENDPOINT - ? `${protocol}://${options.STORAGE_S3_ENDPOINT}${port}` - : undefined, - region: options.AWS_REGION ?? options.AWS_DEFAULT_REGION ?? options.STORAGE_S3_REGION, forcePathStyle: true, + region: options.AWS_REGION, }) try { diff --git a/nitro.config.ts b/nitro.config.ts index 9f4f5d4..79887c7 100644 --- a/nitro.config.ts +++ b/nitro.config.ts @@ -38,5 +38,4 @@ export default defineNitroConfig({ exclude: ['../../docs'], }, }, - compatibilityDate: '2025-02-01', }) From dc6de048632e8a5483bd48f9dbf9325aac26f5c0 Mon Sep 17 00:00:00 2001 From: "Connor Bell (Makeshift)" Date: Sun, 30 Mar 2025 11:59:41 +0100 Subject: [PATCH 5/6] fix: re-add accidentally-removed key --- nitro.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/nitro.config.ts b/nitro.config.ts index 7a062b3..cc16ddb 100644 --- a/nitro.config.ts +++ b/nitro.config.ts @@ -36,4 +36,5 @@ export default defineNitroConfig({ exclude: ['../../docs'], }, }, + compatibilityDate: '2025-02-01', }) From 8328de8f18182d4c27a6c7e995cfb39523564ad6 Mon Sep 17 00:00:00 2001 From: "Connor Bell (Makeshift)" Date: Tue, 8 Apr 2025 13:31:11 +0100 Subject: [PATCH 6/6] fix(s3): update test env vars --- .env | 10 ++++------ tests/.env.s3.storage | 8 +++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.env b/.env index e8c29c1..168eb9c 100644 --- a/.env +++ b/.env @@ -8,11 +8,9 @@ STORAGE_FILESYSTEM_PATH=.data/storage/filesystem # s3 # STORAGE_DRIVER=s3 # STORAGE_S3_BUCKET=test -# STORAGE_S3_ENDPOINT=localhost -# STORAGE_S3_PORT=9000 -# STORAGE_S3_USE_SSL=false -# STORAGE_S3_ACCESS_KEY=minioadmin -# STORAGE_S3_SECRET_KEY=minioadmin +# AWS_ENDPOINT_URL=http://minio:9000 +# AWS_ACCESS_KEY_ID=minioadmin +# AWS_SECRET_ACCESS_KEY=minioadmin # sqlite DB_DRIVER=sqlite @@ -32,4 +30,4 @@ DB_SQLITE_PATH=.data/sqlite.db # DB_MYSQL_HOST=localhost # DB_MYSQL_USER=root # DB_MYSQL_PASSWORD=root -# DB_MYSQL_PORT=3306 \ No newline at end of file +# DB_MYSQL_PORT=3306 diff --git a/tests/.env.s3.storage b/tests/.env.s3.storage index f77c9bf..36e4e95 100644 --- a/tests/.env.s3.storage +++ b/tests/.env.s3.storage @@ -1,7 +1,5 @@ STORAGE_DRIVER=s3 STORAGE_S3_BUCKET=test -STORAGE_S3_ENDPOINT=localhost -STORAGE_S3_PORT=9000 -STORAGE_S3_USE_SSL=false -STORAGE_S3_ACCESS_KEY=minioadmin -STORAGE_S3_SECRET_KEY=minioadmin \ No newline at end of file +AWS_ENDPOINT_URL=http://localhost:9000 +AWS_ACCESS_KEY_ID=minioadmin +AWS_SECRET_ACCESS_KEY=minioadmin