refactor: update storage configuration for AWS S3-compatible object storage

- Migrate environment variables to AWS-style naming
- Implement signed request generation for S3 storage
- Update storage utility functions to use signed requests
- Add new dev script to run without mocks
- Update documentation and example environment file

Author: kentcdodds

.env.example

@@ -21,7 +21,8 @@ GITHUB_TOKEN="MOCK_GITHUB_TOKEN"
 ALLOW_INDEXING="true"
 
 # Tigris Object Storage (S3-compatible) Configuration
-STORAGE_ACCESS_KEY="mock-access-key"
-STORAGE_SECRET_KEY="mock-secret-key"
-STORAGE_BUCKET="my-app-bucket"
-STORAGE_ENDPOINT="https://my-app-storage.example.com"
+AWS_ACCESS_KEY_ID="mock-access-key"
+AWS_SECRET_ACCESS_KEY="mock-secret-key"
+AWS_REGION="auto"
+AWS_ENDPOINT_URL_S3="https://fly.storage.tigris.dev"
+BUCKET_NAME="mock-bucket"

app/routes/resources+/note-images.$imageId.tsx

@@ -1,6 +1,6 @@
 import { invariantResponse } from '@epic-web/invariant'
 import { prisma } from '#app/utils/db.server.ts'
-import { getImageUrl } from '#app/utils/storage.server.ts'
+import { getSignedGetRequestInfo } from '#app/utils/storage.server.ts'
 import { type Route } from './+types/note-images.$imageId.ts'
 
 export async function loader({ params }: Route.LoaderArgs) {
@@ -10,5 +10,7 @@ export async function loader({ params }: Route.LoaderArgs) {
 		select: { storageKey: true },
 	})
 	invariantResponse(noteImage, 'Note image not found', { status: 404 })
-	return fetch(getImageUrl(noteImage.storageKey))
+
+	const { url, headers } = getSignedGetRequestInfo(noteImage.storageKey)
+	return fetch(url, { headers })
 }

app/routes/resources+/user-images.$imageId.tsx

@@ -1,6 +1,6 @@
 import { invariantResponse } from '@epic-web/invariant'
 import { prisma } from '#app/utils/db.server.ts'
-import { getImageUrl } from '#app/utils/storage.server.ts'
+import { getSignedGetRequestInfo } from '#app/utils/storage.server.ts'
 import { type Route } from './+types/user-images.$imageId.ts'
 
 export async function loader({ params }: Route.LoaderArgs) {
@@ -10,5 +10,6 @@ export async function loader({ params }: Route.LoaderArgs) {
 		select: { storageKey: true },
 	})
 	invariantResponse(userImage, 'User image not found', { status: 404 })
-	return fetch(getImageUrl(userImage.storageKey))
+	const { url, headers } = getSignedGetRequestInfo(userImage.storageKey)
+	return fetch(url, { headers })
 }

app/utils/env.server.ts

@@ -19,10 +19,11 @@ const schema = z.object({
 	ALLOW_INDEXING: z.enum(['true', 'false']).optional(),
 
 	// Tigris Object Storage Configuration
-	STORAGE_ACCESS_KEY: z.string(),
-	STORAGE_SECRET_KEY: z.string(),
-	STORAGE_BUCKET: z.string(),
-	STORAGE_ENDPOINT: z.string().url(),
+	AWS_ACCESS_KEY_ID: z.string(),
+	AWS_SECRET_ACCESS_KEY: z.string(),
+	AWS_REGION: z.string(),
+	AWS_ENDPOINT_URL_S3: z.string().url(),
+	BUCKET_NAME: z.string(),
 })
 
 declare global {

app/utils/storage.server.ts

@@ -1,20 +1,19 @@
+import { createHash, createHmac } from 'crypto'
 import { type FileUpload } from '@mjackson/form-data-parser'
 import { createId } from '@paralleldrive/cuid2'
 
-const STORAGE_ENDPOINT = process.env.STORAGE_ENDPOINT
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
-const STORAGE_ACCESS_KEY = process.env.STORAGE_ACCESS_KEY
-const STORAGE_SECRET_KEY = process.env.STORAGE_SECRET_KEY
+const STORAGE_ENDPOINT = process.env.AWS_ENDPOINT_URL_S3
+const STORAGE_BUCKET = process.env.BUCKET_NAME
+const STORAGE_ACCESS_KEY = process.env.AWS_ACCESS_KEY_ID
+const STORAGE_SECRET_KEY = process.env.AWS_SECRET_ACCESS_KEY
+const STORAGE_REGION = process.env.AWS_REGION
 
 async function uploadToStorage(file: File | FileUpload, key: string) {
-	const url = getImageUrl(key)
+	const { url, headers } = getSignedPutRequestInfo(file, key)
+
 	const uploadResponse = await fetch(url, {
 		method: 'PUT',
-		headers: {
-			'Content-Type': file.type,
-			Authorization: `Basic ${btoa(`${STORAGE_ACCESS_KEY}:${STORAGE_SECRET_KEY}`)}`,
-			'x-amz-meta-upload-date': new Date().toISOString(),
-		},
+		headers,
 		body: file instanceof File ? file : Buffer.from(await file.arrayBuffer()),
 	})
 
@@ -50,6 +49,123 @@ export async function uploadNoteImage(
 	return uploadToStorage(file, key)
 }
 
-export function getImageUrl(imageId: string) {
+function getImageUrl(imageId: string) {
 	return `${STORAGE_ENDPOINT}/${STORAGE_BUCKET}/${imageId}`
 }
+
+function hmacSha256(key: string | Buffer, message: string) {
+	const hmac = createHmac('sha256', key)
+	hmac.update(message)
+	return hmac.digest()
+}
+
+function sha256(message: string) {
+	const hash = createHash('sha256')
+	hash.update(message)
+	return hash.digest('hex')
+}
+
+function getSignatureKey(
+	key: string,
+	dateStamp: string,
+	regionName: string,
+	serviceName: string,
+) {
+	const kDate = hmacSha256(`AWS4${key}`, dateStamp)
+	const kRegion = hmacSha256(kDate, regionName)
+	const kService = hmacSha256(kRegion, serviceName)
+	const kSigning = hmacSha256(kService, 'aws4_request')
+	return kSigning
+}
+
+function getBaseSignedRequestInfo({
+	method,
+	key,
+	contentType,
+	uploadDate,
+}: {
+	method: 'GET' | 'PUT'
+	key: string
+	contentType?: string
+	uploadDate?: string
+}) {
+	const url = getImageUrl(key)
+	const endpoint = new URL(url)
+
+	// Prepare date strings
+	const amzDate = new Date().toISOString().replace(/[:-]|\.\d{3}/g, '')
+	const dateStamp = amzDate.slice(0, 8)
+
+	// Build headers array conditionally
+	const headers = [
+		...(contentType ? [`content-type:${contentType}`] : []),
+		`host:${endpoint.host}`,
+		`x-amz-content-sha256:UNSIGNED-PAYLOAD`,
+		`x-amz-date:${amzDate}`,
+		...(uploadDate ? [`x-amz-meta-upload-date:${uploadDate}`] : []),
+	]
+
+	const canonicalHeaders = headers.join('\n') + '\n'
+	const signedHeaders = headers.map((h) => h.split(':')[0]).join(';')
+
+	const canonicalRequest = [
+		method,
+		`/${STORAGE_BUCKET}/${key}`,
+		'', // canonicalQueryString
+		canonicalHeaders,
+		signedHeaders,
+		'UNSIGNED-PAYLOAD',
+	].join('\n')
+
+	// Prepare string to sign
+	const algorithm = 'AWS4-HMAC-SHA256'
+	const credentialScope = `${dateStamp}/${STORAGE_REGION}/s3/aws4_request`
+	const stringToSign = [
+		algorithm,
+		amzDate,
+		credentialScope,
+		sha256(canonicalRequest),
+	].join('\n')
+
+	// Calculate signature
+	const signingKey = getSignatureKey(
+		STORAGE_SECRET_KEY,
+		dateStamp,
+		STORAGE_REGION,
+		's3',
+	)
+	const signature = createHmac('sha256', signingKey)
+		.update(stringToSign)
+		.digest('hex')
+
+	const baseHeaders = {
+		'X-Amz-Date': amzDate,
+		'X-Amz-Content-SHA256': 'UNSIGNED-PAYLOAD',
+		Authorization: [
+			`${algorithm} Credential=${STORAGE_ACCESS_KEY}/${credentialScope}`,
+			`SignedHeaders=${signedHeaders}`,
+			`Signature=${signature}`,
+		].join(', '),
+	}
+
+	return { url, baseHeaders }
+}
+
+function getSignedPutRequestInfo(file: File | FileUpload, key: string) {
+	const uploadDate = new Date().toISOString()
+	const { url, baseHeaders } = getBaseSignedRequestInfo({
+		method: 'PUT',
+		key,
+		contentType: file.type,
+		uploadDate,
+	})
+
+	return {
+		url,
+		headers: {
+			...baseHeaders,
+			'Content-Type': file.type,
+			'X-Amz-Meta-Upload-Date': uploadDate,
+		},
+	}
+}

docs/deployment.md

@@ -108,7 +108,9 @@ Prior to your first deployment, you'll need to do a few things:
 
    This will create a Tigris object storage bucket for both your production and
    staging environments. The bucket will be used for storing uploaded files and
-   other objects in your application.
+   other objects in your application. This will also automatically create the
+   necessary environment variables for your app. During local development, this
+   is completely mocked out so you don't need to worry about it.
 
 9. Commit!
 

package.json

@@ -16,6 +16,7 @@
     "build:server": "tsx ./other/build-server.ts",
     "predev": "npm run build:icons --silent",
     "dev": "cross-env NODE_ENV=development MOCKS=true node ./server/dev-server.js",
+    "dev:no-mocks": "cross-env NODE_ENV=development node ./server/dev-server.js",
     "prisma:studio": "prisma studio",
     "format": "prettier --write .",
     "lint": "eslint .",

server/dev-server.js

@@ -10,7 +10,6 @@ if (process.env.NODE_ENV === 'production') {
 		shell: true,
 		env: {
 			FORCE_COLOR: true,
-			MOCKS: true,
 			...process.env,
 		},
 		// https://github.com/sindresorhus/execa/issues/433

tests/mocks/tigris.ts

@@ -12,10 +12,10 @@ const __dirname = path.dirname(__filename)
 const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures')
 const MOCK_STORAGE_DIR = path.join(FIXTURES_DIR, 'uploaded')
 const FIXTURES_IMAGES_DIR = path.join(FIXTURES_DIR, 'images')
-const STORAGE_ENDPOINT = process.env.STORAGE_ENDPOINT
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
-const STORAGE_ACCESS_KEY = process.env.STORAGE_ACCESS_KEY
-const STORAGE_SECRET_KEY = process.env.STORAGE_SECRET_KEY
+const STORAGE_ENDPOINT = process.env.AWS_ENDPOINT_URL_S3
+const STORAGE_BUCKET = process.env.BUCKET_NAME
+const STORAGE_ACCESS_KEY = process.env.AWS_ACCESS_KEY_ID
+const STORAGE_SECRET_KEY = process.env.AWS_SECRET_ACCESS_KEY
 
 function validateAuth(headers: Headers) {
 	const authHeader = headers.get('Authorization')