Migrate from storing images in SQLite to Tigris

.env.example

@@ -19,3 +19,10 @@ GITHUB_TOKEN="MOCK_GITHUB_TOKEN"
 # set this to false to prevent search engines from indexing the website
 # default to allow indexing for seo safety
 ALLOW_INDEXING="true"
+
+# Tigris Object Storage (S3-compatible) Configuration
+AWS_ACCESS_KEY_ID="mock-access-key"
+AWS_SECRET_ACCESS_KEY="mock-secret-key"
+AWS_REGION="auto"
+AWS_ENDPOINT_URL_S3="https://fly.storage.tigris.dev"
+BUCKET_NAME="mock-bucket"

.gitignore

@@ -14,6 +14,7 @@ node_modules
 /playwright-report/
 /playwright/.cache/
 /tests/fixtures/email/
+/tests/fixtures/uploaded/
 /coverage
 
 /other/cache.db

app/routes/resources+/download-user-data.tsx

@@ -18,7 +18,7 @@ export async function loader({ request }: Route.LoaderArgs) {
 					id: true,
 					createdAt: true,
 					updatedAt: true,
-					contentType: true,
+					objectKey: true,
 				},
 			},
 			notes: {
@@ -28,7 +28,7 @@ export async function loader({ request }: Route.LoaderArgs) {
 							id: true,
 							createdAt: true,
 							updatedAt: true,
-							contentType: true,
+							objectKey: true,
 						},
 					},
 				},

app/routes/resources+/note-images.$imageId.tsx

@@ -1,22 +1,24 @@
 import { invariantResponse } from '@epic-web/invariant'
 import { prisma } from '#app/utils/db.server.ts'
+import { getSignedGetRequestInfo } from '#app/utils/storage.server.ts'
 import { type Route } from './+types/note-images.$imageId.ts'
 
 export async function loader({ params }: Route.LoaderArgs) {
 	invariantResponse(params.imageId, 'Image ID is required', { status: 400 })
-	const image = await prisma.noteImage.findUnique({
+	const noteImage = await prisma.noteImage.findUnique({
 		where: { id: params.imageId },
-		select: { contentType: true, blob: true },
+		select: { objectKey: true },
 	})
+	invariantResponse(noteImage, 'Note image not found', { status: 404 })
 
-	invariantResponse(image, 'Not found', { status: 404 })
+	const { url, headers } = getSignedGetRequestInfo(noteImage.objectKey)
+	const response = await fetch(url, { headers })
 
-	return new Response(image.blob, {
-		headers: {
-			'Content-Type': image.contentType,
-			'Content-Length': Buffer.byteLength(image.blob).toString(),
-			'Content-Disposition': `inline; filename="${params.imageId}"`,
-			'Cache-Control': 'public, max-age=31536000, immutable',
-		},
+	const cacheHeaders = new Headers(response.headers)
+	cacheHeaders.set('Cache-Control', 'public, max-age=31536000, immutable')
+
+	return new Response(response.body, {
+		status: response.status,
+		headers: cacheHeaders,
 	})
 }

app/routes/resources+/user-images.$imageId.tsx

@@ -1,22 +1,24 @@
 import { invariantResponse } from '@epic-web/invariant'
 import { prisma } from '#app/utils/db.server.ts'
+import { getSignedGetRequestInfo } from '#app/utils/storage.server.ts'
 import { type Route } from './+types/user-images.$imageId.ts'
 
 export async function loader({ params }: Route.LoaderArgs) {
 	invariantResponse(params.imageId, 'Image ID is required', { status: 400 })
-	const image = await prisma.userImage.findUnique({
+	const userImage = await prisma.userImage.findUnique({
 		where: { id: params.imageId },
-		select: { contentType: true, blob: true },
+		select: { objectKey: true },
 	})
+	invariantResponse(userImage, 'User image not found', { status: 404 })
 
-	invariantResponse(image, 'Not found', { status: 404 })
+	const { url, headers } = getSignedGetRequestInfo(userImage.objectKey)
+	const response = await fetch(url, { headers })
 
-	return new Response(image.blob, {
-		headers: {
-			'Content-Type': image.contentType,
-			'Content-Length': Buffer.byteLength(image.blob).toString(),
-			'Content-Disposition': `inline; filename="${params.imageId}"`,
-			'Cache-Control': 'public, max-age=31536000, immutable',
-		},
+	const cacheHeaders = new Headers(response.headers)
+	cacheHeaders.set('Cache-Control', 'public, max-age=31536000, immutable')
+
+	return new Response(response.body, {
+		status: response.status,
+		headers: cacheHeaders,
 	})
 }

app/routes/settings+/profile.photo.tsx

@@ -1,7 +1,7 @@
 import { getFormProps, getInputProps, useForm } from '@conform-to/react'
 import { getZodConstraint, parseWithZod } from '@conform-to/zod'
 import { invariantResponse } from '@epic-web/invariant'
-import { type FileUpload, parseFormData } from '@mjackson/form-data-parser'
+import { parseFormData } from '@mjackson/form-data-parser'
 import { type SEOHandle } from '@nasa-gcn/remix-seo'
 import { useState } from 'react'
 import { data, redirect, Form, useNavigation } from 'react-router'
@@ -12,12 +12,12 @@ import { Icon } from '#app/components/ui/icon.tsx'
 import { StatusButton } from '#app/components/ui/status-button.tsx'
 import { requireUserId } from '#app/utils/auth.server.ts'
 import { prisma } from '#app/utils/db.server.ts'
-import { uploadHandler } from '#app/utils/file-uploads.server.ts'
 import {
 	getUserImgSrc,
 	useDoubleCheck,
 	useIsPending,
 } from '#app/utils/misc.tsx'
+import { uploadProfileImage } from '#app/utils/storage.server.ts'
 import { type Route } from './+types/profile.photo.ts'
 import { type BreadcrumbHandle } from './profile.tsx'
 
@@ -66,11 +66,7 @@ export async function loader({ request }: Route.LoaderArgs) {
 export async function action({ request }: Route.ActionArgs) {
 	const userId = await requireUserId(request)
 
-	const formData = await parseFormData(
-		request,
-		{ maxFileSize: MAX_SIZE },
-		async (file: FileUpload) => uploadHandler(file),
-	)
+	const formData = await parseFormData(request, { maxFileSize: MAX_SIZE })
 	const submission = await parseWithZod(formData, {
 		schema: PhotoFormSchema.transform(async (data) => {
 			if (data.intent === 'delete') return { intent: 'delete' }
@@ -79,7 +75,7 @@ export async function action({ request }: Route.ActionArgs) {
 				intent: data.intent,
 				image: {
 					contentType: data.photoFile.type,
-					blob: Buffer.from(await data.photoFile.arrayBuffer()),
+					objectKey: await uploadProfileImage(userId, data.photoFile),
 				},
 			}
 		}),

app/routes/users+/$username.test.tsx

@@ -26,6 +26,7 @@ test('The user profile when not logged in as self', async () => {
 			path: '/users/:username',
 			Component: UsernameRoute,
 			loader,
+			HydrateFallback: () => <div>Loading...</div>,
 		},
 	])
 
@@ -70,6 +71,7 @@ test('The user profile when logged in as self', async () => {
 				args.request.headers.set('cookie', cookieHeader)
 				return rootLoader({ ...args, context: args.context })
 			},
+			HydrateFallback: () => <div>Loading...</div>,
 			children: [
 				{
 					path: 'users/:username',
@@ -85,7 +87,7 @@ test('The user profile when logged in as self', async () => {
 	])
 
 	const routeUrl = `/users/${user.username}`
-	await render(<App initialEntries={[routeUrl]} />)
+	render(<App initialEntries={[routeUrl]} />)
 
 	await screen.findByRole('heading', { level: 1, name: user.name! })
 	await screen.findByRole('img', { name: user.name! })

app/routes/users+/$username_+/__note-editor.server.tsx

@@ -1,11 +1,11 @@
 import { parseWithZod } from '@conform-to/zod'
-import { type FileUpload, parseFormData } from '@mjackson/form-data-parser'
+import { parseFormData } from '@mjackson/form-data-parser'
 import { createId as cuid } from '@paralleldrive/cuid2'
 import { data, redirect, type ActionFunctionArgs } from 'react-router'
 import { z } from 'zod'
 import { requireUserId } from '#app/utils/auth.server.ts'
 import { prisma } from '#app/utils/db.server.ts'
-import { uploadHandler } from '#app/utils/file-uploads.server.ts'
+import { uploadNoteImage } from '#app/utils/storage.server.ts'
 import {
 	MAX_UPLOAD_SIZE,
 	NoteEditorSchema,
@@ -20,18 +20,16 @@ function imageHasFile(
 
 function imageHasId(
 	image: ImageFieldset,
-): image is ImageFieldset & { id: NonNullable<ImageFieldset['id']> } {
-	return image.id != null
+): image is ImageFieldset & { id: string } {
+	return Boolean(image.id)
 }
 
 export async function action({ request }: ActionFunctionArgs) {
 	const userId = await requireUserId(request)
 
-	const formData = await parseFormData(
-		request,
-		{ maxFileSize: MAX_UPLOAD_SIZE },
-		async (file: FileUpload) => uploadHandler(file),
-	)
+	const formData = await parseFormData(request, {
+		maxFileSize: MAX_UPLOAD_SIZE,
+	})
 
 	const submission = await parseWithZod(formData, {
 		schema: NoteEditorSchema.superRefine(async (data, ctx) => {
@@ -48,16 +46,18 @@ export async function action({ request }: ActionFunctionArgs) {
 				})
 			}
 		}).transform(async ({ images = [], ...data }) => {
+			const noteId = data.id ?? cuid()
 			return {
 				...data,
+				id: noteId,
 				imageUpdates: await Promise.all(
 					images.filter(imageHasId).map(async (i) => {
 						if (imageHasFile(i)) {
 							return {
 								id: i.id,
 								altText: i.altText,
 								contentType: i.file.type,
-								blob: Buffer.from(await i.file.arrayBuffer()),
+								objectKey: await uploadNoteImage(userId, noteId, i.file),
 							}
 						} else {
 							return {
@@ -75,7 +75,7 @@ export async function action({ request }: ActionFunctionArgs) {
 							return {
 								altText: image.altText,
 								contentType: image.file.type,
-								blob: Buffer.from(await image.file.arrayBuffer()),
+								objectKey: await uploadNoteImage(userId, noteId, image.file),
 							}
 						}),
 				),
@@ -101,8 +101,9 @@ export async function action({ request }: ActionFunctionArgs) {
 
 	const updatedNote = await prisma.note.upsert({
 		select: { id: true, owner: { select: { username: true } } },
-		where: { id: noteId ?? '__new_note__' },
+		where: { id: noteId },
 		create: {
+			id: noteId,
 			ownerId: userId,
 			title,
 			content,
@@ -115,7 +116,11 @@ export async function action({ request }: ActionFunctionArgs) {
 				deleteMany: { id: { notIn: imageUpdates.map((i) => i.id) } },
 				updateMany: imageUpdates.map((updates) => ({
 					where: { id: updates.id },
-					data: { ...updates, id: updates.blob ? cuid() : updates.id },
+					data: {
+						...updates,
+						// If the image is new, we need to generate a new ID to bust the cache.
+						id: updates.objectKey ? cuid() : updates.id,
+					},
 				})),
 				create: newImages,
 			},

app/utils/auth.server.ts

@@ -8,6 +8,7 @@ import { prisma } from './db.server.ts'
 import { combineHeaders, downloadFile } from './misc.tsx'
 import { type ProviderUser } from './providers/provider.ts'
 import { authSessionStorage } from './session.server.ts'
+import { uploadProfileImage } from './storage.server.ts'
 
 export const SESSION_EXPIRATION_TIME = 1000 * 60 * 60 * 24 * 30
 export const getSessionExpirationDate = () =>
@@ -160,21 +161,36 @@ export async function signupWithConnection({
 	providerName: Connection['providerName']
 	imageUrl?: string
 }) {
-	const session = await prisma.session.create({
+	const user = await prisma.user.create({
 		data: {
-			expirationDate: getSessionExpirationDate(),
-			user: {
-				create: {
-					email: email.toLowerCase(),
-					username: username.toLowerCase(),
-					name,
-					roles: { connect: { name: 'user' } },
-					connections: { create: { providerId, providerName } },
-					image: imageUrl
-						? { create: await downloadFile(imageUrl) }
-						: undefined,
+			email: email.toLowerCase(),
+			username: username.toLowerCase(),
+			name,
+			roles: { connect: { name: 'user' } },
+			connections: { create: { providerId, providerName } },
+		},
+		select: { id: true },
+	})
+
+	if (imageUrl) {
+		const imageFile = await downloadFile(imageUrl)
+		await prisma.user.update({
+			where: { id: user.id },
+			data: {
+				image: {
+					create: {
+						objectKey: await uploadProfileImage(user.id, imageFile),
+					},
 				},
 			},
+		})
+	}
+
+	// Create and return the session
+	const session = await prisma.session.create({
+		data: {
+			expirationDate: getSessionExpirationDate(),
+			userId: user.id,
 		},
 		select: { id: true, expirationDate: true },
 	})

app/utils/env.server.ts

@@ -17,6 +17,13 @@ const schema = z.object({
 	GITHUB_CLIENT_SECRET: z.string().default('MOCK_GITHUB_CLIENT_SECRET'),
 	GITHUB_TOKEN: z.string().default('MOCK_GITHUB_TOKEN'),
 	ALLOW_INDEXING: z.enum(['true', 'false']).optional(),
+
+	// Tigris Object Storage Configuration
+	AWS_ACCESS_KEY_ID: z.string(),
+	AWS_SECRET_ACCESS_KEY: z.string(),
+	AWS_REGION: z.string(),
+	AWS_ENDPOINT_URL_S3: z.string().url(),
+	BUCKET_NAME: z.string(),
 })
 
 declare global {

app/utils/misc.tsx

@@ -281,8 +281,11 @@ export async function downloadFile(url: string, retries: number = 0) {
 			throw new Error(`Failed to fetch image with status ${response.status}`)
 		}
 		const contentType = response.headers.get('content-type') ?? 'image/jpg'
-		const blob = Buffer.from(await response.arrayBuffer())
-		return { contentType, blob }
+		const arrayBuffer = await response.arrayBuffer()
+		const file = new File([arrayBuffer], 'downloaded-file', {
+			type: contentType,
+		})
+		return file
 	} catch (e) {
 		if (retries > MAX_RETRIES) throw e
 		return downloadFile(url, retries + 1)