Merge pull request #2891 from SeedCompany/local-file-bucket-urls-compatibility-with-interface

Author: CarsonF

src/components/file/bucket/composite-bucket.ts

@@ -5,6 +5,7 @@ import { NotFoundException } from '~/common';
 import {
   FileBucket,
   GetObjectOutput,
+  InvalidSignedUrlException,
   PutObjectInput,
   SignedOp,
 } from './file-bucket';
@@ -41,6 +42,18 @@ export class CompositeBucket extends FileBucket {
     return await source.getSignedUrl(operation, input);
   }
 
+  async parseSignedUrl(url: URL) {
+    const results = await Promise.allSettled(
+      this.sources.map(async (source) => await source.parseSignedUrl(url)),
+    );
+    for (const result of results) {
+      if (result.status === 'fulfilled') {
+        return result.value;
+      }
+    }
+    throw new InvalidSignedUrlException();
+  }
+
   async getObject(key: string): Promise<GetObjectOutput> {
     const [source] = await this.selectSource(key);
     return await source.getObject(key);

src/components/file/bucket/file-bucket.ts

@@ -5,11 +5,18 @@ import {
 } from '@aws-sdk/client-s3';
 import { RequestPresigningArguments } from '@aws-sdk/types';
 import { Type } from '@nestjs/common';
+import { MaybeAsync } from '@seedcompany/common';
 import { Command } from '@smithy/smithy-client';
 import { NodeJsRuntimeStreamingBlobPayloadInputTypes } from '@smithy/types/dist-types/streaming-payload/streaming-blob-payload-input-types';
 import { Readable } from 'stream';
-import { Except, Merge, SetNonNullable, SetRequired } from 'type-fest';
-import { DurationIn } from '~/common';
+import {
+  Except,
+  LiteralUnion,
+  Merge,
+  SetNonNullable,
+  SetRequired,
+} from 'type-fest';
+import { DurationIn, InputException, InputExceptionArgs } from '~/common';
 
 // Limit body to only `Readable` which is always the case for Nodejs execution.
 export type GetObjectOutput = Merge<AwsGetObjectOutput, { Body: Readable }>;
@@ -41,6 +48,10 @@ export abstract class FileBucket {
     operation: Type<Command<TCommandInput, any, any>>,
     input: SignedOp<TCommandInput>,
   ): Promise<string>;
+  abstract parseSignedUrl(url: URL): MaybeAsync<{
+    Key: string;
+    operation: LiteralUnion<'PutObject' | 'GetObject', string>;
+  }>;
 
   abstract getObject(key: string): Promise<GetObjectOutput>;
   abstract headObject(key: string): Promise<HeadObjectOutput>;
@@ -52,3 +63,9 @@ export abstract class FileBucket {
     await this.deleteObject(oldKey);
   }
 }
+
+export class InvalidSignedUrlException extends InputException {
+  constructor(...args: InputExceptionArgs) {
+    super(...InputException.parseArgs('Invalid signed URL', args));
+  }
+}

src/components/file/bucket/local-bucket.ts

@@ -10,10 +10,10 @@ import { DateTime, Duration } from 'luxon';
 import { URL } from 'node:url';
 import { Readable } from 'stream';
 import { assert } from 'ts-essentials';
-import { InputException } from '~/common';
 import {
   FileBucket,
   GetObjectOutput,
+  InvalidSignedUrlException,
   PutObjectInput,
   SignedOp,
 } from './file-bucket';
@@ -87,7 +87,7 @@ export abstract class LocalBucket<
     input: SignedOp<TCommandInput>,
   ) {
     const signed = JSON.stringify({
-      operation: operation.constructor.name,
+      operation: operation.name.replace(/Command$/, ''),
       ...input,
       signing: {
         ...input.signing,
@@ -108,27 +108,40 @@ export abstract class LocalBucket<
     operation: Type<Command<TCommandInput, any, any>>,
     url: string,
   ): SignedOp<TCommandInput> & { Key: string } {
-    let raw;
+    let u: URL;
     try {
-      raw = new URL(url).searchParams.get('signed');
+      u = new URL(url);
     } catch (e) {
-      raw = url;
+      u = new URL('http://localhost');
+      u.searchParams.set('signed', url);
     }
-    assert(typeof raw === 'string');
-    let parsed;
     try {
-      parsed = JSON.parse(raw) as SignedOp<TCommandInput> & {
+      const parsed = this.parseSignedUrl(u) as SignedOp<TCommandInput> & {
         operation: string;
-        Key: string;
       };
-      assert(parsed.operation === operation.constructor.name);
+      assert(
+        parsed.operation === operation.name ||
+          `${parsed.operation}Command` === operation.name,
+      );
+      return parsed;
+    } catch (e) {
+      throw new InvalidSignedUrlException(e);
+    }
+  }
+
+  parseSignedUrl(url: URL) {
+    const raw = url.searchParams.get('signed');
+    let parsed;
+    try {
+      parsed = JSON.parse(raw || '') as SignedOp<{ operation: string }>;
+      assert(typeof parsed.operation === 'string');
       assert(typeof parsed.Key === 'string');
       assert(typeof parsed.signing.expiresIn === 'number');
     } catch (e) {
-      throw new InputException(e);
+      throw new InvalidSignedUrlException(e);
     }
     if (DateTime.local() > DateTime.fromMillis(parsed.signing.expiresIn)) {
-      throw new InputException('url expired');
+      throw new InvalidSignedUrlException('URL expired');
     }
     return parsed;
   }

src/components/file/bucket/readonly-bucket.ts

@@ -18,6 +18,10 @@ export class ReadonlyBucket extends FileBucket {
     return await this.source.getSignedUrl(operation, input);
   }
 
+  async parseSignedUrl(url: URL) {
+    return await this.source.parseSignedUrl(url);
+  }
+
   async getObject(key: string) {
     return await this.source.getObject(key);
   }

src/components/file/bucket/s3-bucket.ts

@@ -3,11 +3,17 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Type } from '@nestjs/common';
 import { bufferFromStream } from '@seedcompany/common';
 import { Command } from '@smithy/smithy-client';
+import got from 'got';
 import { Duration } from 'luxon';
 import { join } from 'path/posix';
 import { Readable } from 'stream';
 import { NotFoundException } from '~/common';
-import { FileBucket, PutObjectInput, SignedOp } from './file-bucket';
+import {
+  FileBucket,
+  InvalidSignedUrlException,
+  PutObjectInput,
+  SignedOp,
+} from './file-bucket';
 
 /**
  * A bucket that actually connects to S3.
@@ -37,6 +43,25 @@ export class S3Bucket extends FileBucket {
     });
   }
 
+  async parseSignedUrl(url: URL) {
+    if (
+      !url.hostname.startsWith(this.bucket + '.') ||
+      !url.hostname.endsWith('.amazonaws.com')
+    ) {
+      throw new InvalidSignedUrlException();
+    }
+
+    try {
+      await got.head(url);
+    } catch (e) {
+      throw new InvalidSignedUrlException(e);
+    }
+
+    const Key = url.pathname.slice(1);
+    const operation = url.searchParams.get('x-id')!;
+    return { Key, operation };
+  }
+
   async getObject(key: string) {
     const file = await this.s3
       .getObject({

src/components/file/local-bucket.controller.ts

@@ -3,55 +3,65 @@ import {
   Get,
   Headers,
   Put,
-  Query,
   Request,
   Response,
 } from '@nestjs/common';
 import { Request as IRequest, Response as IResponse } from 'express';
 import { DateTime } from 'luxon';
+import { URL } from 'node:url';
 import rawBody from 'raw-body';
-import { InputException, ServerException } from '../../common';
-import { FileBucket, LocalBucket } from './bucket';
+import { InputException } from '~/common';
+import { FileBucket, InvalidSignedUrlException } from './bucket';
 
+/**
+ * This fakes S3 web hosting for use with LocalBuckets.
+ */
 @Controller(LocalBucketController.path)
 export class LocalBucketController {
   static path = '/local-bucket';
 
-  private readonly bucket: LocalBucket | undefined;
-  constructor(bucket: FileBucket) {
-    this.bucket = bucket instanceof LocalBucket ? bucket : undefined;
-  }
+  constructor(private readonly bucket: FileBucket) {}
 
   @Put()
   async upload(
     @Headers('content-type') contentType: string,
-    @Query('signed') signed: string,
     @Request() req: IRequest,
   ) {
-    if (!this.bucket) {
-      throw new ServerException('Cannot upload file here');
-    }
     // Chokes on json files because they are parsed with body-parser.
     // Need to disable it for this path or create a workaround.
     const contents = await rawBody(req);
     if (!contents) {
       throw new InputException();
     }
 
-    await this.bucket.upload(signed, {
+    const url = new URL(`https://localhost${req.url}`);
+    const parsed = await this.bucket.parseSignedUrl(url);
+    if (parsed.operation !== 'PutObject') {
+      throw new InvalidSignedUrlException();
+    }
+    await this.bucket.putObject({
+      Key: parsed.Key,
       Body: contents,
       ContentType: contentType,
     });
+
     return { ok: true };
   }
 
   @Get()
-  async download(@Query('signed') signed: string, @Response() res: IResponse) {
-    if (!this.bucket) {
-      throw new ServerException('Cannot download file here');
+  async download(@Request() req: IRequest, @Response() res: IResponse) {
+    const url = new URL(`https://localhost${req.url}`);
+    const { Key, operation, ...rest } = await this.bucket.parseSignedUrl(url);
+    if (operation !== 'GetObject') {
+      throw new InvalidSignedUrlException();
     }
-
-    const out = await this.bucket.download(signed);
+    const signedParams = Object.fromEntries(
+      Object.entries(rest).flatMap(([k, v]) =>
+        v != null ? [[k.replace(/^Response/, ''), v]] : [],
+      ),
+    );
+    const bucketObject = await this.bucket.getObject(Key);
+    const out = { ...bucketObject, ...signedParams };
 
     const headers = {
       'Cache-Control': out.CacheControl,