Merge pull request #2888 from SeedCompany/dev-upload

Author: CarsonF

package.json

@@ -63,10 +63,12 @@
     "extensionless": "^1.4.5",
     "fast-safe-stringify": "^2.1.1",
     "fastest-levenshtein": "^1.0.16",
+    "file-type": "^18.5.0",
     "got": "^13.0.0",
     "graphql": "^16.8.0",
     "graphql-parse-resolve-info": "^4.13.0",
     "graphql-scalars": "^1.22.2",
+    "graphql-upload": "^16.0.2",
     "human-format": "^1.2.0",
     "image-size": "^1.0.2",
     "ioredis": "^5.3.2",
@@ -77,6 +79,7 @@
     "lodash": "npm:lodash-es@^4.17.21",
     "lru-cache": "^7.18.3",
     "luxon": "^3.4.0",
+    "mime": "beta",
     "nanoid": "^4.0.2",
     "neo4j-driver": "^5.12.0",
     "nestjs-console": "^9.0.0",
@@ -91,6 +94,7 @@
     "reflect-metadata": "^0.1.13",
     "rimraf": "^5.0.1",
     "rxjs": "^7.8.1",
+    "sanitize-filename": "^1.6.3",
     "ts-essentials": "^9.3.2",
     "winston": "^3.10.0",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.0/xlsx-0.20.0.tgz",
@@ -107,6 +111,7 @@
     "@types/express": "^4.17.17",
     "@types/express-serve-static-core": "^4.17.35",
     "@types/ffprobe": "^1.1.5",
+    "@types/graphql-upload": "^16.0.1",
     "@types/jest": "^29.5.3",
     "@types/jsonwebtoken": "^9.0.2",
     "@types/lodash": "^4.14.197",

src/common/scalars.ts

@@ -1,6 +1,7 @@
 import { Type } from '@nestjs/common';
 import { CustomScalar } from '@nestjs/graphql';
 import { GraphQLScalarType } from 'graphql';
+import UploadScalar from 'graphql-upload/GraphQLUpload.mjs';
 import { DateScalar, DateTimeScalar } from './luxon.graphql';
 import { RichTextScalar } from './rich-text.scalar';
 import { UrlScalar } from './url.field';
@@ -12,5 +13,6 @@ export const getRegisteredScalars = (): Scalar[] => [
   DateScalar,
   DateTimeScalar,
   RichTextScalar,
+  UploadScalar,
   UrlScalar,
 ];

src/components/engagement/events/engagement-updated.event.ts

@@ -11,20 +11,20 @@ export class EngagementUpdatedEvent {
   constructor(
     public updated: UnsecuredDto<Engagement>,
     readonly previous: UnsecuredDto<Engagement>,
-    readonly updates: UpdateLanguageEngagement | UpdateInternshipEngagement,
+    readonly input: UpdateLanguageEngagement | UpdateInternshipEngagement,
     readonly session: Session,
   ) {}
 
   isLanguageEngagement(): this is EngagementUpdatedEvent & {
-    engagement: UnsecuredDto<LanguageEngagement>;
-    updates: UpdateLanguageEngagement;
+    updated: UnsecuredDto<LanguageEngagement>;
+    input: UpdateLanguageEngagement;
   } {
     return this.updated.__typename === 'LanguageEngagement';
   }
 
   isInternshipEngagement(): this is EngagementUpdatedEvent & {
-    engagement: UnsecuredDto<InternshipEngagement>;
-    updates: UpdateInternshipEngagement;
+    updated: UnsecuredDto<InternshipEngagement>;
+    input: UpdateInternshipEngagement;
   } {
     return this.updated.__typename === 'InternshipEngagement';
   }

src/components/file/bucket/composite-bucket.ts

@@ -2,7 +2,12 @@ import { HeadObjectOutput } from '@aws-sdk/client-s3';
 import { Type } from '@nestjs/common';
 import { Command } from '@smithy/smithy-client';
 import { NotFoundException } from '~/common';
-import { FileBucket, GetObjectOutput, SignedOp } from './file-bucket';
+import {
+  FileBucket,
+  GetObjectOutput,
+  PutObjectInput,
+  SignedOp,
+} from './file-bucket';
 
 /**
  * A bucket that is composed of multiple other sources.
@@ -46,6 +51,12 @@ export class CompositeBucket extends FileBucket {
     return output;
   }
 
+  async putObject(input: PutObjectInput): Promise<void> {
+    await this.doAndThrowAllErrors(
+      this.writableSources.map((bucket) => bucket.putObject(input)),
+    );
+  }
+
   async copyObject(oldKey: string, newKey: string): Promise<void> {
     const [existing] = await this.selectSources(oldKey, this.writableSources);
     await this.doAndThrowAllErrors(

src/components/file/bucket/file-bucket.ts

@@ -1,17 +1,29 @@
 import {
   GetObjectOutput as AwsGetObjectOutput,
+  PutObjectCommandInput as AwsPutObjectCommandInput,
   HeadObjectOutput,
 } from '@aws-sdk/client-s3';
 import { RequestPresigningArguments } from '@aws-sdk/types';
 import { Type } from '@nestjs/common';
 import { Command } from '@smithy/smithy-client';
+import { NodeJsRuntimeStreamingBlobPayloadInputTypes } from '@smithy/types/dist-types/streaming-payload/streaming-blob-payload-input-types';
 import { Readable } from 'stream';
-import { Merge } from 'type-fest';
+import { Except, Merge, SetNonNullable, SetRequired } from 'type-fest';
 import { DurationIn } from '~/common';
 
 // Limit body to only `Readable` which is always the case for Nodejs execution.
 export type GetObjectOutput = Merge<AwsGetObjectOutput, { Body: Readable }>;
 
+export type PutObjectInput = Merge<
+  SetNonNullable<
+    SetRequired<Except<AwsPutObjectCommandInput, 'Bucket'>, 'ContentType'>,
+    'Key'
+  >,
+  {
+    Body: NodeJsRuntimeStreamingBlobPayloadInputTypes;
+  }
+>;
+
 export type SignedOp<T extends object> = Omit<T, 'Bucket'> & {
   Key: string;
   signing: Merge<RequestPresigningArguments, { expiresIn: DurationIn }>;
@@ -32,6 +44,7 @@ export abstract class FileBucket {
 
   abstract getObject(key: string): Promise<GetObjectOutput>;
   abstract headObject(key: string): Promise<HeadObjectOutput>;
+  abstract putObject(input: PutObjectInput): Promise<void>;
   abstract copyObject(oldKey: string, newKey: string): Promise<void>;
   abstract deleteObject(key: string): Promise<void>;
   async moveObject(oldKey: string, newKey: string) {

src/components/file/bucket/local-bucket.ts

@@ -3,13 +3,20 @@ import {
   PutObjectCommand as PutObject,
 } from '@aws-sdk/client-s3';
 import { Type } from '@nestjs/common';
+import { bufferFromStream } from '@seedcompany/common';
 import { Command } from '@smithy/smithy-client';
 import { pickBy } from 'lodash';
 import { DateTime, Duration } from 'luxon';
 import { URL } from 'node:url';
+import { Readable } from 'stream';
 import { assert } from 'ts-essentials';
 import { InputException } from '~/common';
-import { FileBucket, GetObjectOutput, SignedOp } from './file-bucket';
+import {
+  FileBucket,
+  GetObjectOutput,
+  PutObjectInput,
+  SignedOp,
+} from './file-bucket';
 
 export interface LocalBucketOptions {
   baseUrl: URL;
@@ -62,6 +69,19 @@ export abstract class LocalBucket<
 
   protected abstract saveFile(key: string, file: FakeAwsFile): Promise<void>;
 
+  async putObject(input: PutObjectInput) {
+    const buffer =
+      input.Body instanceof Readable
+        ? await bufferFromStream(input.Body)
+        : Buffer.from(input.Body);
+    await this.saveFile(input.Key, {
+      LastModified: new Date(),
+      ...input,
+      Body: buffer,
+      ContentLength: buffer.byteLength,
+    });
+  }
+
   async getSignedUrl<TCommandInput extends object>(
     operation: Type<Command<TCommandInput, any, any>>,
     input: SignedOp<TCommandInput>,

src/components/file/bucket/readonly-bucket.ts

@@ -26,6 +26,10 @@ export class ReadonlyBucket extends FileBucket {
     return await this.source.headObject(key);
   }
 
+  async putObject(_input: unknown) {
+    throw new Error('File bucket is readonly and cannot put objects');
+  }
+
   async copyObject(_oldKey: string, _newKey: string) {
     throw new Error('File bucket is readonly and cannot copy objects');
   }

src/components/file/bucket/s3-bucket.ts

@@ -1,12 +1,13 @@
 import { NoSuchKey, S3 } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Type } from '@nestjs/common';
+import { bufferFromStream } from '@seedcompany/common';
 import { Command } from '@smithy/smithy-client';
 import { Duration } from 'luxon';
 import { join } from 'path/posix';
 import { Readable } from 'stream';
 import { NotFoundException } from '~/common';
-import { FileBucket, SignedOp } from './file-bucket';
+import { FileBucket, PutObjectInput, SignedOp } from './file-bucket';
 
 /**
  * A bucket that actually connects to S3.
@@ -58,6 +59,22 @@ export class S3Bucket extends FileBucket {
       .catch(handleNotFound);
   }
 
+  async putObject(input: PutObjectInput) {
+    // S3 needs to know the content length either from body or the header.
+    // Since we streams don't have that, and we don't know from file, we need to
+    // buffer it. This way we can know the length to send to S3.
+    const fixedLengthBody =
+      input.Body instanceof Readable
+        ? await bufferFromStream(input.Body)
+        : input.Body;
+    await this.s3.putObject({
+      ...input,
+      Key: this.fullKey(input.Key),
+      Bucket: this.bucket,
+      Body: fixedLengthBody,
+    });
+  }
+
   async copyObject(oldKey: string, newKey: string) {
     await this.s3
       .copyObject({

src/components/file/dto/upload.dto.ts

@@ -1,4 +1,7 @@
 import { Field, InputType, ObjectType } from '@nestjs/graphql';
+import { stripIndent } from 'common-tags';
+import UploadScalar from 'graphql-upload/GraphQLUpload.mjs';
+import type { FileUpload } from 'graphql-upload/Upload.mjs';
 import { ID, IdField } from '~/common';
 import { MediaUserMetadata } from '../media/media.dto';
 
@@ -8,22 +11,44 @@ export abstract class RequestUploadOutput {
   readonly id: ID;
 
   @Field({
-    description: 'A pre-signed url to upload the file to',
+    description: stripIndent`
+      A temporary url to upload the file to.
+      It should be a an HTTP PUT request with the file as the body.
+      The Content-Type header should be set to the mime type of the file.
+    `,
   })
   readonly url: string;
 }
 
 @InputType()
 export abstract class CreateDefinedFileVersionInput {
   @IdField({
-    description: 'The ID returned from the `requestFileUpload` mutation',
+    description: stripIndent`
+      The ID returned from the \`requestFileUpload\` mutation.
+      This _can_ be skipped if \`file\` is provided.
+    `,
+    nullable: true,
+  })
+  readonly uploadId?: ID;
+
+  @Field(() => UploadScalar, {
+    description: stripIndent`
+      A file directly uploaded.
+      This is mainly here to allow usage with Apollo Studio/Sandbox.
+      For production, prefer the \`url\` from the \`RequestUploadOutput\`.
+    `,
+    nullable: true,
   })
-  readonly uploadId: ID;
+  readonly file?: Promise<FileUpload>;
 
   @Field({
-    description: 'The file name',
+    description: stripIndent`
+      The file name. This is generally required.
+      It's only optional if \`file\` is provided.
+    `,
+    nullable: true,
   })
-  readonly name: string;
+  readonly name?: string;
 
   @Field({
     description:

src/components/file/events/after-file-upload.event.ts

@@ -1,9 +1,9 @@
-import { File } from '../dto';
+import { File, FileVersion } from '../dto';
 
 /**
  * Emitted as the last step of the file upload process.
  * Feel free to throw to abort mutation.
  */
 export class AfterFileUploadEvent {
-  constructor(readonly file: File) {}
+  constructor(readonly file: File, readonly newVersion: FileVersion) {}
 }