Queues via BullMQ

.yarn/patches/ioredis-mock-npm-8.9.0-530d4422b9.patch

@@ -0,0 +1,29 @@
+diff --git a/lib/index.js b/lib/index.js
+index a82c2975d19bfcce4179d27594e26e0c47ef7a1f..2587c1b714d3b11bb52bfd708f4cb256dc511859 100644
+--- a/lib/index.js
++++ b/lib/index.js
+@@ -3824,7 +3824,7 @@ var { lua: lua2, to_luastring: toLuaString2 } = import_fengari2.default, defineR
+   let rawArgs = vm.extractArgs(), returnError = rawArgs[0], result;
+   try {
+     let args = rawArgs.slice(1), name = args[0].toLowerCase();
+-    result = commands_exports[name].bind(this)(...args.slice(1)), name === "hgetall" && (result = [].concat(...Object.entries(result)));
++    result = this.luaCommands[name].call(this, ...args.slice(1)), name === "hgetall" && (result = [].concat(...Object.entries(result)));
+   } catch (err) {
+     if (!returnError)
+       throw err;
+@@ -3845,6 +3845,7 @@ function defineArgv(vm, numberOfKeys, commandArgs) {
+ var customCommand = (numberOfKeys, luaCode) => function(...luaScriptArgs) {
+   let vm = init();
+   defineRedisObject(vm)(callToRedisCommand(vm).bind(this)), defineKeys.bind(this)(vm, numberOfKeys, luaScriptArgs), defineArgv.bind(this)(vm, numberOfKeys, luaScriptArgs);
++  this.vmInit?.(vm);
+   let topBeforeExecute = lua2.lua_gettop(vm.L);
+   vm.luaExecString(luaCode);
+   let retVal = vm.popReturnValue(topBeforeExecute);
+@@ -6243,6 +6244,7 @@ var defaultOptions = {
+     removeFrom(this.channels), removeFrom(this.patternChannels);
+   }
+   _initCommands() {
++    this.luaCommands = { ...commands_exports };
+     Object.keys(commands_exports).forEach((command3) => {
+       let commandName = command3 === "evaluate" ? "eval" : command3;
+       this[commandName] = command(

package.json

@@ -29,6 +29,7 @@
     "type-check": "tsc -p tsconfig.check.json"
   },
   "dependencies": {
+    "@aws-lambda-powertools/jmespath": "^2.31.0",
     "@aws-sdk/client-s3": "^3.440.0",
     "@aws-sdk/s3-request-presigner": "^3.440.0",
     "@faire/mjml-react": "^3.5.0",
@@ -48,6 +49,7 @@
     "@n1ru4l/graphql-live-query": "^0.10.0",
     "@n1ru4l/graphql-live-query-patch-jsondiffpatch": "^0.8.0",
     "@n1ru4l/in-memory-live-query-store": "^0.10.0",
+    "@nestjs/bullmq": "^11.0.4",
     "@nestjs/common": "^11.1.0",
     "@nestjs/core": "^11.1.0",
     "@nestjs/graphql": "^13.2.0",
@@ -61,6 +63,7 @@
     "@seedcompany/scripture": "^1.0.0",
     "argon2": "^0.43.0",
     "aws-xray-sdk-core": "^3.5.3",
+    "bullmq": "^5.69.0",
     "chalk": "^5.3.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
@@ -76,6 +79,8 @@
     "fastest-levenshtein": "^1.0.16",
     "fastify": "^5.0.0",
     "fastify-raw-body": "^5.0.0",
+    "fengari": "^0.1.5",
+    "fengari-interop": "^0.1.4",
     "file-type": "^20.5.0",
     "gel": "^2.1.0-canary.20250319T143140",
     "glob": "^11.0.2",
@@ -90,6 +95,7 @@
     "image-size": "^2.0.2",
     "indent-string": "^5.0.0",
     "ioredis": "^5.3.2",
+    "ioredis-mock": "patch:ioredis-mock@npm%3A8.9.0#~/.yarn/patches/ioredis-mock-npm-8.9.0-530d4422b9.patch",
     "iso-3166-1": "^2.1.1",
     "jsonwebtoken": "^9.0.2",
     "lazy-get-decorator": "^2.2.1",
@@ -98,6 +104,7 @@
     "luxon": "^3.7.2",
     "mime": "^4.0.7",
     "mjml": "^4.15.3",
+    "msgpackr": "^1.11.2",
     "nanoid": "^5.1.5",
     "neo4j-driver": "^5.28.1",
     "p-retry": "^6.2.1",
@@ -130,6 +137,7 @@
     "@tsconfig/strictest": "^2.0.2",
     "@types/common-tags": "^1.8.3",
     "@types/ffprobe": "^1.1.7",
+    "@types/ioredis-mock": "^8.2.5",
     "@types/jsonwebtoken": "^9.0.4",
     "@types/lodash": "^4.14.200",
     "@types/luxon": "^3.7.1",

src/common/exceptions/exception.ts

@@ -14,13 +14,3 @@ export abstract class Exception extends Error {
 export class ServerException extends Exception {}
 
 export class ClientException extends Exception {}
-
-export function getCauseList(ex: Error, includeSelf = true): readonly Error[] {
-  const previous: Error[] = includeSelf ? [ex] : [];
-  let current = ex;
-  while (current.cause instanceof Error) {
-    current = current.cause;
-    previous.push(current);
-  }
-  return previous;
-}

src/common/functions/get-cause-list.ts

@@ -0,0 +1,9 @@
+export function getCauseList(ex: Error, includeSelf = true): readonly Error[] {
+  const previous: Error[] = includeSelf ? [ex] : [];
+  let current = ex;
+  while (current.cause instanceof Error) {
+    current = current.cause;
+    previous.push(current);
+  }
+  return previous;
+}

src/common/functions/index.ts

@@ -2,6 +2,7 @@ export * from './and-call';
 export * from './first-or';
 export * from './firstLettersOfWords';
 export * from './generate-id';
+export * from './get-cause-list';
 export * from './lazy-record';
 export * from './lazy-ref';
 export * from './map-or-else';

src/common/graphql/index.ts

@@ -37,6 +37,7 @@ export * from './objects/concretes/secured-scalars';
 export * from './objects/concretes/variant.dto';
 
 export * from './objects/mappers/as-update.type';
+export * from './objects/mappers/declare-fields';
 export * from './objects/mappers/type-mappers';
 
 export * from './types/gql-context.type';

src/common/graphql/objects/mappers/declare-fields.ts

@@ -0,0 +1,18 @@
+import type { Type } from '@nestjs/common';
+import { Field, type FieldOptions, type ReturnTypeFunc } from '@nestjs/graphql';
+import { entries } from '@seedcompany/common';
+
+/**
+ * A helper to declare @Field decorators externally
+ */
+export const declareGqlFields = <T>(
+  obj: Type<T>,
+  fields: { [_ in keyof T]?: FieldOptions & { type: ReturnTypeFunc } },
+) => {
+  for (const [name, val] of entries(fields)) {
+    if (val) {
+      const { type, ...options } = val;
+      Field(type, options as FieldOptions)(obj.prototype, name);
+    }
+  }
+};

src/common/scalars/duration.scalar.ts

@@ -0,0 +1,75 @@
+import { type CustomScalar, Scalar } from '@nestjs/graphql';
+import { mapEntries, type Nil } from '@seedcompany/common';
+import type { DurationIn } from '@seedcompany/common/temporal/luxon';
+import { stripIndent } from 'common-tags';
+import { GraphQLError, Kind, type ValueNode } from 'graphql';
+import { GraphQLDuration } from 'graphql-scalars';
+import { Duration } from 'luxon';
+
+@Scalar('Duration', () => Duration)
+export class DurationScalar implements CustomScalar<string, DurationIn | null> {
+  description = (
+    stripIndent(GraphQLDuration.description!) +
+    '\n' +
+    stripIndent`
+      For inputs, ISO strings are accepted, but also many other forms:
+      An integer/float can be used to specify in milliseconds.
+      An object declaring numeric values for each unit: \`{ days: 2 }\`
+      Or a human-readable string: \`"1 day 5 hours 3 mins 2s"\`
+    `
+  ).replaceAll(/\n(?!\n)/g, '\n\n');
+
+  parseLiteral(ast: ValueNode, variables: Record<string, unknown> | Nil) {
+    if (ast.kind === Kind.NULL) {
+      return null; // idk if this is necessary
+    }
+    if (
+      ast.kind === Kind.STRING ||
+      ast.kind === Kind.FLOAT ||
+      ast.kind === Kind.INT
+    ) {
+      return Duration.from(ast.value);
+    }
+    if (ast.kind === Kind.OBJECT) {
+      return Duration.fromObject(
+        Object.fromEntries(
+          mapEntries(ast.fields, ({ name: { value: name }, value }) => {
+            if (
+              value.kind !== Kind.INT &&
+              value.kind !== Kind.FLOAT &&
+              value.kind !== Kind.VARIABLE
+            ) {
+              throw new GraphQLError(
+                `Duration object literals can only have numeric values for each unit`,
+              );
+            }
+            const parsed =
+              value.kind === Kind.VARIABLE
+                ? variables?.[value.name.value]
+                : parseFloat(value.value);
+
+            return [name, parsed];
+          }),
+        ),
+      );
+    }
+
+    throw new GraphQLError(`Durations cannot be parsed as: ${ast.kind}`);
+  }
+
+  parseValue(value: any) {
+    return Duration.from(value);
+  }
+
+  serialize(value: unknown) {
+    if (Duration.isDuration(value)) {
+      return value.toISO();
+    }
+    if (typeof value === 'string') {
+      return value;
+    }
+    throw new GraphQLError(
+      'Expected a Duration but received a: ' + typeof value,
+    );
+  }
+}

src/common/scalars/index.ts

@@ -1,13 +1,17 @@
 import { type Type } from '@nestjs/common';
 import { type CustomScalar } from '@nestjs/graphql';
 import { type GraphQLScalarType } from 'graphql';
+import { DurationScalar } from './duration.scalar';
 import { FileUploadScalar } from './file-upload.scalar';
+import { JmesPathScalar } from './jmes-path.scalar';
 import { InlineMarkdownScalar, MarkdownScalar } from './markdown.scalar';
 import { RichTextScalar } from './rich-text.scalar';
 import { DateScalar, DateTimeScalar } from './temporal.scalar';
 import { UrlScalar } from './url.scalar';
 
+export * from './duration.scalar';
 export * from './file-upload.scalar';
+export * from './jmes-path.scalar';
 export * from './temporal.scalar';
 export * from './markdown.scalar';
 export * from './rich-text.scalar';
@@ -18,6 +22,8 @@ type Scalar = GraphQLScalarType | Type<CustomScalar<any, any>>;
 export const getRegisteredScalars = (): Scalar[] => [
   DateScalar,
   DateTimeScalar,
+  DurationScalar,
+  JmesPathScalar,
   RichTextScalar,
   FileUploadScalar,
   UrlScalar,

src/common/scalars/jmes-path.scalar.ts

@@ -0,0 +1,26 @@
+import { search } from '@aws-lambda-powertools/jmespath';
+import { stripIndent } from 'common-tags';
+import { GraphQLError, GraphQLScalarType, GraphQLString } from 'graphql';
+
+const base = GraphQLString.toConfig();
+export const JmesPathScalar = new GraphQLScalarType({
+  ...base,
+  name: 'JMESPath',
+  description: stripIndent`
+    A JMESPath expression.
+
+    See https://jmespath.org/ for details
+  `,
+  specifiedByURL: 'https://jmespath.org/specification.html',
+  parseLiteral: (node) => validate(base.parseLiteral(node)),
+  parseValue: (value) => validate(base.parseValue(value)),
+});
+
+const validate = (value: string) => {
+  try {
+    search(value, {});
+  } catch (e) {
+    throw new GraphQLError((e as Error).message.replaceAll(/"/g, '`'));
+  }
+  return value;
+};

src/core/config/config.service.ts

@@ -315,6 +315,12 @@ export const makeConfig = (env: EnvironmentService) =>
 
     redis = {
       url: env.string('REDIS_URL').optional(),
+      prefix: env.string('REDIS_PREFIX').optional(),
+    };
+
+    bull = {
+      url: env.string('BULL_URL').optional() ?? this.redis.url,
+      prefix: env.string('BULL_PREFIX').optional() ?? this.redis.prefix,
     };
 
     /**

src/core/core.module.ts

@@ -22,6 +22,7 @@ import { GelModule } from './gel/gel.module';
 import { GraphqlModule } from './graphql';
 import { HttpModule } from './http';
 import { LiveQueryModule } from './live-query/live-query.module';
+import { QueueModule } from './queue/queue.module';
 import { ResourceModule } from './resources/resource.module';
 import { ScalarProviders } from './scalars.resolver';
 import { ShutdownHookProvider } from './shutdown.hook';
@@ -52,6 +53,7 @@ import { WebhooksModule } from './webhooks/webhooks.module';
     ValidationModule,
     AuthenticationModule,
     WebhooksModule,
+    QueueModule,
   ],
   providers: [
     AwsS3Factory,
@@ -84,6 +86,7 @@ import { WebhooksModule } from './webhooks/webhooks.module';
     ValidationModule,
     AuthenticationModule,
     WebhooksModule,
+    QueueModule,
   ],
 })
 export class CoreModule {}

src/core/exception/exception.normalizer.ts

@@ -6,7 +6,6 @@ import { entries, isNotFalsy, simpleSwitch } from '@seedcompany/common';
 import * as Gel from 'gel';
 import * as GelTags from 'gel/dist/errors/tags.js';
 import { GraphQLError } from 'graphql';
-import addIndent from 'indent-string';
 import { lowerCase, uniq } from 'lodash';
 import type { AbstractClass } from 'type-fest';
 import {
@@ -23,8 +22,7 @@ import type { ConfigService } from '~/core/config';
 import * as Neo from '../database/errors';
 import { ExclusivityViolationError } from '../gel/errors';
 import { ResourcesHost } from '../resources/resources.host';
-import { isSrcFrame } from './is-src-frame';
-import { normalizeFramePath } from './normalize-frame-path';
+import { prettyStack } from './pretty-stack';
 
 interface NormalizeParams {
   ex: Error;
@@ -76,7 +74,9 @@ export class ExceptionNormalizer {
       code: codes[0],
       codes: new JsonSet(codes),
       ...extensions,
-      stack: this.getStack(params),
+      stack: prettyStack(params.ex, {
+        relativePaths: !this.config?.jest,
+      }),
     };
   }
 
@@ -294,21 +294,6 @@ export class ExceptionNormalizer {
     return Object.assign(wrapped, { idNotFound: id });
   }
 
-  private getStack({ ex }: NormalizeParams) {
-    return getCauseList(ex)
-      .map((e) =>
-        (e.stack ?? e.message)
-          .split('\n')
-          .filter(isSrcFrame)
-          .map((frame: string) =>
-            this.config?.jest ? frame : normalizeFramePath(frame),
-          )
-          .join('\n'),
-      )
-      .map((e, i) => addIndent(i > 0 ? `[cause]: ${e}` : e, i * 2))
-      .join('\n');
-  }
-
   private httpException(ex: Nest.HttpException) {
     const res = ex.getResponse();
     const {

src/core/exception/index.ts

@@ -1,3 +1,2 @@
-export * from './is-src-frame';
-export * from './normalize-frame-path';
+export * from './pretty-stack';
 export * from './jest-skip-source-file';