re2: env based queue selection algo (#1775)

* re2: fix @trigger.dev/core exports

* re2: WIP env based queue selection algo

* more wip

* WIP

* Get run engine tests to pass

* Adding tests for the fair dequeueing strat in the run engine

* Configure the new queue selection strategy in the webapp and get it all building and typechecks passing

* webapp now uses built packages, building redis-worker, run-engine, database, using better tsconfig setups for tests, moving isomorphic code into core/v3/isomorphic

* Fixed webapp typechecks

* dev now depends on build, fixed supervisor typecheck

* Fixed run engine tests

* Fixed e2e tests

Author: ericallam

.eslintrc.js

@@ -141,6 +141,14 @@
       "command": "pnpm run test --filter @internal/run-engine",
       "cwd": "${workspaceFolder}",
       "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug RunQueue tests",
+      "command": "pnpm run test ./src/engine/tests/waitpoints.test.ts",
+      "cwd": "${workspaceFolder}/internal-packages/run-engine",
+      "sourceMaps": true
     }
   ]
 }

.vscode/launch.json

@@ -4,9 +4,5 @@
   "compilerOptions": {
     "rootDir": "src",
     "outDir": "dist"
-  },
-  "paths": {
-    "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
-    "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
   }
 }

apps/supervisor/tsconfig.json

@@ -1,10 +1,5 @@
 {
-  "plugins": [
-    "@trigger.dev/eslint-plugin",
-    "react-hooks",
-    "@typescript-eslint/eslint-plugin",
-    "import"
-  ],
+  "plugins": ["react-hooks", "@typescript-eslint/eslint-plugin", "import"],
   "parser": "@typescript-eslint/parser",
   "overrides": [
     {

apps/webapp/.eslintrc

@@ -11,7 +11,7 @@ import {
   RunPanelIconSection,
   RunPanelProperties,
 } from "./RunCard";
-import { DisplayProperty } from "@trigger.dev/core";
+import type { DisplayProperty } from "@trigger.dev/core";
 
 export function TriggerDetail({
   trigger,

apps/webapp/app/components/run/TriggerDetail.tsx

@@ -14,7 +14,7 @@ import {
   OperatingSystemContextProvider,
   OperatingSystemPlatform,
 } from "./components/primitives/OperatingSystemProvider";
-import { getSharedSqsEventConsumer } from "./services/events/sqsEventConsumer";
+import { getSharedSqsEventConsumer } from "./services/events/sqsEventConsumer.server";
 import { singleton } from "./utils/singleton";
 
 const ABORT_DELAY = 30000;

apps/webapp/app/entry.server.tsx

@@ -419,6 +419,12 @@ const EnvironmentSchema = z.object({
   RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(60_000),
   RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(60_000),
   RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS: z.coerce.boolean().default(false),
+  RUN_ENGINE_PARENT_QUEUE_LIMIT: z.coerce.number().int().default(1000),
+  RUN_ENGINE_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75),
+  RUN_ENGINE_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3),
+  RUN_ENGINE_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25),
+  RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0),
+  RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(),
 
   RUN_ENGINE_WORKER_REDIS_HOST: z
     .string()

apps/webapp/app/env.server.ts

@@ -1,4 +1,4 @@
-import { Prettify } from "@trigger.dev/core";
+import type { Prettify } from "@trigger.dev/core";
 import { TaskRun } from "@trigger.dev/database";
 import { SyncedShapeData, useSyncedShape } from "./useSyncedShape";
 

apps/webapp/app/hooks/useSyncTraceRuns.ts

@@ -1,7 +1,7 @@
 import { z } from "zod";
 import { PrismaClient, prisma } from "~/db.server";
 import { sortEnvironments } from "~/utils/environmentSort";
-import { httpEndpointUrl } from "~/services/httpendpoint/HandleHttpEndpointService";
+import { httpEndpointUrl } from "~/services/httpendpoint/HandleHttpEndpointService.server";
 import { getSecretStore } from "~/services/secrets/secretStore.server";
 import { projectPath } from "~/utils/pathBuilder";
 

apps/webapp/app/presenters/HttpEndpointPresenter.server.ts

@@ -11,7 +11,7 @@ import { eventRepository } from "~/v3/eventRepository.server";
 import { machinePresetFromName } from "~/v3/machinePresets.server";
 import { FINAL_ATTEMPT_STATUSES, isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
-import { getMaxDuration } from "@trigger.dev/core/v3/apps";
+import { getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
 import { logger } from "~/services/logger.server";
 import { getTaskEventStoreTableForRun, TaskEventStoreTable } from "~/v3/taskEventStore.server";
 import { Pi } from "lucide-react";