Improved schedule engine (#2174)

Author: ericallam

apps/webapp/app/components/runs/v3/RetryDeploymentIndexingDialog.tsx

@@ -692,6 +692,46 @@ const EnvironmentSchema = z.object({
   COMMON_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"),
   COMMON_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"),
 
+  SCHEDULE_ENGINE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
+  SCHEDULE_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"),
+  SCHEDULE_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(1),
+  SCHEDULE_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(1),
+  SCHEDULE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
+  SCHEDULE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50),
+  SCHEDULE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50),
+  SCHEDULE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(30_000),
+  SCHEDULE_WORKER_DISTRIBUTION_WINDOW_SECONDS: z.coerce.number().int().default(30),
+
+  SCHEDULE_WORKER_REDIS_HOST: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_HOST),
+  SCHEDULE_WORKER_REDIS_READER_HOST: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_READER_HOST),
+  SCHEDULE_WORKER_REDIS_READER_PORT: z.coerce
+    .number()
+    .optional()
+    .transform(
+      (v) =>
+        v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined)
+    ),
+  SCHEDULE_WORKER_REDIS_PORT: z.coerce
+    .number()
+    .optional()
+    .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)),
+  SCHEDULE_WORKER_REDIS_USERNAME: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_USERNAME),
+  SCHEDULE_WORKER_REDIS_PASSWORD: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_PASSWORD),
+  SCHEDULE_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"),
+  SCHEDULE_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"),
+
   TASK_EVENT_PARTITIONING_ENABLED: z.string().default("0"),
   TASK_EVENT_PARTITIONED_WINDOW_IN_SECONDS: z.coerce.number().int().default(60), // 1 minute
 

apps/webapp/app/env.server.ts

@@ -190,6 +190,11 @@ export class NextRunListPresenter {
       prisma: this.replica as PrismaClient,
     });
 
+    function clampToNow(date: Date): Date {
+      const now = new Date();
+      return date > now ? now : date;
+    }
+
     const { runs, pagination } = await runsRepository.listRuns({
       organizationId,
       environmentId,
@@ -200,8 +205,8 @@ export class NextRunListPresenter {
       tags,
       scheduleId,
       period: periodMs ?? undefined,
-      from,
-      to,
+      from: time.from ? time.from.getTime() : undefined,
+      to: time.to ? clampToNow(time.to).getTime() : undefined,
       isTest,
       rootOnly,
       batchId,

apps/webapp/app/presenters/v3/NextRunListPresenter.server.ts

@@ -179,6 +179,12 @@ export class RunListPresenter extends BasePresenter {
 
     const periodMs = time.period ? parse(time.period) : undefined;
 
+    function clampToNow(date: Date): Date {
+      const now = new Date();
+
+      return date > now ? now : date;
+    }
+
     //get the runs
     const runs = await this._replica.$queryRaw<
       {
@@ -282,7 +288,9 @@ WHERE
         : Prisma.empty
     }
     ${
-      time.to ? Prisma.sql`AND tr."createdAt" <= ${time.to.toISOString()}::timestamp` : Prisma.empty
+      time.to
+        ? Prisma.sql`AND tr."createdAt" <= ${clampToNow(time.to).toISOString()}::timestamp`
+        : Prisma.sql`AND tr."createdAt" <= CURRENT_TIMESTAMP`
     }
     ${
       tags && tags.length > 0

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -1,10 +1,12 @@
-import { ArrowPathIcon, ArrowUturnLeftIcon, BookOpenIcon } from "@heroicons/react/20/solid";
-import { type MetaFunction, Outlet, useLocation, useParams, useNavigate } from "@remix-run/react";
+import { ArrowUturnLeftIcon, BookOpenIcon } from "@heroicons/react/20/solid";
+import { type MetaFunction, Outlet, useLocation, useNavigate, useParams } from "@remix-run/react";
 import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { useEffect } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { PromoteIcon } from "~/assets/icons/PromoteIcon";
 import { DeploymentsNone, DeploymentsNoneDev } from "~/components/BlankStatePanels";
+import { GitMetadata } from "~/components/GitMetadata";
 import { UserAvatar } from "~/components/UserProfilePhoto";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Badge } from "~/components/primitives/Badge";
@@ -34,7 +36,6 @@ import {
   deploymentStatusDescription,
   deploymentStatuses,
 } from "~/components/runs/v3/DeploymentStatus";
-import { RetryDeploymentIndexingDialog } from "~/components/runs/v3/RetryDeploymentIndexingDialog";
 import {
   PromoteDeploymentDialog,
   RollbackDeploymentDialog,
@@ -52,8 +53,6 @@ import { EnvironmentParamSchema, docsPath, v3DeploymentPath } from "~/utils/path
 import { createSearchParams } from "~/utils/searchParams";
 import { deploymentIndexingIsRetryable } from "~/v3/deploymentStatus";
 import { compareDeploymentVersions } from "~/v3/utils/deploymentVersions";
-import { useEffect } from "react";
-import { GitMetadata } from "~/components/GitMetadata";
 
 export const meta: MetaFunction = () => {
   return [
@@ -388,26 +387,6 @@ function DeploymentActionsCell({
               />
             </Dialog>
           )}
-          {canRetryIndexing && (
-            <Dialog>
-              <DialogTrigger asChild>
-                <Button
-                  variant="small-menu-item"
-                  LeadingIcon={ArrowPathIcon}
-                  leadingIconClassName="text-blue-500"
-                  fullWidth
-                  textAlignLeft
-                >
-                  Retry indexing…
-                </Button>
-              </DialogTrigger>
-              <RetryDeploymentIndexingDialog
-                projectId={project.id}
-                deploymentShortCode={deployment.shortCode}
-                redirectPath={`${location.pathname}${location.search}`}
-              />
-            </Dialog>
-          )}
         </>
       }
     />