Instance auto restart

app/api/util.ts

@@ -99,8 +99,19 @@ const instanceActions = {
   // https://github.com/oxidecomputer/omicron/blob/6dd9802/nexus/db-queries/src/db/datastore/instance.rs#L865
   delete: ['stopped', 'failed'],
 
-  // https://github.com/oxidecomputer/omicron/blob/3093818/nexus/db-queries/src/db/datastore/instance.rs#L1030-L1043
-  update: ['stopped', 'failed', 'creating'],
+  // not all updates are restricted by state
+
+  // resize means changes to ncpus and memory
+  // https://github.com/oxidecomputer/omicron/blob/0c6ab099e/nexus/db-queries/src/db/datastore/instance.rs#L1282-L1288
+  // https://github.com/oxidecomputer/omicron/blob/0c6ab099e/nexus/db-model/src/instance_state.rs#L39-L42
+  resize: ['stopped', 'failed', 'creating'],
+
+  // https://github.com/oxidecomputer/omicron/blob/0c6ab099e/nexus/db-queries/src/db/datastore/instance.rs#L1209-L1215
+  updateBootDisk: ['stopped', 'failed', 'creating'],
+
+  // there are no state restrictions on setting the auto restart policy, so we
+  // don't have a helper for it
+  // https://github.com/oxidecomputer/omicron/blob/0c6ab099e/nexus/db-queries/src/db/datastore/instance.rs#L1050-L1058
 
   // reboot and stop are kind of weird!
   // https://github.com/oxidecomputer/omicron/blob/6dd9802/nexus/src/app/instance.rs#L790-L798
@@ -140,6 +151,11 @@ export function instanceTransitioning({ runState }: Instance) {
   )
 }
 
+/** Cooling down after failed auto-restart */
+export function instanceCoolingDown(i: Instance) {
+  return i.runState === 'failed' && i.autoRestartCooldownExpiration
+}
+
 const diskActions = {
   // this is a weird one because the list of states is dynamic and it includes
   // 'creating' in the unwind of the disk create saga, but does not include

app/components/DocsPopover.tsx

@@ -48,8 +48,8 @@ export const DocsPopover = ({ heading, icon, summary, links }: DocsPopoverProps)
         <Info16Icon aria-label="Links to docs" className="shrink-0" />
       </PopoverButton>
       <PopoverPanel
-        // DocsPopoverPanel needed for enter animation
-        className="DocsPopoverPanel z-10 w-96 rounded-lg border bg-raise border-secondary elevation-2"
+        // popover-panel needed for enter animation
+        className="popover-panel z-10 w-96 rounded-lg border bg-raise border-secondary elevation-2"
         anchor={{ to: 'bottom end', gap: 12 }}
       >
         <div className="px-4">

app/components/HL.tsx

@@ -10,7 +10,7 @@ import { classed } from '~/util/classed'
 // note parent with secondary text color must have 'group' on it for
 // this to work. see Toast for an example
 export const HL = classed.span`
-  text-sans-md text-raise
+  text-semi-md text-raise
   group-[.text-accent-secondary]:text-accent
   group-[.text-error-secondary]:text-error
   group-[.text-info-secondary]:text-info

app/components/InstanceAutoRestartPopover.tsx

@@ -0,0 +1,147 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * Copyright Oxide Computer Company
+ */
+import { CloseButton, Popover, PopoverButton, PopoverPanel } from '@headlessui/react'
+import { formatDistanceToNow } from 'date-fns'
+import { type ReactNode } from 'react'
+import { Link } from 'react-router'
+import { match } from 'ts-pattern'
+
+import {
+  AutoRestart12Icon,
+  NextArrow12Icon,
+  OpenLink12Icon,
+} from '@oxide/design-system/icons/react'
+
+import type { Instance } from '~/api'
+import { HL } from '~/components/HL'
+import { useInstanceSelector } from '~/hooks/use-params'
+import { Badge } from '~/ui/lib/Badge'
+import { Spinner } from '~/ui/lib/Spinner'
+import { links } from '~/util/links'
+import { pb } from '~/util/path-builder'
+
+const helpText = {
+  enabled: (
+    <>
+      The control plane will attempt to automatically restart this instance after entering
+      the <HL>failed</HL> state.
+    </>
+  ),
+  disabled: (
+    <>
+      The control plane will not attempt to automatically restart it after entering the{' '}
+      <HL>failed</HL> state.
+    </>
+  ),
+  never: (
+    <>
+      Instance auto-restart policy is set to never. The control plane will not attempt to
+      automatically restart it after entering the <HL>failed</HL> state.
+    </>
+  ),
+  starting: (
+    <>
+      Instance auto-restart policy is queued to start. The control plane will begin the
+      restart process shortly.
+    </>
+  ),
+}
+
+export const InstanceAutoRestartPopover = ({ instance }: { instance: Instance }) => {
+  const {
+    autoRestartCooldownExpiration: cooldownExpiration,
+    autoRestartPolicy: policy,
+    autoRestartEnabled: enabled,
+  } = instance
+
+  const instanceSelector = useInstanceSelector()
+  const now = new Date()
+  const isQueued = cooldownExpiration && cooldownExpiration < now
+
+  let helpTextState: keyof typeof helpText = 'disabled'
+  if (isQueued) helpTextState = 'starting' // Expiration is in the past and queued for restart
+  if (policy === 'never') helpTextState = 'never' // Will never auto-restart
+  if (enabled) helpTextState = 'enabled' // Restart enabled and cooldown as not expired
+
+  return (
+    <Popover>
+      <PopoverButton
+        className="group flex h-6 w-6 items-center justify-center rounded border border-default hover:bg-hover"
+        aria-label="Auto-restart status"
+      >
+        <AutoRestart12Icon className="shrink-0" aria-hidden />
+      </PopoverButton>
+      <PopoverPanel
+        // popover-panel needed for enter animation
+        className="popover-panel z-10 w-96 rounded-lg border bg-raise border-secondary elevation-2"
+        anchor={{ to: 'bottom start', gap: 12 }}
+      >
+        <PopoverRow label="Auto Restart">
+          {enabled ? <Badge>Enabled</Badge> : <Badge color="neutral">Disabled</Badge>}
+        </PopoverRow>
+        <PopoverRow label="Policy">
+          <CloseButton
+            as={Link}
+            to={pb.instanceSettings(instanceSelector)}
+            className="group -m-1 flex w-full items-center justify-between rounded px-1"
+          >
+            {match(policy)
+              .with('never', () => (
+                <Badge color="neutral" variant="solid">
+                  never
+                </Badge>
+              ))
+              .with('best_effort', () => <Badge>best effort</Badge>)
+              .with(undefined, () => <Badge color="neutral">Default</Badge>)
+              .exhaustive()}
+            <div className="transition-transform group-hover:translate-x-1">
+              <NextArrow12Icon />
+            </div>
+          </CloseButton>
+        </PopoverRow>
+        {cooldownExpiration && (
+          <PopoverRow label="Cooldown">
+            {isQueued ? (
+              <>
+                <Spinner /> Queued for restart…
+              </>
+            ) : (
+              <div>
+                Waiting{' '}
+                <span className="text-tertiary">
+                  ({formatDistanceToNow(cooldownExpiration)})
+                </span>
+              </div>
+            )}
+          </PopoverRow>
+        )}
+        <div className="p-3 text-sans-md text-default">
+          <p className="mb-2 pr-4">{helpText[helpTextState]}</p>
+          <a href={links.instanceUpdateDocs} className="group">
+            <span className="inline-block max-w-[300px] truncate align-middle">
+              Learn about{' '}
+              <span className="group-hover:link-with-underline text-raise">
+                Instance Auto-Restart
+              </span>
+            </span>
+            <OpenLink12Icon className="ml-1 translate-y-[1px] text-secondary" />
+          </a>
+        </div>
+      </PopoverPanel>
+    </Popover>
+  )
+}
+
+const PopoverRow = ({ label, children }: { label: string; children: ReactNode }) => (
+  <div className="flex h-10 items-center border-b border-b-secondary">
+    <div className="w-32 pl-3 pr-2 text-mono-sm text-tertiary">{label}</div>
+    <div className="flex h-10 flex-grow items-center gap-1.5 pr-2 text-sans-md">
+      {children}
+    </div>
+  </div>
+)

app/pages/project/instances/actions.tsx

@@ -147,8 +147,8 @@ export const useMakeInstanceActions = (
         {
           label: 'Resize',
           onActivate: () => onResizeClick?.(instance),
-          disabled: !instanceCan.update(instance) && (
-            <>Only {fancifyStates(instanceCan.update.states)} instances can be resized</>
+          disabled: !instanceCan.resize(instance) && (
+            <>Only {fancifyStates(instanceCan.resize.states)} instances can be resized</>
           ),
         },
         {

app/pages/project/instances/instance/InstancePage.tsx

@@ -9,6 +9,7 @@ import { filesize } from 'filesize'
 import { useId, useMemo, useState } from 'react'
 import { useForm } from 'react-hook-form'
 import { Link, useNavigate, type LoaderFunctionArgs } from 'react-router'
+import { match } from 'ts-pattern'
 
 import {
   apiQueryClient,
@@ -24,11 +25,13 @@ import {
   INSTANCE_MAX_CPU,
   INSTANCE_MAX_RAM_GiB,
   instanceCan,
+  instanceCoolingDown,
   instanceTransitioning,
 } from '~/api/util'
 import { ExternalIps } from '~/components/ExternalIps'
 import { NumberField } from '~/components/form/fields/NumberField'
 import { HL } from '~/components/HL'
+import { InstanceAutoRestartPopover } from '~/components/InstanceAutoRestartPopover'
 import { InstanceDocsPopover } from '~/components/InstanceDocsPopover'
 import { MoreActionsMenu } from '~/components/MoreActionsMenu'
 import { RefreshButton } from '~/components/RefreshButton'
@@ -106,6 +109,20 @@ InstancePage.loader = async ({ params }: LoaderFunctionArgs) => {
 
 const POLL_INTERVAL = 1000
 
+function shouldPoll(instance: Instance) {
+  if (instanceTransitioning(instance)) return 'transition'
+  if (instanceCoolingDown(instance)) return 'cooldown'
+  return null
+}
+
+const PollingSpinner = () => (
+  <Tooltip content="Auto-refreshing while state changes" delay={150}>
+    <button type="button">
+      <Spinner className="ml-2" />
+    </button>
+  </Tooltip>
+)
+
 export function InstancePage() {
   const instanceSelector = useInstanceSelector()
   const [resizeInstance, setResizeInstance] = useState(false)
@@ -130,11 +147,11 @@ export function InstancePage() {
     },
     {
       refetchInterval: ({ state: { data: instance } }) =>
-        instance && instanceTransitioning(instance) ? POLL_INTERVAL : false,
+        instance && shouldPoll(instance) ? POLL_INTERVAL : false,
     }
   )
 
-  const polling = instanceTransitioning(instance)
+  const pollReason = shouldPoll(instance)
 
   const { data: nics } = usePrefetchedApiQuery('instanceNetworkInterfaceList', {
     query: {
@@ -203,15 +220,13 @@ export function InstancePage() {
             <span className="ml-1 text-tertiary"> {memory.unit}</span>
           </PropertiesTable.Row>
           <PropertiesTable.Row label="state">
-            <div className="flex">
+            <div className="flex items-center gap-2">
               <InstanceStateBadge state={instance.runState} />
-              {polling && (
-                <Tooltip content="Auto-refreshing while state changes" delay={150}>
-                  <button type="button">
-                    <Spinner className="ml-2" />
-                  </button>
-                </Tooltip>
-              )}
+              {match(pollReason)
+                .with('transition', () => <PollingSpinner />)
+                .with('cooldown', () => <InstanceAutoRestartPopover instance={instance} />)
+                .with(null, () => null)
+                .exhaustive()}
             </div>
           </PropertiesTable.Row>
           <PropertiesTable.Row label="vpc">
@@ -241,6 +256,7 @@ export function InstancePage() {
         <Tab to={pb.instanceMetrics(instanceSelector)}>Metrics</Tab>
         <Tab to={pb.instanceNetworking(instanceSelector)}>Networking</Tab>
         <Tab to={pb.instanceConnect(instanceSelector)}>Connect</Tab>
+        <Tab to={pb.instanceSettings(instanceSelector)}>Settings</Tab>
       </RouteTabs>
       {resizeInstance && (
         <ResizeInstanceModal
@@ -298,7 +314,7 @@ export function ResizeInstanceModal({
     mode: 'onChange',
   })
 
-  const canResize = instanceCan.update(instance)
+  const canResize = instanceCan.resize(instance)
   const willChange =
     form.watch('ncpus') !== instance.ncpus || form.watch('memory') !== instance.memory / GiB
   const isDisabled = !form.formState.isValid || !canResize || !willChange