ereport: Task faulted/panicked (#2341)

Depends on #2313, #2350, #2358
Fixes #2309 

It's currently somewhat difficult to become aware of Hubris task panics
and other task faults in a production environment. While MGS can ask the
SP to list task dumps as part of the API for reading dumps, this
requires that the control plane (or faux-mgs user) proactively ask the
SP whether it has any record of panicked tasks, rather than recording
panics as they occur. Therefore, we should have a proactive notification
from the SP indicating that task faults have occurred.

This commit adds code to `packrat` for producing an ereport when a task
has faulted. This could eventually be used by the control plane to
trigger dump collection and produce a service bundle. In addition, it
will provide a more permanent record that a task faulted at a particular
time, even if the SP that contains the faulted task is later reset or
replaced with an entirely different SP. This works using an approach
similar to the one described by @cbiffle in [this comment][1]. There's a
detailed description of how this works [in the module-level RustDoc for
`ereport.rs` in Packrat][2].

The ereports that come out of this thing look like this:

```console
eliza@hekate ~/Code/oxide/hubris $ faux-mgs --interface eno1np0 --discovery-addr '[fe80::0c1d:deff:fef0:d922]:11111' ereports
Jan 15 10:27:41.370 INFO creating SP handle on interface eno1np0, component: faux-mgs
Jan 15 10:27:41.372 INFO initial discovery complete, addr: [fe80::c1d:deff:fef0:d922%2]:11111, interface: eno1np0, socket: control-plane-agent, component: faux-mgs
restart ID: 4e54b7f1-e13a-d9bb-709a-c7e863d64a64
restart IDs did not match (requested 00000000-0000-0000-0000-000000000000)
count: 4

ereports:
0x1: {
    "ereport_message_version": Number(0),
    "hubris_task_gen": Number(0),
    "hubris_task_name": String("packrat"),
    "hubris_uptime_ms": Number(0),
    "lost": Null,
}

0x2: {
    "ereport_message_version": Number(0),
    "hubris_task_gen": Number(0),
    "hubris_task_name": String("ereportulator"),
    "hubris_uptime_ms": Number(378010),
    "k": String("hubris.fault.panic"),
    "msg": String("panicked at task/ereportulator/src/main.rs:158:9:\nim dead lol"),
    "v": Number(0),
}

0x3: {
    "by": Object {
        "gen": Number(0),
        "task": String("jefe"),
    },
    "ereport_message_version": Number(0),
    "hubris_task_gen": Number(0),
    "hubris_task_name": String("user_leds"),
    "hubris_uptime_ms": Number(382914),
    "k": String("hubris.fault.injected"),
    "v": Number(0),
}

0x4: {
    "by": Object {
        "gen": Number(0),
        "task": String("jefe"),
    },
    "ereport_message_version": Number(0),
    "hubris_task_gen": Number(1),
    "hubris_task_name": String("ereportulator"),
    "hubris_uptime_ms": Number(388215),
    "k": String("hubris.fault.injected"),
    "v": Number(0),
}
```


[1]:
  #2309 (comment)
[2]:
  https://github.com/oxidecomputer/hubris/blob/1e2b121fcd165dcdbcbdfdc69c36d35520dfa954/task/packrat/src/ereport.rs#L15-L95

Author: hawkw

Cargo.lock

@@ -33,6 +33,9 @@ net = "jefe-state-change"
 host_sp_comms = "jefe-state-change"
 spd = "jefe-state-change"
 
+[tasks.jefe.config.on-task-fault]
+packrat = "task-faulted"
+
 [tasks.jefe.config.allowed-callers]
 set_state = ["cosmo_seq"]
 set_reset_reason = ["sys"]
@@ -128,11 +131,11 @@ notifications = ["i2c1-irq", "i2c2-irq", "i2c3-irq", "i2c4-irq"]
 [tasks.packrat]
 name = "task-packrat"
 priority = 1
-stacksize = 1096
+stacksize = 1352
 start = true
-# task-slots is explicitly empty: packrat should not send IPCs!
-task-slots = []
+task-slots = ["jefe"]
 features = ["cosmo", "ereport"]
+notifications = ["task-faulted"]
 
 [tasks.rng_driver]
 features = ["h753", "ereport"]

app/cosmo/base.toml

@@ -28,6 +28,9 @@ net = "jefe-state-change"
 host_sp_comms = "jefe-state-change"
 spd = "jefe-state-change"
 
+[tasks.jefe.config.on-task-fault]
+packrat = "task-faulted"
+
 [tasks.jefe.config.allowed-callers]
 set_state = ["gimlet_seq"]
 set_reset_reason = ["sys"]
@@ -122,11 +125,11 @@ notifications = ["i2c1-irq", "jefe-state-change"]
 [tasks.packrat]
 name = "task-packrat"
 priority = 1
-stacksize = 1096
+stacksize = 1352
 start = true
-# task-slots is explicitly empty: packrat should not send IPCs!
-task-slots = []
+task-slots = ["jefe"]
 features = ["gimlet", "ereport"]
+notifications = ["task-faulted"]
 
 [tasks.thermal]
 name = "task-thermal"

app/gimlet/base.toml

@@ -23,6 +23,9 @@ request_reset = ["hiffy", "control_plane_agent", "udprpc"]
 [tasks.jefe.config.on-state-change]
 host_sp_comms = "jefe-state-change"
 
+[tasks.jefe.config.on-task-fault]
+packrat = "task-faulted"
+
 [tasks.sys]
 # Enable EXTI in the sys task so that we can notify sprot when the RoT
 # raises an IRQ.
@@ -48,10 +51,10 @@ owner = {name = "sprot", notification = "rot_irq"}
 name = "task-packrat"
 priority = 1
 start = true
-# task-slots is explicitly empty: packrat should not send IPCs!
-task-slots = []
-stacksize = 1096
+task-slots = ["jefe"]
+stacksize = 1336
 features = ["ereport"]
+notifications = ["task-faulted"]
 
 [tasks.control_plane_agent]
 name = "task-control-plane-agent"

app/gimletlet/app.toml

@@ -12,8 +12,13 @@ interrupts = {"uart8.irq" = "usart-irq"}
 
 # Ereport stuff
 [tasks.packrat]
-stacksize = 1096
+stacksize = 1344
+task-slots = ["jefe"]
 features = ["ereport"]
+notifications = ["task-faulted"]
+
+[tasks.jefe.config.on-task-fault]
+packrat = "task-faulted"
 
 [tasks.snitch]
 name = "task-snitch"

app/grapefruit/app-dev.toml

@@ -6,7 +6,7 @@ board = "oxcon2023g0"
 
 [kernel]
 name = "oxcon2023g0"
-requires = {flash = 11744, ram = 1296}
+requires = {flash = 11776, ram = 1296}
 stacksize = 640
 
 [tasks.jefe]

app/oxcon2023g0/app.toml

@@ -27,6 +27,9 @@ extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 [tasks.jefe.config.on-state-change]
 net = "jefe-state-change"
 
+[tasks.jefe.config.on-task-fault]
+packrat = "task-faulted"
+
 [tasks.jefe.config.allowed-callers]
 set_reset_reason = ["sys"]
 request_reset = ["hiffy", "control_plane_agent"]
@@ -118,11 +121,11 @@ notifications = ["i2c2-irq", "i2c3-irq"]
 [tasks.packrat]
 name = "task-packrat"
 priority = 1
-stacksize = 1096
+stacksize = 1336
 start = true
-# task-slots is explicitly empty: packrat should not send IPCs!
-task-slots = []
+task-slots = ["jefe"]
 features = ["ereport"]
+notifications = ["task-faulted"]
 
 [tasks.sequencer]
 name = "drv-psc-seq-server"

app/psc/base.toml

@@ -28,6 +28,9 @@ extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 set_reset_reason = ["sys"]
 request_reset = ["hiffy", "control_plane_agent"]
 
+[tasks.jefe.config.on-task-fault]
+packrat = "task-faulted"
+
 [tasks.sys]
 name = "drv-stm32xx-sys"
 features = ["h753", "exti", "no-panic"]
@@ -257,11 +260,11 @@ notifications = ["socket", "timer"]
 [tasks.packrat]
 name = "task-packrat"
 priority = 1
-stacksize = 1096
+stacksize = 1336
 start = true
-# task-slots is explicitly empty: packrat should not send IPCs!
-task-slots = []
+task-slots = ["jefe"]
 features = ["ereport"]
+notifications = ["task-faulted"]
 
 [tasks.sequencer]
 name = "drv-sidecar-seq-server"

app/sidecar/base.toml

@@ -20,5 +20,12 @@ Interface(
             ),
             idempotent: true,
         ),
+        "panicme": (
+            doc: "Make the ereportulator panic",
+            args: {
+            },
+            reply: Simple("()"),
+            idempotent: true,
+        ),
     }
 )

idl/ereportulator.idol

@@ -482,12 +482,6 @@ fn find_faulted_task(
     message: USlice<u8>,
     response: USlice<u8>,
 ) -> Result<NextTask, UserError> {
-    if caller != 0 {
-        return Err(UserError::Unrecoverable(FaultInfo::SyscallUsage(
-            UsageError::NotSupervisor,
-        )));
-    }
-
     let index = deserialize_message::<u32>(&tasks[caller], message)? as usize;
 
     // Note: we explicitly permit index == tasks.len(), which causes us to wrap