tenstorrent: bh_chip: add pgood fault interrupt

Adds interrupts on pgood gpio fall and rise to enable the pgood fault sequence.

When a pgood fault is detected, the board fault LED is turned on and ASIC reset
is asserted. If pgood comes back up, then the LED is cleared and the ASIC is
brought out of reset. If it falls again within 1 second, the pgood severe fault
state is entered and can only be cleared with a power cycle.

Signed-off-by: Petra Alexson <[email protected]>

Author: kikkiakki

app/dmc/src/main.c

@@ -221,14 +221,6 @@ int main(void)
 		}
 	}
 
-	ARRAY_FOR_EACH_PTR(BH_CHIPS, chip) {
-		ret = therm_trip_gpio_setup(chip);
-		if (ret != 0) {
-			LOG_ERR("%s() failed: %d", "therm_trip_gpio_setup", ret);
-			return ret;
-		}
-	}
-
 	if (IS_ENABLED(CONFIG_TT_FWUPDATE)) {
 		if (!tt_fwupdate_is_confirmed()) {
 			if (bist_rc < 0) {
@@ -267,10 +259,24 @@ int main(void)
 		}
 	}
 
-	if (IS_ENABLED(CONFIG_TT_ASSEMBLY_TEST) && board_fault_led.port != NULL) {
+	/* Set up GPIOs */
+	if (board_fault_led.port != NULL) {
 		gpio_pin_configure_dt(&board_fault_led, GPIO_OUTPUT_INACTIVE);
 	}
 
+	ARRAY_FOR_EACH_PTR(BH_CHIPS, chip) {
+		ret = therm_trip_gpio_setup(chip);
+		if (ret != 0) {
+			LOG_ERR("%s() failed: %d", "therm_trip_gpio_setup", ret);
+			return ret;
+		}
+		ret = pgood_gpio_setup(chip);
+		if (ret != 0) {
+			LOG_ERR("%s() failed: %d", "pgood_gpio_setup", ret);
+			return ret;
+		}
+	}
+
 	if (IS_ENABLED(CONFIG_JTAG_LOAD_BOOTROM)) {
 		ARRAY_FOR_EACH_PTR(BH_CHIPS, chip) {
 			ret = jtag_bootrom_init(chip);
@@ -337,6 +343,33 @@ int main(void)
 			}
 		}
 
+		/* handler for PGOOD */
+		ARRAY_FOR_EACH_PTR(BH_CHIPS, chip) {
+			if (chip->data.pgood_fall_triggered && !chip->data.pgood_severe_fault) {
+				int64_t current_uptime_ms = k_uptime_get();
+				/* Assert board fault */
+				gpio_pin_set_dt(&board_fault_led, 1);
+				/* Report over SMBus - to add later */
+				/* Assert ASIC reset */
+				bh_chip_assert_asic_reset(chip);
+				/* If pgood went down again within 1 second */
+				if (chip->data.pgood_last_trip_ms != 0 &&
+				    current_uptime_ms - chip->data.pgood_last_trip_ms < 1000) {
+					/* Assert more severe fault over IPMI - to add later */
+					chip->data.pgood_severe_fault = true;
+				}
+				chip->data.pgood_last_trip_ms = current_uptime_ms;
+				chip->data.pgood_fall_triggered = false;
+			}
+			if (chip->data.pgood_rise_triggered && !chip->data.pgood_severe_fault) {
+				/* Follow out of reset procedure */
+				bh_chip_reset_chip(chip, true);
+				/* Clear board fault */
+				gpio_pin_set_dt(&board_fault_led, 0);
+				chip->data.pgood_rise_triggered = false;
+			}
+		}
+
 		/* TODO(drosen): Turn this into a task which will re-arm until static data is sent
 		 */
 		ARRAY_FOR_EACH_PTR(BH_CHIPS, chip) {

include/tenstorrent/bh_chip.h

@@ -54,13 +54,20 @@ struct bh_chip_data {
 	bool trigger_reset;
 
 	/* notify the main thread to handle therm trip */
-	bool therm_trip_triggered;
+	volatile bool therm_trip_triggered;
+
+	/* notify the main thread to handle pgood events */
+	volatile bool pgood_fall_triggered;
+	volatile bool pgood_rise_triggered;
+	bool pgood_severe_fault;
+	int64_t pgood_last_trip_ms;
 };
 
 struct bh_chip {
 	const struct bh_chip_config config;
 	struct bh_chip_data data;
 	struct gpio_callback therm_trip_cb;
+	struct gpio_callback pgood_cb;
 };
 
 #define DT_PHANDLE_OR_CHILD(node_id, name)                                                         \
@@ -103,8 +110,8 @@ extern struct bh_chip BH_CHIPS[BH_CHIP_COUNT];
 	  HAS_DT_PHANDLE_OR_CHILD(DT_PHANDLE_BY_IDX(n, prop, idx), strapping),                     \
 	  (DT_FOREACH_CHILD(DT_PHANDLE_OR_CHILD(DT_PHANDLE_BY_IDX(n, prop, idx), strapping),       \
 			    INIT_STRAP)),                                                          \
-	  ())},                                    \
-				},                                                                 \
+	  ())},                                            \
+			},                                                                         \
 			},
 
 #define BH_CHIP_PRIMARY_INDEX DT_PROP(DT_PATH(chips), primary)
@@ -132,6 +139,9 @@ void bh_chip_deassert_spi_reset(const struct bh_chip *chip);
 int bh_chip_reset_chip(struct bh_chip *chip, bool force_reset);
 
 int therm_trip_gpio_setup(struct bh_chip *chip);
+int pgood_gpio_setup(struct bh_chip *chip);
+
+void pgood_fault_work_handler(struct k_work *work);
 
 #ifdef __cplusplus
 }

lib/tenstorrent/bh_chip/bh_chip.c

@@ -113,6 +113,7 @@ int bh_chip_reset_chip(struct bh_chip *chip, bool force_reset)
 void therm_trip_detected(const struct device *dev, struct gpio_callback *cb, uint32_t pins)
 {
 	struct bh_chip *chip = CONTAINER_OF(cb, struct bh_chip, therm_trip_cb);
+
 	chip->data.therm_trip_triggered = true;
 	bh_chip_cancel_bus_transfer_set(chip);
 	tt_event_post(TT_EVENT_WAKE);
@@ -127,13 +128,47 @@ int therm_trip_gpio_setup(struct bh_chip *chip)
 	if (ret != 0) {
 		return ret;
 	}
+	gpio_init_callback(&chip->therm_trip_cb, therm_trip_detected,
+			   BIT(chip->config.therm_trip.pin));
+	ret = gpio_add_callback_dt(&chip->config.therm_trip, &chip->therm_trip_cb);
+	if (ret != 0) {
+		return ret;
+	}
 	ret = gpio_pin_interrupt_configure_dt(&chip->config.therm_trip, GPIO_INT_EDGE_TO_ACTIVE);
+
+	return ret;
+}
+
+void pgood_change_detected(const struct device *dev, struct gpio_callback *cb, uint32_t pins)
+{
+	struct bh_chip *chip = CONTAINER_OF(cb, struct bh_chip, pgood_cb);
+
+	/* Sample PGOOD to see if it rose or fell */
+	/* TODO: could setup rising interrupt only after falling triggered */
+	if (gpio_pin_get_dt(&chip->config.pgood)) {
+		chip->data.pgood_rise_triggered = true;
+	} else {
+		chip->data.pgood_fall_triggered = true;
+	}
+	tt_event_post(TT_EVENT_WAKE);
+}
+
+int pgood_gpio_setup(struct bh_chip *chip)
+{
+	/* Set up PGOOD interrupt */
+	int ret;
+
+	ret = gpio_pin_configure_dt(&chip->config.pgood, GPIO_INPUT);
 	if (ret != 0) {
 		return ret;
 	}
-	gpio_init_callback(&chip->therm_trip_cb, therm_trip_detected,
-			   BIT(chip->config.therm_trip.pin));
-	ret = gpio_add_callback(chip->config.therm_trip.port, &chip->therm_trip_cb);
+	gpio_init_callback(&chip->pgood_cb, pgood_change_detected, BIT(chip->config.pgood.pin));
+	ret = gpio_add_callback_dt(&chip->config.pgood, &chip->pgood_cb);
+	if (ret != 0) {
+		return ret;
+	}
+
+	ret = gpio_pin_interrupt_configure_dt(&chip->config.pgood, GPIO_INT_EDGE_BOTH);
 
 	return ret;
 }