drivers: can: mcux: flexcan: fix handling of RTR frames

When installing a RX filter, the driver uses "filter->rtr &
filter->rtr_mask" for setting the filter mask. It should just be using
filter->rtr_mask, otherwise filters for non-RTR frames will match RTR
frames as well.

When transmitting a RTR frame, the hardware automatically switches the
mailbox used for TX to RX in order to receive the reply. This, however,
does not match the Zephyr CAN driver model, where mailboxes are
dedicated to either RX or TX. Attempting to reuse the TX mailbox (which
was automatically switched to an RX mailbox by the hardware) fails on
the first call, after which the mailbox is reset and can be reused for
TX. To overcome this, the driver must abort the RX mailbox operation
when the hardware performs the TX to RX switch.

Fixes: #47902

Signed-off-by: Henrik Brix Andersen <[email protected]>

Author: henrikbrixandersen

drivers/can/can_mcux_flexcan.c

@@ -253,13 +253,11 @@ static void mcux_flexcan_copy_zfilter_to_mbconfig(const struct zcan_filter *src,
 	if (src->id_type == CAN_STANDARD_IDENTIFIER) {
 		dest->format = kFLEXCAN_FrameFormatStandard;
 		dest->id = FLEXCAN_ID_STD(src->id);
-		*mask = FLEXCAN_RX_MB_STD_MASK(src->id_mask,
-					       src->rtr & src->rtr_mask, 1);
+		*mask = FLEXCAN_RX_MB_STD_MASK(src->id_mask, src->rtr_mask, 1);
 	} else {
 		dest->format = kFLEXCAN_FrameFormatExtend;
 		dest->id = FLEXCAN_ID_EXT(src->id);
-		*mask = FLEXCAN_RX_MB_EXT_MASK(src->id_mask,
-					       src->rtr & src->rtr_mask, 1);
+		*mask = FLEXCAN_RX_MB_EXT_MASK(src->id_mask, src->rtr_mask, 1);
 	}
 
 	if ((src->rtr & src->rtr_mask) == CAN_DATAFRAME) {
@@ -646,6 +644,7 @@ static inline void mcux_flexcan_transfer_rx_idle(const struct device *dev,
 static FLEXCAN_CALLBACK(mcux_flexcan_transfer_callback)
 {
 	struct mcux_flexcan_data *data = (struct mcux_flexcan_data *)userData;
+	const struct mcux_flexcan_config *config = data->dev->config;
 
 	switch (status) {
 	case kStatus_FLEXCAN_UnHandled:
@@ -654,6 +653,7 @@ static FLEXCAN_CALLBACK(mcux_flexcan_transfer_callback)
 		mcux_flexcan_transfer_error_status(data->dev, (uint64_t)result);
 		break;
 	case kStatus_FLEXCAN_TxSwitchToRx:
+		FLEXCAN_TransferAbortReceive(config->base, &data->handle, (uint64_t)result);
 		__fallthrough;
 	case kStatus_FLEXCAN_TxIdle:
 		/* The result field is a MB value which is limited to 32bit value */