RDMA/mana_ib: add support of multiple ports

If the HW indicates support of multiple ports for rdma, create an IB device
with a port per netdev in the ethernet mana driver.

CM is only available on port 1, but RC QPs are supported on all
ports.

Signed-off-by: Konstantin Taranov <[email protected]>
Link: https://patch.msgid.link/[email protected]
Signed-off-by: Leon Romanovsky <[email protected]>

Author: Konstantin Taranov

drivers/infiniband/hw/mana/device.c

@@ -77,28 +77,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
 	struct gdma_context *gc = dev->gdma_dev->gdma_context;
 	struct mana_context *mc = gc->mana.driver_data;
 	struct net_device *ndev;
+	int i;
 
 	/* Only process events from our parent device */
-	if (event_dev != mc->ports[0])
-		return NOTIFY_DONE;
-
-	switch (event) {
-	case NETDEV_CHANGEUPPER:
-		ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
-		/*
-		 * RDMA core will setup GID based on updated netdev.
-		 * It's not possible to race with the core as rtnl lock is being
-		 * held.
-		 */
-		ib_device_set_netdev(&dev->ib_dev, ndev, 1);
-
-		/* mana_get_primary_netdev() returns ndev with refcount held */
-		netdev_put(ndev, &dev->dev_tracker);
-
-		return NOTIFY_OK;
-	default:
-		return NOTIFY_DONE;
-	}
+	for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
+		if (event_dev == mc->ports[i]) {
+			switch (event) {
+			case NETDEV_CHANGEUPPER:
+				ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
+				/*
+				 * RDMA core will setup GID based on updated netdev.
+				 * It's not possible to race with the core as rtnl lock is being
+				 * held.
+				 */
+				ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
+
+				/* mana_get_primary_netdev() returns ndev with refcount held */
+				if (ndev)
+					netdev_put(ndev, &dev->dev_tracker);
+
+				return NOTIFY_OK;
+			default:
+				return NOTIFY_DONE;
+			}
+		}
+	return NOTIFY_DONE;
 }
 
 static int mana_ib_probe(struct auxiliary_device *adev,
@@ -111,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 	struct net_device *ndev;
 	struct mana_ib_dev *dev;
 	u8 mac_addr[ETH_ALEN];
-	int ret;
+	int ret, i;
 
 	dev = ib_alloc_device(mana_ib_dev, ib_dev);
 	if (!dev)
@@ -126,34 +129,11 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 
 	if (mana_ib_is_rnic(dev)) {
 		dev->ib_dev.phys_port_cnt = 1;
-		ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
-		if (!ndev) {
-			ret = -ENODEV;
-			ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
-			goto free_ib_device;
-		}
-		ether_addr_copy(mac_addr, ndev->dev_addr);
-		addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
-		ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
-		/* mana_get_primary_netdev() returns ndev with refcount held */
-		netdev_put(ndev, &dev->dev_tracker);
-		if (ret) {
-			ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
-			goto free_ib_device;
-		}
-
-		dev->nb.notifier_call = mana_ib_netdev_event;
-		ret = register_netdevice_notifier(&dev->nb);
-		if (ret) {
-			ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
-				  ret);
-			goto free_ib_device;
-		}
-
+		addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
 		ret = mana_ib_gd_query_adapter_caps(dev);
 		if (ret) {
 			ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
-			goto deregister_net_notifier;
+			goto free_ib_device;
 		}
 
 		ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
@@ -163,16 +143,42 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 		ret = mana_ib_create_eqs(dev);
 		if (ret) {
 			ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
-			goto deregister_net_notifier;
+			goto free_ib_device;
 		}
 
 		ret = mana_ib_gd_create_rnic_adapter(dev);
 		if (ret)
 			goto destroy_eqs;
 
-		ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
+		if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
+			dev->ib_dev.phys_port_cnt = mc->num_ports;
+
+		for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
+			ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
+			if (!ndev) {
+				ret = -ENODEV;
+				ibdev_err(&dev->ib_dev,
+					  "Failed to get netdev for IB port %d", i + 1);
+				goto destroy_rnic;
+			}
+			ether_addr_copy(mac_addr, ndev->dev_addr);
+			ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
+			/* mana_get_primary_netdev() returns ndev with refcount held */
+			netdev_put(ndev, &dev->dev_tracker);
+			if (ret) {
+				ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
+				goto destroy_rnic;
+			}
+			ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
+			if (ret) {
+				ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
+				goto destroy_rnic;
+			}
+		}
+		dev->nb.notifier_call = mana_ib_netdev_event;
+		ret = register_netdevice_notifier(&dev->nb);
 		if (ret) {
-			ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
+			ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
 			goto destroy_rnic;
 		}
 	} else {
@@ -188,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 				       MANA_AV_BUFFER_SIZE, 0);
 	if (!dev->av_pool) {
 		ret = -ENOMEM;
-		goto destroy_rnic;
+		goto deregister_net_notifier;
 	}
 
 	ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -205,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 
 deallocate_pool:
 	dma_pool_destroy(dev->av_pool);
+deregister_net_notifier:
+	if (mana_ib_is_rnic(dev))
+		unregister_netdevice_notifier(&dev->nb);
 destroy_rnic:
 	if (mana_ib_is_rnic(dev))
 		mana_ib_gd_destroy_rnic_adapter(dev);
 destroy_eqs:
 	if (mana_ib_is_rnic(dev))
 		mana_ib_destroy_eqs(dev);
-deregister_net_notifier:
-	if (mana_ib_is_rnic(dev))
-		unregister_netdevice_notifier(&dev->nb);
 free_ib_device:
 	xa_destroy(&dev->qp_table_wq);
 	ib_dealloc_device(&dev->ib_dev);
@@ -227,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
 	ib_unregister_device(&dev->ib_dev);
 	dma_pool_destroy(dev->av_pool);
 	if (mana_ib_is_rnic(dev)) {
+		unregister_netdevice_notifier(&dev->nb);
 		mana_ib_gd_destroy_rnic_adapter(dev);
 		mana_ib_destroy_eqs(dev);
-		unregister_netdevice_notifier(&dev->nb);
 	}
 	xa_destroy(&dev->qp_table_wq);
 	ib_dealloc_device(&dev->ib_dev);

drivers/infiniband/hw/mana/main.c

@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 
 	if (mana_ib_is_rnic(dev)) {
-		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
-		immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+		if (port_num == 1) {
+			immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+			immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+		} else {
+			immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
+						    | RDMA_CORE_CAP_ETH_AH;
+			immutable->max_mad_size = 0;
+		}
 	} else {
 		immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
 	}
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
 	props->pkey_tbl_len = 1;
 	if (mana_ib_is_rnic(dev)) {
 		props->gid_tbl_len = 16;
-		props->port_cap_flags = IB_PORT_CM_SUP;
 		props->ip_gids = true;
+		if (port == 1)
+			props->port_cap_flags = IB_PORT_CM_SUP;
 	}
 
 	return 0;

drivers/infiniband/hw/mana/mana_ib.h

@@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req {
 enum mana_ib_adapter_features {
 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
 	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
+	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
 };
 
 struct mana_ib_query_adapter_caps_resp {