Skip to content

Commit fd9d02f

Browse files
syaakovsyaakov
committed
added mana driver support
Signed-off-by: syaakov <[email protected]>
1 parent 86fefb8 commit fd9d02f

16 files changed

+292
-2
lines changed

‎doc/trex_appendix_mana.asciidoc‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
MANA support
2+
===========
3+
:quotes.++:
4+
:numbered:
5+
:web_server_url: http://trex-tgn.cisco.com/trex
6+
:local_web_server_url: csi-wiki-01:8181/trex
7+
:toclevels: 4
8+
9+
include::trex_ga.asciidoc[]
10+
11+
== Overview
12+
13+
* Supported from TRex version v3.05.
14+
* This driver does not come with trex pkg and should be build by the user.
15+
* This driver is not included in our regression testing suite. As a result, there may be untested scenarios or issues that have not been identified.
16+
17+
== Configuration And Build
18+
19+
[source,bash]
20+
----
21+
[bash]>cd linux_dpdk/
22+
[bash]>./b configure --no-ofed-check --with-mana
23+
[bash]>./b build
24+
[bash]>cd ../scripts/
25+
[bash]>sudo ./azure_mana_trex_setup.sh
26+
----
27+
28+
== Running
29+
30+
[source,bash]
31+
----
32+
[bash]>sudo ./t-rex-64 -i -c 1 --no-ofed-check
33+
----

‎doc/trex_book.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ To test it you can run `sudo dmidecode -t memory | grep CHANNEL` and check CHANN
154154
| Amazon ENA | paravirtualized | + | - | Amazon Cloud
155155
| MS Failsafe | paravirtualized | + | - | Azure with DPDK mlx5 support
156156
| memif | shared memory rings | + | - | see link:https://doc.dpdk.org/guides/nics/memif.html[memif] support multi core for STL. ASTF only one core
157+
| MANA | paravirtualized | - | - | Microsoft Azure
157158
|=================
158159

159160
[NOTE]
@@ -2426,6 +2427,9 @@ link:trex_appendix_software_mode.html[software mode]
24262427
anchor:azure[]
24272428
link:trex_appendix_azure.html[Azure DPDK support]
24282429

2430+
anchor:mana_support[]
2431+
link:trex_appendix_mana.html[Mana support]
2432+
24292433

24302434

24312435

‎doc/ws_main.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -800,7 +800,8 @@ def build(bld):
800800
'active_flows',
801801
'fixing_pcaps',
802802
'software_mode',
803-
'azure'
803+
'azure',
804+
'mana'
804805
]
805806
for appendix_name in toc_appendixes:
806807
src_name = 'trex_appendix_%s.asciidoc waf.css' % appendix_name

‎linux_dpdk/scripts/azure_mana_trex_setup.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
3+
ip link
4+
5+
PRIMARY1="eth1"
6+
SECONDARY1="`ip -br link show master $PRIMARY1 | awk '{ print $1 }'`"
7+
# Get mac address for MANA interface (should match primary)
8+
MANA_MAC1="`ip -br link show master $PRIMARY1 | awk '{ print $3 }'`"
9+
# get MANA device bus info to pass to TREX/DPDK
10+
BUS_INFO1="`ethtool -i $SECONDARY1 | grep bus-info | awk '{ print $2 }'`"
11+
12+
sudo ethtool -K $PRIMARY1 gso off gro off tso off lro off
13+
sudo ethtool -K $SECONDARY1 gso off gro off tso off
14+
sleep 2
15+
16+
# Set MANA inetrfaces DOWN bore starting TREX/DPDK
17+
sudo ip link set $PRIMARY1 down
18+
sudo ip link set $SECONDARY1 down
19+
sleep 2
20+
21+
PRIMARY2="eth2"
22+
SECONDARY2="`ip -br link show master $PRIMARY2 | awk '{ print $1 }'`"
23+
# Get mac address for MANA interface (should match primary)
24+
MANA_MAC2="`ip -br link show master $PRIMARY2 | awk '{ print $3 }'`"
25+
# get MANA device bus info to pass to TREX/DPDK
26+
BUS_INFO2="`ethtool -i $SECONDARY2 | grep bus-info | awk '{ print $2 }'`"
27+
28+
sudo ethtool -K $PRIMARY2 gso off gro off tso off lro off
29+
sudo ethtool -K $SECONDARY2 gso off gro off tso off
30+
sleep 2
31+
32+
# Set MANA insterfaces DOWN before starting TREX/DPDK
33+
sudo ip link set $PRIMARY2 down
34+
sudo ip link set $SECONDARY2 down
35+
sleep 2
36+
37+
sudo modprobe uio_hv_generic
38+
sleep 2
39+
40+
NET_UUID="f8615163-df3e-46c5-913f-f2d2f965ed0e"
41+
echo $NET_UUID | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/new_id
42+
43+
echo " Unbind $PRIMARY1 from kernel"
44+
DEV_UUID1=$(basename $(readlink /sys/class/net/$PRIMARY1/device))
45+
echo $DEV_UUID1 | sudo tee /sys/bus/vmbus/drivers/hv_netvsc/unbind
46+
echo " Bind $PRIMARY1 to UIO"
47+
echo $DEV_UUID1 | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/bind
48+
49+
echo " Unbind $PRIMARY2 from kernel"
50+
DEV_UUID2=$(basename $(readlink /sys/class/net/$PRIMARY2/device))
51+
echo $DEV_UUID2 | sudo tee /sys/bus/vmbus/drivers/hv_netvsc/unbind
52+
echo " Bind $PRIMARY2 to UIO"
53+
echo $DEV_UUID2 | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/bind
54+
55+
ip link
56+
57+
echo ""
58+
echo "Info for updating trex_cfg.yaml: "
59+
echo " interfaces: ['$BUS_INFO1', '$BUS_INFO2'] "
60+
echo " ext_dpdk_opt: ['--vdev=net_vdev_netvsc,ignore=1', '--vdev=net_vdev_netvsc,ignore=1'] "
61+
echo " interfaces_vdevs : ['$DEV_UUID1', '$DEV_UUID2'] "
62+
echo ""
63+

‎linux_dpdk/ws_main.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ def options(opt):
147147
opt.add_option('--publish-commit', '--publish_commit', dest='publish_commit', default=False, action='store', help="Specify commit id for 'publish_both' option (Please make sure it's good!)")
148148
opt.add_option('--no-bnxt', dest='no_bnxt', default=False, action='store_true', help="don't use bnxt dpdk driver. use with ./b configure --no-bnxt. no need to run build with it")
149149
opt.add_option('--no-mlx', dest='no_mlx', default=(True if march == 'aarch64' else False), action='store', help="don't use mlx4/mlx5 dpdk driver. use with ./b configure --no-mlx. no need to run build with it")
150+
opt.add_option('--with-mana', dest='with_mana', default=False, action='store_true', help="Use Mana dpdk driver. Use with ./b configure --with-mana.")
150151
opt.add_option('--with-ntacc', dest='with_ntacc', default=False, action='store_true', help="Use Napatech dpdk driver. Use with ./b configure --with-ntacc.")
151152
opt.add_option('--with-bird', default=False, action='store_true', help="Build Bird server. Use with ./b configure --with-bird.")
152153
opt.add_option('--new-memory', default=False, action='store_true', help="Build by new DPDK memory subsystem.")
@@ -823,6 +824,7 @@ def configure(conf):
823824
conf.check_cxx(lib = 'z', errmsg = missing_pkg_msg(fedora = 'zlib-devel', ubuntu = 'zlib1g-dev'))
824825
no_mlx = conf.options.no_mlx
825826
no_bnxt = conf.options.no_bnxt
827+
with_mana = conf.options.with_mana
826828
with_ntacc = conf.options.with_ntacc
827829
with_bird = conf.options.with_bird
828830
with_sanitized = conf.options.sanitized
@@ -860,9 +862,15 @@ def configure(conf):
860862
if conf.env.TAP:
861863
conf.configure_tap(mandatory = False)
862864

865+
conf.env.WITH_MANA = with_mana
863866
conf.env.WITH_NTACC = with_ntacc
864867
conf.env.WITH_BIRD = with_bird
865868

869+
if with_mana:
870+
conf.get_ld_search_path(mandatory = True)
871+
conf.check_cxx(lib = 'mana', errmsg = 'Could not find library mana, will use internal version.', mandatory = False)
872+
Logs.pprint('YELLOW', 'Building mana PMD')
873+
866874
if with_ntacc:
867875
ntapi_ok = conf.check_ntapi(mandatory = False)
868876
if not ntapi_ok:
@@ -1752,6 +1760,17 @@ def getstatusoutput(cmd):
17521760

17531761
])
17541762

1763+
mana_dpdk_src = SrcGroup(
1764+
dir = 'src/dpdk/drivers/',
1765+
src_list = [
1766+
'net/mana/gdma.c',
1767+
'net/mana/mana.c',
1768+
'net/mana/mp.c',
1769+
'net/mana/mr.c',
1770+
'net/mana/rx.c',
1771+
'net/mana/tx.c',
1772+
])
1773+
17551774
ntacc_dpdk_src = SrcGroup(dir='src/dpdk/drivers/net/ntacc',
17561775
src_list=[
17571776
'filter_ntacc.c',
@@ -1926,6 +1945,10 @@ def getstatusoutput(cmd):
19261945
libmnl_src
19271946
])
19281947

1948+
mana_dpdk =SrcGroups([
1949+
mana_dpdk_src
1950+
])
1951+
19291952
ntacc_dpdk =SrcGroups([
19301953
ntacc_dpdk_src
19311954
])
@@ -2128,6 +2151,7 @@ def getstatusoutput(cmd):
21282151
../src/dpdk/drivers/net/ixgbe/base/
21292152
../src/dpdk/drivers/net/igc/
21302153
../src/dpdk/drivers/net/igc/base/
2154+
../src/dpdk/drivers/net/mana/
21312155
../src/dpdk/drivers/net/mlx4/
21322156
../src/dpdk/drivers/net/mlx5/
21332157
../src/dpdk/drivers/net/ntacc/
@@ -2342,6 +2366,9 @@ def get_target_l2fwd (self):
23422366
def get_dpdk_target (self):
23432367
return self.update_executable_name("dpdk")
23442368

2369+
def get_mana_target (self):
2370+
return self.update_executable_name("mana")
2371+
23452372
def get_ntacc_target (self):
23462373
return self.update_executable_name("ntacc")
23472374

@@ -2354,6 +2381,9 @@ def get_libmnl_target (self):
23542381
def get_mlx4_target (self):
23552382
return self.update_executable_name("mlx4")
23562383

2384+
def get_manaso_target (self):
2385+
return self.update_executable_name("libmana")+'.so'
2386+
23572387
def get_ntaccso_target (self):
23582388
return self.update_executable_name("libntacc")+'.so'
23592389

@@ -2626,6 +2656,17 @@ def build_prog (bld, build_obj):
26262656
target = build_obj.get_mlx4_target()
26272657
)
26282658

2659+
if bld.env.WITH_MANA == True:
2660+
bld.shlib(
2661+
feature = 'c',
2662+
includes = dpdk_includes_path +
2663+
bld.env.dpdk_includes_verb_path,
2664+
cflags = (cflags + DPDK_FLAGS),
2665+
use = ['ibverbs', 'mana'],
2666+
source = mana_dpdk.file_list(top),
2667+
target = build_obj.get_mana_target()
2668+
)
2669+
26292670
if bld.env.WITH_NTACC == True:
26302671
bld.shlib(
26312672
features='c',
@@ -2756,6 +2797,16 @@ def build(bld):
27562797
bld.env.libmnl_path=' \n ../external_libs/libmnl/include/ \n'
27572798
bld.env.mlx5_kw = {}
27582799

2800+
if bld.env.WITH_MANA == True:
2801+
Logs.pprint('GREEN', 'Info: Using external libverbs libmana.')
2802+
if not bld.env.LD_SEARCH_PATH:
2803+
bld.fatal('LD_SEARCH_PATH is empty, run configure')
2804+
from waflib.Tools.ccroot import SYSTEM_LIB_PATHS
2805+
SYSTEM_LIB_PATHS.extend(bld.env.LD_SEARCH_PATH)
2806+
2807+
bld.read_shlib(name='ibverbs')
2808+
bld.read_shlib(name='mana')
2809+
27592810
if bld.env.WITH_BIRD:
27602811
bld(rule=build_bird, source='compile_bird.py', target='bird')
27612812

@@ -2809,6 +2860,11 @@ def install_single_system (bld, exec_p, build_obj):
28092860

28102861
# SO libraries below
28112862

2863+
# MANA
2864+
do_create_link(src = os.path.realpath(o + build_obj.get_manaso_target()),
2865+
name = build_obj.get_manaso_target(),
2866+
where = so_path)
2867+
28122868
# NTACC
28132869
do_create_link(src = os.path.realpath(o + build_obj.get_ntaccso_target()),
28142870
name = build_obj.get_ntaccso_target(),

‎scripts/azure_mana_trex_setup.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
ip link
4+
5+
PRIMARY1="eth1"
6+
SECONDARY1="`ip -br link show master $PRIMARY1 | awk '{ print $1 }'`"
7+
# Get mac address for MANA interface (should match primary)
8+
MANA_MAC1="`ip -br link show master $PRIMARY1 | awk '{ print $3 }'`"
9+
# get MANA device bus info to pass to TREX/DPDK
10+
BUS_INFO1="`ethtool -i $SECONDARY1 | grep bus-info | awk '{ print $2 }'`"
11+
12+
sudo ethtool -K $PRIMARY1 gso off gro off tso off lro off
13+
sudo ethtool -K $SECONDARY1 gso off gro off tso off
14+
sleep 2
15+
16+
# Set MANA inetrfaces DOWN bore starting TREX/DPDK
17+
sudo ip link set $PRIMARY1 down
18+
sudo ip link set $SECONDARY1 down
19+
sleep 2
20+
21+
PRIMARY2="eth2"
22+
SECONDARY2="`ip -br link show master $PRIMARY2 | awk '{ print $1 }'`"
23+
# Get mac address for MANA interface (should match primary)
24+
MANA_MAC2="`ip -br link show master $PRIMARY2 | awk '{ print $3 }'`"
25+
# get MANA device bus info to pass to TREX/DPDK
26+
BUS_INFO2="`ethtool -i $SECONDARY2 | grep bus-info | awk '{ print $2 }'`"
27+
28+
sudo ethtool -K $PRIMARY2 gso off gro off tso off lro off
29+
sudo ethtool -K $SECONDARY2 gso off gro off tso off
30+
sleep 2
31+
32+
# Set MANA insterfaces DOWN before starting TREX/DPDK
33+
sudo ip link set $PRIMARY2 down
34+
sudo ip link set $SECONDARY2 down
35+
sleep 2
36+
37+
sudo modprobe uio_hv_generic
38+
sleep 2
39+
40+
NET_UUID="f8615163-df3e-46c5-913f-f2d2f965ed0e"
41+
echo $NET_UUID | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/new_id
42+
43+
echo " Unbind $PRIMARY1 from kernel"
44+
DEV_UUID1=$(basename $(readlink /sys/class/net/$PRIMARY1/device))
45+
echo $DEV_UUID1 | sudo tee /sys/bus/vmbus/drivers/hv_netvsc/unbind
46+
echo " Bind $PRIMARY1 to UIO"
47+
echo $DEV_UUID1 | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/bind
48+
49+
echo " Unbind $PRIMARY2 from kernel"
50+
DEV_UUID2=$(basename $(readlink /sys/class/net/$PRIMARY2/device))
51+
echo $DEV_UUID2 | sudo tee /sys/bus/vmbus/drivers/hv_netvsc/unbind
52+
echo " Bind $PRIMARY2 to UIO"
53+
echo $DEV_UUID2 | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/bind
54+
55+
ip link
56+
57+
echo ""
58+
echo "Info for updating trex_cfg.yaml: "
59+
echo " interfaces: ['$BUS_INFO1', '$BUS_INFO2'] "
60+
echo " ext_dpdk_opt: ['--vdev=net_vdev_netvsc,ignore=1', '--vdev=net_vdev_netvsc,ignore=1'] "
61+
echo " interfaces_vdevs : ['$DEV_UUID1', '$DEV_UUID2'] "
62+
echo ""

‎scripts/dpdk_nic_bind.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def fix_path():
7979
# list of supported DPDK drivers
8080
# ,
8181

82-
dpdk_and_kernel=[ "mlx5_core", "mlx5_ib", 'mlx4_core', 'mlx4_ib' ]
82+
dpdk_and_kernel=[ "mlx5_core", "mlx5_ib", 'mlx4_core', 'mlx4_ib', "mana", "mana_ib" ]
8383

8484
if march == 'ppc64le':
8585
dpdk_drivers = ["vfio-pci"]

‎scripts/dpdk_setup_ports.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,11 @@
4040
# 32 : no errors - mlx5 share object should be loaded
4141
# 48 : no errors - both mlx4/mlx5 share object should be loaded
4242
# 64 : no errors - napatech 3GD should be running
43+
# 80 : no errors - mana shared object shoould be loaded
4344
MLX4_EXIT_CODE = 16
4445
MLX5_EXIT_CODE = 32
4546
NTACC_EXIT_CODE = 64
47+
MANA_EXIT_CODE = 80
4648
class VFIOBindErr(Exception): pass
4749
class PCIgenericBindErr(Exception): pass
4850

@@ -967,6 +969,7 @@ def do_run (self, only_check_all_mlx=False):
967969

968970

969971
Broadcom_cnt=0;
972+
Mana_cnt=0
970973
# check how many mellanox cards we have
971974
Mellanox_cnt=0;
972975
mlx5_present=0;
@@ -993,6 +996,8 @@ def do_run (self, only_check_all_mlx=False):
993996
mlx4_present = MLX4_EXIT_CODE
994997
if 'Broadcom' in self.m_devices[key]['Vendor_str']:
995998
Broadcom_cnt += 1
999+
if 'Microsoft' in self.m_devices[key]['Vendor_str']:
1000+
Mana_cnt += 1
9961001

9971002
if not (pa() and pa().dump_interfaces):
9981003
if (Mellanox_cnt > 0) and ((Mellanox_cnt + dummy_cnt) != len(if_list)):
@@ -1103,6 +1108,8 @@ def do_run (self, only_check_all_mlx=False):
11031108
raise DpdkSetup('Unable to bind interfaces to driver igb_uio.')
11041109
elif Mellanox_cnt:
11051110
return mlx5_present + mlx4_present
1111+
elif Mana_cnt:
1112+
return MANA_EXIT_CODE
11061113
elif Napatech_cnt:
11071114
return NTACC_EXIT_CODE
11081115

‎scripts/t-rex-64‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ elif [ $RESULT -eq 2 ]; then
1919
echo "ERROR encountered while configuring TRex system"
2020
exit $RESULT
2121

22+
elif [ $RESULT -eq 80 ]; then
23+
EXTRA_INPUT_ARGS="--mana-so"
24+
2225
elif [ $RESULT -eq 64 ]; then
2326
EXTRA_INPUT_ARGS="--ntacc-so"
2427

‎src/drivers/trex_driver_base.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,9 +151,11 @@ CTRexExtendedDriverDb::CTRexExtendedDriverDb() {
151151
register_driver(std::string("net_vmxnet3"), CTRexExtendedDriverVmxnet3::create);
152152
register_driver(std::string("net_virtio"), CTRexExtendedDriverVirtio::create);
153153
register_driver(std::string("net_ena"),CTRexExtendedDriverVirtio::create);
154+
register_driver(std::string("net_mana"),CTRexExtendedDriverVirtio::create);
154155
register_driver(std::string("net_iavf"), CTRexExtendedDriverIavf::create);
155156
register_driver(std::string("net_i40e_vf"), CTRexExtendedDriverIavf::create);
156157
register_driver(std::string("net_ixgbe_vf"), CTRexExtendedDriverIxgbevf::create);
158+
register_driver(std::string("net_mana"), CTRexExtendedDriverMana::create);
157159
register_driver(std::string("net_netvsc"), CTRexExtendedDriverNetvsc::create);
158160
register_driver(std::string("net_bonding"), CTRexExtendedDriverBonding::create);
159161

0 commit comments

Comments
 (0)