Traffic class not fully applied via UCX_IB_TRAFFIC_CLASS in one direction #10325
Description
Describe the bug
While running the line below, I can see that the traffic is almost fully set to DSCP=0x6a (TC=106) on the local node, but the remote node sends all traffic set to DSCP=0. If I revert the nodes (running same line changing the hosts order only), the behavior is the same. Not sure if this is an issue with ucx or nccl.
Local node:
root@ubuntu-1:~# mpirun --mca pml ucx -x NCCL_IB_HCA=mlx5_1:1 -x NCCL_DEBUG=INFO -x UCX_TLS=rc -x UCX_IB_TRAFFIC_CLASS=106 -x UCX_NET_DEVICES=mlx5_1:1 --allow-run-as-root -np 2 --host 192.168.101.100,192.168.103.100 /root/nccl-tests/build/all_reduce_perf -g 2 -n 100000
root@ubuntu-1:~# ethtool -S ens8np0 | egrep prio | egrep -v ": 0" | egrep packets > 1.txt ; sleep 1; ethtool -S ens8np0 | egrep prio | egrep -v ": 0" | egrep packets > 2.txt ; diff -y --suppress-common-lines 1.txt 2.txt | awk '{a=$5-$2;print $0, " DIFF: " a;}'
rx_prio0_packets: 4939696624 | rx_prio0_packets: 4943711983 DIFF: 4015359
tx_prio0_packets: 7537608762 | tx_prio0_packets: 7537659712 DIFF: 50950 --> 50Kpps set to DSCP=0
tx_prio3_packets: 4671266264 | tx_prio3_packets: 4675230410 DIFF: 3964146. -> 3.9Mpps set to DSCP 0x6a
Remote node:
root@ubuntu-2:~# ethtool -S ens8np0 | egrep prio | egrep -v ": 0" | egrep packets > 1.txt ; sleep 1; ethtool -S ens8np0 | egrep prio | egrep -v ": 0" | egrep packets > 2.txt ; diff -y --suppress-common-lines 1.txt 2.txt | awk '{a=$5-$2;print $0, " DIFF: " a;}'
rx_prio0_packets: 127229074542 | rx_prio0_packets: 127229125669 DIFF: 51127
tx_prio0_packets: 155034442491 | tx_prio0_packets: 155038455955 DIFF: 4013464 --> all set to DSCP=0
rx_prio3_packets: 4824616129 | rx_prio3_packets: 4828578862 DIFF: 3962733
Steps to Reproduce
Run:
mpirun --mca pml ucx -x NCCL_IB_HCA=mlx5_1:1 -x NCCL_DEBUG=INFO -x UCX_TLS=rc -x UCX_IB_TRAFFIC_CLASS=106 -x UCX_NET_DEVICES=mlx5_1:1 --allow-run-as-root -np 2 --host 192.168.101.100,192.168.103.100 /root/nccl-tests/build/all_reduce_perf -g 2 -n 100000
- UCX version used (from github branch XX or release YY) + UCX configure flags (can be checked by
ucx_info -v)
root@ubuntu-1:~# ucx_info -v
UCT version=1.12.1 revision 0000000
configured with: --build=x86_64-linux-gnu --prefix=/usr --includedir=${prefix}/include --mandir=${prefix}/share/man --infodir=${prefix}/share/info --sysconfdir=/etc --localstatedir=/var --disable-option-checking --disable-silent-rules --libdir=${prefix}/lib/x86_64-linux-gnu --runstatedir=/run --disable-maintainer-mode --disable-dependency-tracking --enable-mt --with-verbs --disable-backtrace-detail --disable-logging --enable-devel-headers --enable-examples --enable-cma
root@ubuntu-2:~# ucx_info -v
UCT version=1.12.1 revision 0000000
configured with: --build=x86_64-linux-gnu --prefix=/usr --includedir=${prefix}/include --mandir=${prefix}/share/man --infodir=${prefix}/share/info --sysconfdir=/etc --localstatedir=/var --disable-option-checking --disable-silent-rules --libdir=${prefix}/lib/x86_64-linux-gnu --runstatedir=/run --disable-maintainer-mode --disable-dependency-tracking --enable-mt --with-verbs --disable-backtrace-detail --disable-logging --enable-devel-headers --enable-examples --enable-cma
- Any UCX environment variables used
-x UCX_IB_TRAFFIC_CLASS=106 -x UCX_NET_DEVICES=mlx5_1:1
NCCL variables:
root@ubuntu-1:~# set | egrep NCCL
NCCL_HOME=/usr/local
NCCL_IB_HCA=mlx5_1:1
NCCL_IB_SL=10
NCCL_IB_TC=106
NCCL_NET_ADDRESS=192.168.101.100
NCCL_NET_IF=ens8np0
NCCL_NET_PEER=192.168.103.100:10000
NCCL_NET_PORT=10000
NCCL_OOB_NET_ENABLE=0
NCCL_SOCKET_IFNAME=ens8np0
root@ubuntu-2:~# set | egrep NCCL
NCCL_HOME=/usr/local
NCCL_IB_HCA=mlx5_1:1
NCCL_IB_SL=10
NCCL_IB_TC=106
NCCL_NET_ADDRESS=192.168.103.100
NCCL_NET_IF=ens8np0
NCCL_NET_PEER=192.168.101.100:10000
NCCL_NET_PORT=10000
NCCL_OOB_NET_ENABLE=0
NCCL_SOCKET_IFNAME=ens8np0
Setup and versions
- OS version (e.g Linux distro) + CPU architecture (x86_64/aarch64/ppc64le/...)
cat /etc/issueorcat /etc/redhat-release+uname -a
root@ubuntu-1:# cat /etc/issue# uname -a
Ubuntu 22.04.5 LTS \n \l
root@ubuntu-1:
Linux ubuntu-1 5.15.0-125-generic #135-Ubuntu SMP Fri Sep 27 13:53:58 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux
- For Nvidia Bluefield SmartNIC include
cat /etc/mlnx-release(the string identifies software and firmware setup)
root@ubuntu-1:~# ibstat
CA 'mlx5_0'
CA type: MT4129
Number of ports: 1
Firmware version: 28.42.1000
Hardware version: 0
Node GUID: 0xa088c20300c7b626
System image GUID: 0xa088c20300c7b626
Port 1:
State: Active
Physical state: LinkUp
Rate: 400
Base lid: 0
LMC: 0
SM lid: 0
Capability mask: 0x00010000
Port GUID: 0xa288c2fffec7b626
Link layer: Ethernet
CA 'mlx5_1'
CA type: MT4129
Number of ports: 1
Firmware version: 28.42.1000
Hardware version: 0
Node GUID: 0xa088c20300caeaa8
System image GUID: 0xa088c20300caeaa8
Port 1:
State: Active
Physical state: LinkUp
Rate: 400
Base lid: 0
LMC: 0
SM lid: 0
Capability mask: 0x00010000
Port GUID: 0xa288c2fffecaeaa8
Link layer: Ethernet
- For RDMA/IB/RoCE related issues:
root@ubuntu-1:~# ibv_devinfo -vv
hca_id: mlx5_0
transport: InfiniBand (0)
fw_ver: 28.42.1000
node_guid: a088:c203:00c7:b626
sys_image_guid: a088:c203:00c7:b626
vendor_id: 0x02c9
vendor_part_id: 4129
hw_ver: 0x0
board_id: MT_0000000838
phys_port_cnt: 1
max_mr_size: 0xffffffffffffffff
page_size_cap: 0xfffffffffffff000
max_qp: 131072
max_qp_wr: 32768
device_cap_flags: 0xed721c36
BAD_PKEY_CNTR
BAD_QKEY_CNTR
AUTO_PATH_MIG
CHANGE_PHY_PORT
PORT_ACTIVE_EVENT
SYS_IMAGE_GUID
RC_RNR_NAK_GEN
MEM_WINDOW
XRC
MEM_MGT_EXTENSIONS
MEM_WINDOW_TYPE_2B
RAW_IP_CSUM
MANAGED_FLOW_STEERING
Unknown flags: 0xC8400000
max_sge: 30
max_sge_rd: 30
max_cq: 16777216
max_cqe: 4194303
max_mr: 16777216
max_pd: 8388608
max_qp_rd_atom: 16
max_ee_rd_atom: 0
max_res_rd_atom: 2097152
max_qp_init_rd_atom: 16
max_ee_init_rd_atom: 0
atomic_cap: ATOMIC_HCA (1)
max_ee: 0
max_rdd: 0
max_mw: 16777216
max_raw_ipv6_qp: 0
max_raw_ethy_qp: 0
max_mcast_grp: 2097152
max_mcast_qp_attach: 240
max_total_mcast_qp_attach: 503316480
max_ah: 2147483647
max_fmr: 0
max_srq: 8388608
max_srq_wr: 32767
max_srq_sge: 31
max_pkeys: 128
local_ca_ack_delay: 16
general_odp_caps:
ODP_SUPPORT
ODP_SUPPORT_IMPLICIT
rc_odp_caps:
SUPPORT_SEND
SUPPORT_RECV
SUPPORT_WRITE
SUPPORT_READ
SUPPORT_ATOMIC
SUPPORT_SRQ
uc_odp_caps:
NO SUPPORT
ud_odp_caps:
SUPPORT_SEND
xrc_odp_caps:
SUPPORT_SEND
SUPPORT_WRITE
SUPPORT_READ
SUPPORT_ATOMIC
SUPPORT_SRQ
completion timestamp_mask: 0x7fffffffffffffff
hca_core_clock: 1000000kHZ
raw packet caps:
C-VLAN stripping offload
Scatter FCS offload
IP csum offload
Delay drop
device_cap_flags_ex: 0x15ED721C36
RAW_SCATTER_FCS
PCI_WRITE_END_PADDING
Unknown flags: 0x100000000
tso_caps:
max_tso: 262144
supported_qp:
SUPPORT_RAW_PACKET
rss_caps:
max_rwq_indirection_tables: 1048576
max_rwq_indirection_table_size: 2048
rx_hash_function: 0x1
rx_hash_fields_mask: 0x800000FF
supported_qp:
SUPPORT_RAW_PACKET
max_wq_type_rq: 8388608
packet_pacing_caps:
qp_rate_limit_min: 1kbps
qp_rate_limit_max: 400000000kbps
supported_qp:
SUPPORT_RAW_PACKET
tag matching not supported
cq moderation caps:
max_cq_count: 65535
max_cq_period: 4095 us
maximum available device memory: 262144Bytes
num_comp_vectors: 63
port: 1
state: PORT_ACTIVE (4)
max_mtu: 4096 (5)
active_mtu: 4096 (5)
sm_lid: 0
port_lid: 0
port_lmc: 0x00
link_layer: Ethernet
max_msg_sz: 0x40000000
port_cap_flags: 0x04010000
port_cap_flags2: 0x0000
max_vl_num: invalid value (0)
bad_pkey_cntr: 0x0
qkey_viol_cntr: 0x0
sm_sl: 0
pkey_tbl_len: 1
gid_tbl_len: 255
subnet_timeout: 0
init_type_reply: 0
active_width: 4X (2)
active_speed: 100.0 Gbps (128)
phys_state: LINK_UP (5)
GID[ 0]: fe80:0000:0000:0000:a288:c2ff:fec7:b626, RoCE v1
GID[ 1]: fe80::a288:c2ff:fec7:b626, RoCE v2
GID[ 2]: 0000:0000:0000:0000:0000:ffff:c0a8:6464, RoCE v1
GID[ 3]: ::ffff:192.168.100.100, RoCE v2
hca_id: mlx5_1
transport: InfiniBand (0)
fw_ver: 28.42.1000
node_guid: a088:c203:00ca:eaa8
sys_image_guid: a088:c203:00ca:eaa8
vendor_id: 0x02c9
vendor_part_id: 4129
hw_ver: 0x0
board_id: MT_0000000838
phys_port_cnt: 1
max_mr_size: 0xffffffffffffffff
page_size_cap: 0xfffffffffffff000
max_qp: 131072
max_qp_wr: 32768
device_cap_flags: 0xed721c36
BAD_PKEY_CNTR
BAD_QKEY_CNTR
AUTO_PATH_MIG
CHANGE_PHY_PORT
PORT_ACTIVE_EVENT
SYS_IMAGE_GUID
RC_RNR_NAK_GEN
MEM_WINDOW
XRC
MEM_MGT_EXTENSIONS
MEM_WINDOW_TYPE_2B
RAW_IP_CSUM
MANAGED_FLOW_STEERING
Unknown flags: 0xC8400000
max_sge: 30
max_sge_rd: 30
max_cq: 16777216
max_cqe: 4194303
max_mr: 16777216
max_pd: 8388608
max_qp_rd_atom: 16
max_ee_rd_atom: 0
max_res_rd_atom: 2097152
max_qp_init_rd_atom: 16
max_ee_init_rd_atom: 0
atomic_cap: ATOMIC_HCA (1)
max_ee: 0
max_rdd: 0
max_mw: 16777216
max_raw_ipv6_qp: 0
max_raw_ethy_qp: 0
max_mcast_grp: 2097152
max_mcast_qp_attach: 240
max_total_mcast_qp_attach: 503316480
max_ah: 2147483647
max_fmr: 0
max_srq: 8388608
max_srq_wr: 32767
max_srq_sge: 31
max_pkeys: 128
local_ca_ack_delay: 16
general_odp_caps:
ODP_SUPPORT
ODP_SUPPORT_IMPLICIT
rc_odp_caps:
SUPPORT_SEND
SUPPORT_RECV
SUPPORT_WRITE
SUPPORT_READ
SUPPORT_ATOMIC
SUPPORT_SRQ
uc_odp_caps:
NO SUPPORT
ud_odp_caps:
SUPPORT_SEND
xrc_odp_caps:
SUPPORT_SEND
SUPPORT_WRITE
SUPPORT_READ
SUPPORT_ATOMIC
SUPPORT_SRQ
completion timestamp_mask: 0x7fffffffffffffff
hca_core_clock: 1000000kHZ
raw packet caps:
C-VLAN stripping offload
Scatter FCS offload
IP csum offload
Delay drop
device_cap_flags_ex: 0x15ED721C36
RAW_SCATTER_FCS
PCI_WRITE_END_PADDING
Unknown flags: 0x100000000
tso_caps:
max_tso: 262144
supported_qp:
SUPPORT_RAW_PACKET
rss_caps:
max_rwq_indirection_tables: 1048576
max_rwq_indirection_table_size: 2048
rx_hash_function: 0x1
rx_hash_fields_mask: 0x800000FF
supported_qp:
SUPPORT_RAW_PACKET
max_wq_type_rq: 8388608
packet_pacing_caps:
qp_rate_limit_min: 1kbps
qp_rate_limit_max: 400000000kbps
supported_qp:
SUPPORT_RAW_PACKET
tag matching not supported
cq moderation caps:
max_cq_count: 65535
max_cq_period: 4095 us
maximum available device memory: 262144Bytes
num_comp_vectors: 63
port: 1
state: PORT_ACTIVE (4)
max_mtu: 4096 (5)
active_mtu: 4096 (5)
sm_lid: 0
port_lid: 0
port_lmc: 0x00
link_layer: Ethernet
max_msg_sz: 0x40000000
port_cap_flags: 0x04010000
port_cap_flags2: 0x0000
max_vl_num: invalid value (0)
bad_pkey_cntr: 0x0
qkey_viol_cntr: 0x0
sm_sl: 0
pkey_tbl_len: 1
gid_tbl_len: 255
subnet_timeout: 0
init_type_reply: 0
active_width: 4X (2)
active_speed: 100.0 Gbps (128)
phys_state: LINK_UP (5)
GID[ 0]: fe80:0000:0000:0000:a288:c2ff:feca:eaa8, RoCE v1
GID[ 1]: fe80::a288:c2ff:feca:eaa8, RoCE v2
GID[ 2]: 0000:0000:0000:0000:0000:ffff:c0a8:6564, RoCE v1
GID[ 3]: ::ffff:192.168.101.100, RoCE v2
- For GPU related issues:
- GPU type
- Cuda:
- Drivers version
- Check if peer-direct is loaded:
lsmod|grep nv_peer_memand/or gdrcopy:lsmod|grep gdrdrv
Additional information (depending on the issue)
root@ubuntu-1:~# ucx_info -d | sed 's/^#//g'
Memory domain: tcp
Component: tcp
register: unlimited, cost: 0 nsec
remote key: 0 bytes
Transport: tcp
Device: ens8np0
Type: network
System device:
capabilities:
bandwidth: 47281.80/ppn + 0.00 MB/sec
latency: 5201 nsec
overhead: 50000 nsec
put_zcopy: <= 18446744073709551590, up to 6 iov
put_opt_zcopy_align: <= 1
put_align_mtu: <= 0
am_short: <= 8K
am_bcopy: <= 8K
am_zcopy: <= 64K, up to 6 iov
am_opt_zcopy_align: <= 1
am_align_mtu: <= 0
am header: <= 8037
connection: to ep, to iface
device priority: 1
device num paths: 1
max eps: 256
device address: 6 bytes
iface address: 2 bytes
ep address: 10 bytes
error handling: peer failure, ep_check, keepalive
Transport: tcp
Device: ens15f0np0
Type: network
System device:
capabilities:
bandwidth: 113.16/ppn + 0.00 MB/sec
latency: 5776 nsec
overhead: 50000 nsec
put_zcopy: <= 18446744073709551590, up to 6 iov
put_opt_zcopy_align: <= 1
put_align_mtu: <= 0
am_short: <= 8K
am_bcopy: <= 8K
am_zcopy: <= 64K, up to 6 iov
am_opt_zcopy_align: <= 1
am_align_mtu: <= 0
am header: <= 8037
connection: to ep, to iface
device priority: 0
device num paths: 1
max eps: 256
device address: 6 bytes
iface address: 2 bytes
ep address: 10 bytes
error handling: peer failure, ep_check, keepalive
Transport: tcp
Device: ens7np0
Type: network
System device:
capabilities:
bandwidth: 47281.80/ppn + 0.00 MB/sec
latency: 5201 nsec
overhead: 50000 nsec
put_zcopy: <= 18446744073709551590, up to 6 iov
put_opt_zcopy_align: <= 1
put_align_mtu: <= 0
am_short: <= 8K
am_bcopy: <= 8K
am_zcopy: <= 64K, up to 6 iov
am_opt_zcopy_align: <= 1
am_align_mtu: <= 0
am header: <= 8037
connection: to ep, to iface
device priority: 1
device num paths: 1
max eps: 256
device address: 6 bytes
iface address: 2 bytes
ep address: 10 bytes
error handling: peer failure, ep_check, keepalive
Transport: tcp
Device: lo
Type: network
System device:
capabilities:
bandwidth: 11.91/ppn + 0.00 MB/sec
latency: 10960 nsec
overhead: 50000 nsec
put_zcopy: <= 18446744073709551590, up to 6 iov
put_opt_zcopy_align: <= 1
put_align_mtu: <= 0
am_short: <= 8K
am_bcopy: <= 8K
am_zcopy: <= 64K, up to 6 iov
am_opt_zcopy_align: <= 1
am_align_mtu: <= 0
am header: <= 8037
connection: to ep, to iface
device priority: 1
device num paths: 1
max eps: 256
device address: 18 bytes
iface address: 2 bytes
ep address: 10 bytes
error handling: peer failure, ep_check, keepalive
Connection manager: tcp
max_conn_priv: 2064 bytes
Memory domain: self
Component: self
register: unlimited, cost: 0 nsec
remote key: 0 bytes
Transport: self
Device: memory0
Type: loopback
System device:
capabilities:
bandwidth: 0.00/ppn + 6911.00 MB/sec
latency: 0 nsec
overhead: 10 nsec
put_short: <= 4294967295
put_bcopy: unlimited
get_bcopy: unlimited
am_short: <= 8K
am_bcopy: <= 8K
domain: cpu
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to iface
device priority: 0
device num paths: 1
max eps: inf
device address: 0 bytes
iface address: 8 bytes
error handling: ep_check
Memory domain: sysv
Component: sysv
allocate: unlimited
remote key: 12 bytes
rkey_ptr is supported
Transport: sysv
Device: memory
Type: intra-node
System device:
capabilities:
bandwidth: 0.00/ppn + 12179.00 MB/sec
latency: 80 nsec
overhead: 10 nsec
put_short: <= 4294967295
put_bcopy: unlimited
get_bcopy: unlimited
am_short: <= 100
am_bcopy: <= 8256
domain: cpu
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to iface
device priority: 0
device num paths: 1
max eps: inf
device address: 8 bytes
iface address: 8 bytes
error handling: ep_check
Memory domain: posix
Component: posix
allocate: <= 261892264K
remote key: 24 bytes
rkey_ptr is supported
Transport: posix
Device: memory
Type: intra-node
System device:
capabilities:
bandwidth: 0.00/ppn + 12179.00 MB/sec
latency: 80 nsec
overhead: 10 nsec
put_short: <= 4294967295
put_bcopy: unlimited
get_bcopy: unlimited
am_short: <= 100
am_bcopy: <= 8256
domain: cpu
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to iface
device priority: 0
device num paths: 1
max eps: inf
device address: 8 bytes
iface address: 8 bytes
error handling: ep_check
Memory domain: mlx5_0
Component: ib
register: unlimited, cost: 180 nsec
remote key: 8 bytes
local memory handle is required for zcopy
Transport: ud_mlx5
Device: mlx5_0:1
Type: network
System device: mlx5_0 (0)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 830 nsec
overhead: 80 nsec
am_short: <= 180
am_bcopy: <= 4088
am_zcopy: <= 4088, up to 3 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 132
connection: to ep, to iface
device priority: 70
device num paths: 1
max eps: inf
device address: 17 bytes
iface address: 3 bytes
ep address: 6 bytes
error handling: peer failure, ep_check
Transport: ud_verbs
Device: mlx5_0:1
Type: network
System device: mlx5_0 (0)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 830 nsec
overhead: 105 nsec
am_short: <= 116
am_bcopy: <= 4088
am_zcopy: <= 4088, up to 5 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 3952
connection: to ep, to iface
device priority: 70
device num paths: 1
max eps: inf
device address: 17 bytes
iface address: 3 bytes
ep address: 6 bytes
error handling: peer failure, ep_check
Transport: dc_mlx5
Device: mlx5_0:1
Type: network
System device: mlx5_0 (0)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 860 nsec
overhead: 40 nsec
put_short: <= 2K
put_bcopy: <= 8256
put_zcopy: <= 1G, up to 11 iov
put_opt_zcopy_align: <= 512
put_align_mtu: <= 4K
get_bcopy: <= 8256
get_zcopy: 65..1G, up to 11 iov
get_opt_zcopy_align: <= 512
get_align_mtu: <= 4K
am_short: <= 2046
am_bcopy: <= 8254
am_zcopy: <= 8254, up to 3 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 138
domain: device
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to iface
device priority: 70
device num paths: 1
max eps: inf
device address: 17 bytes
iface address: 5 bytes
error handling: buffer (zcopy), remote access, peer failure, ep_check
Transport: rc_mlx5
Device: mlx5_0:1
Type: network
System device: mlx5_0 (0)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 800 + 1.000 * N nsec
overhead: 40 nsec
put_short: <= 2K
put_bcopy: <= 8256
put_zcopy: <= 1G, up to 14 iov
put_opt_zcopy_align: <= 512
put_align_mtu: <= 4K
get_bcopy: <= 8256
get_zcopy: 65..1G, up to 14 iov
get_opt_zcopy_align: <= 512
get_align_mtu: <= 4K
am_short: <= 2046
am_bcopy: <= 8254
am_zcopy: <= 8254, up to 3 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 186
domain: device
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to ep
device priority: 70
device num paths: 1
max eps: 256
device address: 17 bytes
ep address: 7 bytes
error handling: buffer (zcopy), remote access, peer failure, ep_check
Transport: rc_verbs
Device: mlx5_0:1
Type: network
System device: mlx5_0 (0)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 800 + 1.000 * N nsec
overhead: 75 nsec
put_short: <= 124
put_bcopy: <= 8256
put_zcopy: <= 1G, up to 5 iov
put_opt_zcopy_align: <= 512
put_align_mtu: <= 4K
get_bcopy: <= 8256
get_zcopy: 65..1G, up to 5 iov
get_opt_zcopy_align: <= 512
get_align_mtu: <= 4K
am_short: <= 123
am_bcopy: <= 8255
am_zcopy: <= 8255, up to 4 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 127
domain: device
atomic_add: 64 bit
atomic_fadd: 64 bit
atomic_cswap: 64 bit
connection: to ep
device priority: 70
device num paths: 1
max eps: 256
device address: 17 bytes
ep address: 5 bytes
error handling: peer failure, ep_check
Memory domain: mlx5_1
Component: ib
register: unlimited, cost: 180 nsec
remote key: 8 bytes
local memory handle is required for zcopy
Transport: ud_mlx5
Device: mlx5_1:1
Type: network
System device: mlx5_1 (1)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 830 nsec
overhead: 80 nsec
am_short: <= 180
am_bcopy: <= 4088
am_zcopy: <= 4088, up to 3 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 132
connection: to ep, to iface
device priority: 70
device num paths: 1
max eps: inf
device address: 17 bytes
iface address: 3 bytes
ep address: 6 bytes
error handling: peer failure, ep_check
Transport: ud_verbs
Device: mlx5_1:1
Type: network
System device: mlx5_1 (1)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 830 nsec
overhead: 105 nsec
am_short: <= 116
am_bcopy: <= 4088
am_zcopy: <= 4088, up to 5 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 3952
connection: to ep, to iface
device priority: 70
device num paths: 1
max eps: inf
device address: 17 bytes
iface address: 3 bytes
ep address: 6 bytes
error handling: peer failure, ep_check
Transport: dc_mlx5
Device: mlx5_1:1
Type: network
System device: mlx5_1 (1)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 860 nsec
overhead: 40 nsec
put_short: <= 2K
put_bcopy: <= 8256
put_zcopy: <= 1G, up to 11 iov
put_opt_zcopy_align: <= 512
put_align_mtu: <= 4K
get_bcopy: <= 8256
get_zcopy: 65..1G, up to 11 iov
get_opt_zcopy_align: <= 512
get_align_mtu: <= 4K
am_short: <= 2046
am_bcopy: <= 8254
am_zcopy: <= 8254, up to 3 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 138
domain: device
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to iface
device priority: 70
device num paths: 1
max eps: inf
device address: 17 bytes
iface address: 5 bytes
error handling: buffer (zcopy), remote access, peer failure, ep_check
Transport: rc_mlx5
Device: mlx5_1:1
Type: network
System device: mlx5_1 (1)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 800 + 1.000 * N nsec
overhead: 40 nsec
put_short: <= 2K
put_bcopy: <= 8256
put_zcopy: <= 1G, up to 14 iov
put_opt_zcopy_align: <= 512
put_align_mtu: <= 4K
get_bcopy: <= 8256
get_zcopy: 65..1G, up to 14 iov
get_opt_zcopy_align: <= 512
get_align_mtu: <= 4K
am_short: <= 2046
am_bcopy: <= 8254
am_zcopy: <= 8254, up to 3 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 186
domain: device
atomic_add: 32, 64 bit
atomic_and: 32, 64 bit
atomic_or: 32, 64 bit
atomic_xor: 32, 64 bit
atomic_fadd: 32, 64 bit
atomic_fand: 32, 64 bit
atomic_for: 32, 64 bit
atomic_fxor: 32, 64 bit
atomic_swap: 32, 64 bit
atomic_cswap: 32, 64 bit
connection: to ep
device priority: 70
device num paths: 1
max eps: 256
device address: 17 bytes
ep address: 7 bytes
error handling: buffer (zcopy), remote access, peer failure, ep_check
Transport: rc_verbs
Device: mlx5_1:1
Type: network
System device: mlx5_1 (1)
capabilities:
bandwidth: 45244.61/ppn + 0.00 MB/sec
latency: 800 + 1.000 * N nsec
overhead: 75 nsec
put_short: <= 124
put_bcopy: <= 8256
put_zcopy: <= 1G, up to 5 iov
put_opt_zcopy_align: <= 512
put_align_mtu: <= 4K
get_bcopy: <= 8256
get_zcopy: 65..1G, up to 5 iov
get_opt_zcopy_align: <= 512
get_align_mtu: <= 4K
am_short: <= 123
am_bcopy: <= 8255
am_zcopy: <= 8255, up to 4 iov
am_opt_zcopy_align: <= 512
am_align_mtu: <= 4K
am header: <= 127
domain: device
atomic_add: 64 bit
atomic_fadd: 64 bit
atomic_cswap: 64 bit
connection: to ep
device priority: 70
device num paths: 1
max eps: 256
device address: 17 bytes
ep address: 5 bytes
error handling: peer failure, ep_check
Connection manager: rdmacm
max_conn_priv: 54 bytes
Memory domain: cma
Component: cma
register: unlimited, cost: 9 nsec
Transport: cma
Device: memory
Type: intra-node
System device:
capabilities:
bandwidth: 0.00/ppn + 11145.00 MB/sec
latency: 80 nsec
overhead: 2000 nsec
put_zcopy: unlimited, up to 16 iov
put_opt_zcopy_align: <= 1
put_align_mtu: <= 1
get_zcopy: unlimited, up to 16 iov
get_opt_zcopy_align: <= 1
get_align_mtu: <= 1
connection: to iface
device priority: 0
device num paths: 1
max eps: inf
device address: 8 bytes
iface address: 4 bytes
error handling: peer failure, ep_check