# https://bugzilla.redhat.com/show_bug.cgi?id=1642355#c13
1. Feature description:
i. High Availability and load balancing for Virtual Functions of different physical ports in Switchdev mode.
2)Precondition:
a. Need to create at least 1 VF on both Physical port and unbind the VF’s
b. Move both of the physical devices from legacy to switchdev mode.
c. Enable HW offload and TC ingress for all interfaces
d. Create OVS bridge and bring up the relevant topology.
3)Traffic between VM’s.
a.Need to make sure for load balance , failover and failback.
b.All the relevant scenarios for legacy bonding.
4)Architectures:
a.x86_64
b.ConnectX-5
# https://bugzilla.redhat.com/show_bug.cgi?id=1759449#c16
I'm not sure you even had VF LAG setup successfully..
do you see something like this in dmesg output ?
# dmesg | grep mlx5 | grep map
[ 597.167386] mlx5_core 0000:03:00.0: lag map port 1:1 port 2:1
If not, then VF LAG was not created, and we need to work on fixing that first, so let me know.
Basically we need to:
1. setup VF LAG (with Xor or LACP modes); example steps in https://bugzilla.redhat.com/show_bug.cgi?id=1642355#c18
we create 1 VF on each PF, but we'll use just one VF for traffic test.
(make sure to enable tc hw offload on all PFs and representors).
2. Send multiple TCP/UDP streams from 1 of the VFs to remote host.
3. Check tx_packets_phy/vport_tx_packets counters on both PFs, both of them should increment (almost evenly).
# https://bugzilla.redhat.com/show_bug.cgi?id=1759449#c10
This is different, you created a bond of the VFs, the bug is when you setup VF LAG (see BZ #1642355), that is to put the PFs on the hypervisor in a bond,
then you run traffic from a VF, and check the statistics on the PFs.
https://bugzilla.redhat.com/show_bug.cgi?id=1759449#c17
[root@hp-dl380g10-04 sriov]# dmesg|grep "lag map port"
[66436.239686] mlx5_core 0000:37:00.0: lag map port 1:1 port 2:1
[66520.725904] mlx5_core 0000:37:00.0: modify lag map port 1:1 port 2:2
[66661.787047] mlx5_core 0000:37:00.0: lag map port 1:1 port 2:2
STEPS:
# Create 1 VF on each PF
[root@hp-dl380g10-04 sriov]# ethtool -i ens2f0
driver: mlx5_core
version: 5.0-0
firmware-version: 16.26.1040 (MT_0000000080)
expansion-rom-version:
bus-info: 0000:37:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
[root@hp-dl380g10-04 sriov]# ethtool -i ens2f1
driver: mlx5_core
version: 5.0-0
firmware-version: 16.26.1040 (MT_0000000080)
expansion-rom-version:
bus-info: 0000:37:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
[root@hp-dl380g10-04 sriov]# echo 1 > /sys/bus/pci/devices/0000\:37\:00.0/sriov_numvfs
[root@hp-dl380g10-04 sriov]# echo 1 > /sys/bus/pci/devices/0000\:37\:00.1/sriov_numvfs
# Create representors
[root@hp-dl380g10-04 sriov]# ethtool -i ens2f2
driver: mlx5_core
version: 5.0-0
firmware-version: 16.26.1040 (MT_0000000080)
expansion-rom-version:
bus-info: 0000:37:00.2
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
[root@hp-dl380g10-04 sriov]# ethtool -i enp55s1f2
driver: mlx5_core
version: 5.0-0
firmware-version: 16.26.1040 (MT_0000000080)
expansion-rom-version:
bus-info: 0000:37:01.2
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
[root@hp-dl380g10-04 sriov]# echo 0000:37:00.2 > /sys/bus/pci/drivers/mlx5_core/unbind
[root@hp-dl380g10-04 sriov]# echo 0000:37:01.2 > /sys/bus/pci/drivers/mlx5_core/unbind
[root@hp-dl380g10-04 sriov]# devlink dev show
pci/0000:37:00.0
pci/0000:37:00.1
[root@hp-dl380g10-04 sriov]# devlink dev eswitch set pci/0000:37:00.0 mode switchdev
[root@hp-dl380g10-04 sriov]# devlink dev eswitch set pci/0000:37:00.1 mode switchdev
# Enable hw-tc-offload
[root@hp-dl380g10-04 sriov]# ethtool -K ens2f0 hw-tc-offload on
[root@hp-dl380g10-04 sriov]# ethtool -K ens2f1 hw-tc-offload on
[root@hp-dl380g10-04 sriov]# ethtool -K eth0 hw-tc-offload on
[root@hp-dl380g10-04 sriov]# ethtool -K eth1 hw-tc-offload on
[root@hp-dl380g10-04 sriov]# tc qdisc add dev ens2f0 ingress
[root@hp-dl380g10-04 sriov]# tc qdisc add dev ens2f1 ingress
[root@hp-dl380g10-04 sriov]# tc qdisc add dev eth0 ingress
[root@hp-dl380g10-04 sriov]# tc qdisc add dev eth1 ingress
# Create bonding, add PFs to bonding
[root@hp-dl380g10-04 sriov]# modprobe -v bonding mode=4 miimon=100
insmod /lib/modules/3.10.0-1062.el7.x86_64/kernel/drivers/net/bonding/bonding.ko.xz mode=4 miimon=100
[root@hp-dl380g10-04 sriov]# ip link set bond0 up
[root@hp-dl380g10-04 sriov]# ifenslave bond0 ens2f0 ens2f1
# Add bonding,representors to bridge
[root@hp-dl380g10-04 sriov]# ovs-vsctl add-br ovsbr0
[root@hp-dl380g10-04 sriov]# ip link set ovsbr0 up
[root@hp-dl380g10-04 sriov]# ovs-vsctl add-port ovsbr0 bond0
[root@hp-dl380g10-04 sriov]# ovs-vsctl add-port ovsbr0 eth0
[root@hp-dl380g10-04 sriov]# ovs-vsctl add-port ovsbr0 eth1
# Add VF to VM, Generate streams on VM to remote via a VF
[root@localhost ~]# ip addr
7: ens7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 22:00:00:00:00:01 brd ff:ff:ff:ff:ff:ff
inet 192.168.10.10/24 scope global ens7
valid_lft forever preferred_lft forever
inet 192.168.11.10/24 scope global ens7
valid_lft forever preferred_lft forever
[root@localhost ~]# netperf -H 192.168.10.1 -t udp_stream -l 30 &
[root@localhost ~]# netperf -H 192.168.11.2 -t udp_stream -l 30 &
# Check statistics on Host, I saw both PFs' statistics are increasing
[root@hp-dl380g10-04 sriov]# ethtool -S ens2f1|grep tx_vport
tx_vport_unicast_packets: 10833182
tx_vport_unicast_bytes: 16161210247
tx_vport_multicast_packets: 26554
tx_vport_multicast_bytes: 3289884
tx_vport_broadcast_packets: 50
tx_vport_broadcast_bytes: 5256
tx_vport_rdma_unicast_packets: 0
tx_vport_rdma_unicast_bytes: 0
tx_vport_rdma_multicast_packets: 0
tx_vport_rdma_multicast_bytes: 0
[root@hp-dl380g10-04 sriov]# ethtool -S ens2f0|grep tx_vport
tx_vport_unicast_packets: 7421777
tx_vport_unicast_bytes: 11063253951
tx_vport_multicast_packets: 50878
tx_vport_multicast_bytes: 5957976
tx_vport_broadcast_packets: 2344
tx_vport_broadcast_bytes: 193408
tx_vport_rdma_unicast_packets: 0
tx_vport_rdma_unicast_bytes: 0
tx_vport_rdma_multicast_packets: 0
tx_vport_rdma_multicast_bytes: 0
[root@hp-dl380g10-04 sriov]# ethtool -S ens2f1|grep tx_packets_phy
tx_packets_phy: 11975992
[root@hp-dl380g10-04 sriov]# ethtool -S ens2f0|grep tx_packets_phy
tx_packets_phy: 11259915