NVIDIA BF3网卡进行网络隔离及ib_traffic压测
提示:这里简述项目相关背景:同机台进行IB打流时候,可以进行网络隔离,设置每张网卡网口的单独网络环境,进行流量转发操作;
例如:BF3调试及工具安装跳过;主要介绍网络隔离IP设置
#如四张BF3单口400G网卡对联,进行网络IP隔离设置
`提示:这里描述项目中遇到的问题:单机台网卡互联默认无法进行流量转发;故借用防火墙规则进行网络IP隔离
例如:网卡信息如下
增加网络隔离:
sudo ip rule show
sudo ip rule add from all lookup local pref 100
sudo ip rule show
sudo ip rule del pref 0
cat /proc/sys/net/ipv4/conf/all/rp_filter
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter'
ibdev2netdev | grep enp28s0f0np0 #NIC1
cat /proc/sys/net/ipv4/conf/enp28s0f0np0/rp_filter
cat /proc/sys/net/ipv4/conf/enp28s0f0np0/accept_local
cat /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_filter
cat /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_ignore
cat /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_announce
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/rp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp28s0f0np0/accept_local'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_ignore'
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_announce'
sudo ifconfig enp28s0f0np0 192.168.103.1/24 up
sudo ip rule add iif enp28s0f0np0 lookup local pref 0
sudo ip rule add from 192.168.103.1 table 10 pref 10
sudo ip route add 192.168.103.0/24 dev enp28s0f0np0 src 192.168.103.1 table 10
sudo ip route add local 192.168.103.1 dev enp28s0f0np0 src 192.168.103.1 table 10
sudo ip rule add to 192.168.103.1 table 10 pref 10
ibdev2netdev | grep enp28s0f0np0
ibdev2netdev | grep enp58s0f0np0 #NIC2
cat /proc/sys/net/ipv4/conf/enp58s0f0np0/rp_filter
cat /proc/sys/net/ipv4/conf/enp58s0f0np0/accept_local
cat /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_filter
cat /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_ignore
cat /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_announce
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/rp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp58s0f0np0/accept_local'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_ignore'
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_announce'
sudo ifconfig enp58s0f0np0 192.168.103.2/24 up
sudo ip rule add iif enp58s0f0np0 lookup local pref 0
sudo ip rule add from 192.168.103.2 table 11 pref 10
sudo ip route add 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.2 table 11
sudo ip route add local 192.168.103.2 dev enp58s0f0np0 src 192.168.103.2 table 11
sudo ip rule add to 192.168.103.2 table 11 pref 10
ibdev2netdev | grep enp58s0f0np0
ibdev2netdev | grep enp158s0f0np0 #NIC3
cat /proc/sys/net/ipv4/conf/enp158s0f0np0/rp_filter
cat /proc/sys/net/ipv4/conf/enp158s0f0np0/accept_local
cat /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_filter
cat /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_ignore
cat /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_announce
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/rp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp158s0f0np0/accept_local'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_ignore'
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_announce'
sudo ifconfig enp158s0f0np0 192.168.103.3/24 up
sudo ip rule add iif enp158s0f0np0 lookup local pref 0
sudo ip rule add from 192.168.103.3 table 12 pref 10
sudo ip route add 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.3 table 12
sudo ip route add local 192.168.103.3 dev enp58s0f0np0 src 192.168.103.3 table 12
sudo ip rule add to 192.168.103.3 table 12 pref 10
ibdev2netdev | grep enp158s0f0np0
ibdev2netdev | grep enp186s0f0np0 #NIC4
cat /proc/sys/net/ipv4/conf/enp186s0f0np0/rp_filter
cat /proc/sys/net/ipv4/conf/enp186s0f0np0/accept_local
cat /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_filter
cat /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_ignore
cat /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_announce
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/rp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp186s0f0np0/accept_local'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_filter'
sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_ignore'
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_announce'
sudo ifconfig enp186s0f0np0 192.168.103.4/24 up
sudo ip rule add iif enp186s0f0np0 lookup local pref 0
sudo ip rule add from 192.168.103.4 table 13 pref 10
sudo ip route add 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.4 table 13
sudo ip route add local 192.168.103.4 dev enp58s0f0np0 src 192.168.103.4 table 13
sudo ip rule add to 192.168.103.4 table 13 pref 10
ibdev2netdev | grep enp58s0f0np0
sudo ip -s -s neigh flush all
sudo ip route flush cache
删除网络IP隔离
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp28s0f0np0/rp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/accept_local'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_ignore'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_announce'
sudo ip rule del to 192.168.103.1 table 10 pref 10 #NIC1
sudo ip route del local 192.168.103.1 dev enp28s0f0np0 src 192.168.103.1 table 10
sudo ip route del 192.168.103.0/24 dev enp28s0f0np0 src 192.168.103.1 table 10
sudo ip rule del from 192.168.103.1 table 10 pref 10
sudo ip rule del iif enp28s0f0np0 lookup local pref 0
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp58s0f0np0/rp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/accept_local'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_ignore'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_announce'
sudo ip rule del to 192.168.103.2 table 11 pref 10 #NIC2
sudo ip route del local 192.168.103.2 dev enp58s0f0np0 src 192.168.103.2 table 11
sudo ip route del 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.2 table 11
sudo ip rule del from 192.168.103.2 table 11 pref 10
sudo ip rule del iif enp58s0f0np0 lookup local pref 0
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp158s0f0np0/rp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/accept_local'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_ignore'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_announce'
sudo ip rule del to 192.168.103.3 table 12 pref 10 #NIC3
sudo ip route del local 192.168.103.3 dev enp158s0f0np0 src 192.168.103.3 table 12
sudo ip route del 192.168.103.0/24 dev enp158s0f0np0 src 192.168.103.3 table 12
sudo ip rule del from 192.168.103.3 table 12 pref 10
sudo ip rule del iif enp158s0f0np0 lookup local pref 0
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp186s0f0np0/rp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/accept_local'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_filter'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_ignore'
sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_announce'
sudo ip rule del to 192.168.103.4 table 13 pref 10 #NIC4
sudo ip route del local 192.168.103.4 dev enp186s0f0np0 src 192.168.103.4 table 13
sudo ip route del 192.168.103.0/24 dev enp186s0f0np0 src 192.168.103.4 table 13
sudo ip rule del from 192.168.103.4 table 13 pref 10
sudo ip rule del iif enp186s0f0np0 lookup local pref 0
sudo ip rule show
sudo ip rule add from all lookup local pref 0
sudo ip rule show
sudo ip rule del pref 100
sudo sysctl -w net.ipv4.conf.all.rp_filter=2
sudo ifconfig enp28s0f0np0 down
sudo ifconfig enp58s0f0np0 down
sudo ifconfig enp158s0f0np0 down
sudo ifconfig enp186s0f0np0 down
设置网络IP后,可以利用iperf或者IB_write_bw进行打流测试,针对test阶段的一些思路,如有问题,希望可以及时交流;
https://blog.youkuaiyun.com/liaomin416100569/article/details/126782055 #简单概述通过linux的虚拟网络隔离,来模拟熟悉交换机,路由器,网卡,网桥等物理设备间的关系和区别。
IB_write_bw命令示例:
ib_write_bw --ib-dev=mlx5_0 --ib-port=1 --port=10001 --connection=RC --qp=1 --run_infinitely &
ib_write_bw --ib-dev=mlx5_1 --ib-port=1 --port=10001 --connection=RC --qp=1 --report_gbits --run_infinitely --CPU-freq --post_list=1 192.168.103.1