nvidia dpu

原创已于 2025-10-29 11:32:42 修改 · 118 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#linux #运维 #服务器

于 2025-10-29 11:07:41 首次发布

host:

10: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 94:6d:ae:4f:3c:90 brd ff:ff:ff:ff:ff:ff
    altname enp8s0f0np0
    inet 1.1.1.1/24 brd 1.1.1.255 scope global eth2
       valid_lft forever preferred_lft forever
    inet6 fe80::966d:aeff:fe4f:3c90/64 scope link
       valid_lft forever preferred_lft forever

ping对端的host，生成下面的规则：

=== chain: 0, prio: 2, level: 0 ===
prio ffff0007bc1fbd20, ref: 2
next_fdb: ffff00008725b000, miss_group: ffff0000946a6710, miss_rule: mlx5_flow_handle ffff000080adc2a0

flow table name:
flow table id: 0x11 table_level: 1,         type: 4 (FS_FT_FDB: 4, FS_FT_NIC_RX: 0, FS_FT_NIC_TX, 1,         max_fte: 2147483647, 7fffffff), refcount: 3
mlx5_flow_table ffff00012a21f600
mlx5_flow_group ffff0000946a0610, id: 0, match_criteria_enable: 0x9, refcount: 2, max_ftes: 16909320, start_index: 0x0
num_fs_actions: 2
fs_fte ffff000085e45dd8 flow_source: 2 (0: any, 1: uplink: 2: local), refcount: 3
       0:  s: 94:6d:ae:4f:3c:90 d: 94:6d:ae:4f:3c:6a ipv: 4  reg_c0: 20000 action:    c
                mlx5_flow_rule ffff0000852a8600, refcount: 1
                mlx5_flow_rule ffff0000852a8400, refcount: 1
                        dest: vport: ffff, vhca_id: 3, flags: 1 (MLX5_FLOW_DEST_VPORT_VHCA_ID: 1, MLX5_FLOW_DEST_VPORT_REFORMAT_ID: 2)
num_fs_actions: 2
fs_fte ffff0000e0e8f1c0 flow_source: 1 (0: any, 1: uplink: 2: local), refcount: 3
       1:  s: 94:6d:ae:4f:3c:6a d: 94:6d:ae:4f:3c:90 ipv: 4  reg_c0: 10000 action:    c
                mlx5_flow_rule ffff0000e451ef00, refcount: 1
                mlx5_flow_rule ffff0000e451f900, refcount: 1
                        dest: vport: 0, vhca_id: 3, flags: 1 (MLX5_FLOW_DEST_VPORT_VHCA_ID: 1, MLX5_FLOW_DEST_VPORT_REFORMAT_ID: 2)

mlx5_vport如下：

mlx5_vport ffff0007c4b5a000 vport:    0, metadata:    2, index:    0 mac: 94:6d:ae:4f:3c:90 enabled: 1 vlan: 0
mlx5_vport ffff0007c4b64000 vport:    1, metadata:    3, index:    1 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b62000 vport:    2, metadata:    4, index:    2 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b66000 vport:    3, metadata:    5, index:    3 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b60000 vport:    4, metadata:    6, index:    4 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b70000 vport: 8000, metadata:    7, index:    5 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b72000 vport: 8001, metadata:    8, index:    6 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b76000 vport: 8002, metadata:    9, index:    7 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b74000 vport: 8003, metadata:    a, index:    8 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b7a000 vport: fffe, metadata:    b, index:    9 mac: 00:00:00:00:00:00 enabled: 0 vlan: 0
mlx5_vport ffff0007c4b86000 vport: ffff, metadata:    1, index:    a mac: 00:00:00:00:00:00 enabled: 0 vlan: 0

从本地pf mac（94:6d:ae:4f:3c:90）到对端pf的mac（94:6d:ae:4f:3c:6a），match reg_c0（0x2），对应vport 0，就是从本地host到pf0hpf，然后再从uplink（0xffff）出去。

========================== devlink.dev.kobj.name: 0000:03:00.0 index: 0 =========================
devlink ffff0007c48fe000
mlx5_core_dev ffff0007c48fe1a0
mlx5_priv ffff0007c48fe990
mlx5_eswitch ffff0007bb7f0000
mlx5_core_dev.coredev_type:
(enum mlx5_coredev_type)MLX5_COREDEV_PF
        === devlink_port start ===
        devlink_port ffff000084218600, port index: 0x30000, 196608
        port.type_eth.ifname: pf0hpf
        94:6d:ae:03:00:4f:3c:90:
        === devlink_port end ===
========================== devlink.dev.kobj.name: mlx5_core.eth.0 index: 1 =========================
devlink ffff0007c4289000
mlx5e_dev.priv.netdev.name: p0
devlink_port: ffff0007c42891a8
        === devlink_port start ===
        devlink_port ffff0007c42891a8, port index: 0x3ffff, 262143
        port.type_eth.ifname: p0
        94:6d:ae:03:00:4f:3c:90:
        === devlink_port end ===

crash> mlx5_devlink_port.vport ffff000084218600
  vport = 0xffff0007c4b5a000

pf0pf对应的devlink_port在mlx5_devlink_port数据结构里面，对应的vport->vport是0。

p0对应的devlink_port在mlx5e_dev里面，不对应mlx5_vport。

crash> mlx5e_dev ffff0007c42891a0  # ffff0007c42891a8 - 8
struct mlx5e_dev {
  priv = 0xffff0007ad978a80,
  dl_port = {

0xfffe是管理口，并没有相应的netdev。

[root@c-237-169-20-pb1-00-0-bf3 ~]# devlink port
pci/0000:03:00.0/196608: type eth netdev pf0hpf flavour pcipf controller 1 pfnum 0 external true splittable false
  function:
    hw_addr 94:6d:ae:4f:3c:90
auxiliary/mlx5_core.eth.0/262143: type eth netdev p0 flavour physical port 0 splittable false
pci/0000:03:00.1/262144: type eth netdev pf1hpf flavour pcipf controller 1 pfnum 1 external true splittable false
  function:
    hw_addr 94:6d:ae:4f:3c:91
auxiliary/mlx5_core.eth.1/327679: type eth netdev p1 flavour physical port 1 splittable false

196608=0x30000

262143=0x3ffff

262144=0x40000

327679=0x4ffff

如果是nic模式的话，管理口是0，miss rule会导到管理口：

esw->manager_vport: 0x0

fs_fte ff3aaab28cfe42e0 flow_source: 0 (0: any, 1: uplink: 2: local), refcount: 2
    124b:  d: 01:00:00:00:00:00 action:    4
                mlx5_flow_rule ff3aaab223a96600, refcount: 1
                        dest: vport: 0, vhca_id: 0, flags: 0 (MLX5_FLOW_DEST_VPORT_VHCA_ID: 1, MLX5_FLOW_DEST_VPORT_REFORMAT_ID: 2)

如果是dpu模式，vport 0被pf0hpf占用，用于和host pf通信，所以管理口变成0xfffe：

esw->manager_vport: 0xfffe

fs_fte ffff0007ad91db00 flow_source: 0 (0: any, 1: uplink: 2: local), refcount: 2
    1352:  d: 00:00:00:00:00:00 action:    4
                mlx5_flow_rule ffff00008509e900, refcount: 1
                        dest: vport: fffe, vhca_id: 0, flags: 0 (MLX5_FLOW_DEST_VPORT_VHCA_ID: 1, MLX5_FLOW_DEST_VPORT_REFORMAT_ID: 2)