【OVS2.5源码解读】datapath主流程分析

最新推荐文章于 2021-08-26 18:02:59 发布

造夢先森

最新推荐文章于 2021-08-26 18:02:59 发布

阅读量1.3k

点赞数

分类专栏： openvswitch 文章标签：源码 ovs2-5 datapath 云计算

本文链接：https://blog.youkuaiyun.com/qq_15437629/article/details/78695250

版权

openvswitch 专栏收录该内容

10 篇文章

订阅专栏

我们知道，ovs为了提高效率，数据包会先在内核层datapath进行流表项匹配处理，对于匹配失败，或者是匹配到表项的action为发向用户层时，才会去用户层继续查找匹配。对于在用户层匹配成功的数据包会按照表项action相应处理，并向内核层下发一条匹配到的表项，方便以后类似数据包直接在内核层完成匹配转发。

这里写图片描述

网络报文到达主机后，最终会到达协议栈的netif_receive_skb函数，该函数会通过设备对象的rx_handler函数把报文交给OVS处理。而该rx_handler函数其实就是OVS 定义的netdev_frame_hook函数，本篇内容就是从netdev_frame_hook函数开始，分析报文在datapath中的整个主处理过程。

1、netdev_frame_hook函数

该函数为OVS与内核桥接点，所以函数定义受内核定义影响

static rx_handler_result_t netdev_frame_hook(struct sk_buff **pskb)  
{  
    struct sk_buff *skb = *pskb;  

    if (unlikely(skb->pkt_type == PACKET_LOOPBACK))  
        return RX_HANDLER_PASS;  

    port_receive(skb);  
    return RX_HANDLER_CONSUMED;  
}  
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,36) || \  
      defined HAVE_RHEL_OVS_HOOK  
/* Called with rcu_read_lock and bottom-halves disabled. */  
static struct sk_buff *netdev_frame_hook(struct sk_buff *skb)  
{  
    if (unlikely(skb->pkt_type == PACKET_LOOPBACK))  
        return skb;  

    port_receive(skb);  
    return NULL;  
}  
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32)  
/* 
 * Used as br_handle_frame_hook.  (Cannot run bridge at the same time, even on 
 * different set of devices!) 
 */  
/* Called with rcu_read_lock and bottom-halves disabled. */  
static struct sk_buff *netdev_frame_hook(struct net_bridge_port *p,  
                     struct sk_buff *skb)  
{  
    port_receive(skb);  
    return NULL;  
}  
#else  
#error  
#endif

2、port_receive函数

#ifndef HAVE_METADATA_DST  
#define port_receive(skb)  netdev_port_receive(skb, NULL)  
#else  
#define port_receive(skb)  netdev_port_receive(skb, skb_tunnel_info(skb))   //报文中包含隧道信息，说明协议栈支持隧道报文了  
#endif

3、netdev_port_receive函数

/* Must be called with rcu_read_lock. */  
void netdev_port_receive(struct sk_buff *skb, struct ip_tunnel_info *tun_info)  
{  
    struct vport *vport;  

    vport = ovs_netdev_get_vport(skb->dev);    //通过netdev设备获得vport对象，是实现在datapath中转发的基础  
    if (unlikely(!vport))  
        goto error;  

    if (unlikely(skb_warn_if_lro(skb)))  
        goto error;  

    /* Make our own copy of the packet.  Otherwise we will mangle the 
     * packet for anyone who came before us (e.g. tcpdump via AF_PACKET). 
     */  
    skb = skb_share_check(skb, GFP_ATOMIC);  
    if (unlikely(!skb))  
        return;  

    skb_push(skb, ETH_HLEN);  
    ovs_skb_postpush_rcsum(skb, skb->data, ETH_HLEN);  
    ovs_vport_receive(vport, skb, tun_info);     //基于vport进行处理  
    return;  
error:  
    kfree_skb(skb);  
}

4、ovs_vport_receive函数

int ovs_vport_receive(struct vport *vport, struct sk_buff *skb,  
              const struct ip_tunnel_info *tun_info)  
{  
    struct sw_flow_key key;  
    int error;  

    OVS_CB(skb)->input_vport = vport;  
    OVS_CB(skb)->mru = 0;  
    if (unlikely(dev_net(skb->dev) != ovs_dp_get_net(vport->dp))) {  
        u32 mark;  

        mark = skb->mark;  
        skb_scrub_packet(skb, true);  
        skb->mark = mark;  
        tun_info = NULL;  
    }  

    ovs_skb_init_inner_protocol(skb);  
    skb_clear_ovs_gso_cb(skb);  
    /* Extract flow from 'skb' into 'key'. */  
    error = ovs_flow_key_extract(tun_info, skb, &key);  //根据报文生成key  
    if (unlikely(error)) {  
        kfree_skb(skb);  
        return error;  
    }  
    ovs_dp_process_packet(skb, &key);   //报文处理  
    return 0;  
}

5、ovs_dp_process_packet函数

void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)  
{  
    const struct vport *p = OVS_CB(skb)->input_vport;  
    struct datapath *dp = p->dp;  
    struct sw_flow *flow;  
    struct sw_flow_actions *sf_acts;  
    struct dp_stats_percpu *stats;  
    u64 *stats_counter;  
    u32 n_mask_hit;  

    stats = this_cpu_ptr(dp->stats_percpu);  

    /* Look up flow. */  
    flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb), //查询转发表  
                     &n_mask_hit);  
    if (unlikely(!flow)) {      //如果没有查到流表，则上送的upcall线程处理  
        struct dp_upcall_info upcall;  
        int error;  

        memset(&upcall, 0, sizeof(upcall));  
        upcall.cmd = OVS_PACKET_CMD_MISS;  
        upcall.portid = ovs_vport_find_upcall_portid(p, skb);  
        upcall.mru = OVS_CB(skb)->mru;  
        error = ovs_dp_upcall(dp, skb, key, &upcall);  
        if (unlikely(error))  
            kfree_skb(skb);  
        else  
            consume_skb(skb);  
        stats_counter = &stats->n_missed;  
        goto out;  
    }  

    ovs_flow_stats_update(flow, key->tp.flags, skb);  
    sf_acts = rcu_dereference(flow->sf_acts);        //获取action  
    ovs_execute_actions(dp, skb, sf_acts, key);     //对报文执行action  

    stats_counter = &stats->n_hit;  

out:  
    /* Update datapath statistics. */  
    u64_stats_update_begin(&stats->syncp);  
    (*stats_counter)++;  
    stats->n_mask_hit += n_mask_hit;  
    u64_stats_update_end(&stats->syncp);  
}

背景：

报文匹配的流程可以说是对一个switch效率影响最大的地方，现在已经有很多厂家（包括一些研究院）说可以做到比OVS效率高十倍的流表匹配。
当然这是有可能的，毕竟都是纯软件，非常容易创新。而ovs也没有做多级流表，跟of标准上不一定比得过别人。我们接下来就分析一下OVS的匹配流程。

TIPS：

整体的匹配逻辑很简单，都是OpenFlow规定好的。
报文通过dp时先查找精确匹配表（facet），如果找不到，那么进行upcall，上送到用户态。
在用户态会查找模糊匹配表（table），这个地方可以说是较没有效率的地方。下面我们看一看具体的代码。

调用流程（内核）：
ovs_vport_receive->ovs_dp_process_received_packet->ovs_flow_tbl_lookup->ovs_dp_upcall->queue_userspace_packet

调用流程（用户态）：
handle_miss_upcalls->handle_flow_miss->rule_dpif_lookup->rule_dpif_lookup__->classifier_lookup->find_match

1.在handle_miss_upcalls里解析了报文，生成了报文的精确匹配项，如果我们把它遮盖掉（mask）一部分，那么它就是一个模糊匹配的flow了。实际上ovs也就是这么做的。
2、注意到每个flow table拥有一个cls，一个cls拥有多个cls_table，cls_table中拥有mask完全一致的flow。
3.而在find_match中，ovs将精确匹配项对应cls_table的mask位置0，然后通过hash查找是否有匹配的flow。

6、ovs_execute_actions函数

int ovs_execute_actions(struct datapath *dp, struct sk_buff *skb,  
            const struct sw_flow_actions *acts,  
            struct sw_flow_key *key)  
{  
    int level = this_cpu_read(exec_actions_level);  
    int err;  

    if (unlikely(level >= EXEC_ACTIONS_LEVEL_LIMIT)) {  
        if (net_ratelimit())  
            pr_warn("%s: packet loop detected, dropping.\n",  
                ovs_dp_name(dp));  

        kfree_skb(skb);  
        return -ELOOP;  
    }  

    this_cpu_inc(exec_actions_level);  
    err = do_execute_actions(dp, skb, key,  
                 acts->actions, acts->actions_len);    //执行action  

    if (!level)  
        process_deferred_actions(dp);  

    this_cpu_dec(exec_actions_level);  

    /* This return status currently does not reflect the errors 
     * encounted during deferred actions execution. Probably needs to 
     * be fixed in the future. 
     */  
    return err;  
}

7、do_execute_actions函数

static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,  
                  struct sw_flow_key *key,  
                  const struct nlattr *attr, int len)  
{  
    /* Every output action needs a separate clone of 'skb', but the common 
     * case is just a single output action, so that doing a clone and 
     * then freeing the original skbuff is wasteful.  So the following code 
     * is slightly obscure just to avoid that. 
     */  
    int prev_port = -1;  
    const struct nlattr *a;  
    int rem;  

    for (a = attr, rem = len; rem > 0;  
         a = nla_next(a, &rem)) {  
        int err = 0;  

        if (unlikely(prev_port != -1)) {     //从某个端口发出  
            struct sk_buff *out_skb = skb_clone(skb, GFP_ATOMIC);    //克隆报文  

            if (out_skb)  
                do_output(dp, out_skb, prev_port, key);    //输出报文，为简化起见，本篇以output为例，其他action在后续分析  

            prev_port = -1;  
        }  

        switch (nla_type(a)) {  
        case OVS_ACTION_ATTR_OUTPUT:  
            prev_port = nla_get_u32(a);  
            break;  

        case OVS_ACTION_ATTR_USERSPACE:  
            output_userspace(dp, skb, key, a, attr, len);   
            break;  

        case OVS_ACTION_ATTR_HASH:  
            execute_hash(skb, key, a);  
            break;  

        case OVS_ACTION_ATTR_PUSH_MPLS:  
            err = push_mpls(skb, key, nla_data(a));  
            break;  

        case OVS_ACTION_ATTR_POP_MPLS:  
            err = pop_mpls(skb, key, nla_get_be16(a));  
            break;  

        case OVS_ACTION_ATTR_PUSH_VLAN:  
            err = push_vlan(skb, key, nla_data(a));  
            break;  

        case OVS_ACTION_ATTR_POP_VLAN:  
            err = pop_vlan(skb, key);  
            break;  

        case OVS_ACTION_ATTR_RECIRC:  
            err = execute_recirc(dp, skb, key, a, rem);  
            if (nla_is_last(a, rem)) {  
                /* If this is the last action, the skb has 
                 * been consumed or freed. 
                 * Return immediately. 
                 */  
                return err;  
            }  
            break;  

        case OVS_ACTION_ATTR_SET:  
            err = execute_set_action(skb, key, nla_data(a));  
            break;  

        case OVS_ACTION_ATTR_SET_MASKED:  
        case OVS_ACTION_ATTR_SET_TO_MASKED:  
            err = execute_masked_set_action(skb, key, nla_data(a));  
            break;  

        case OVS_ACTION_ATTR_SAMPLE:  
            err = sample(dp, skb, key, a, attr, len);  
            break;  

        case OVS_ACTION_ATTR_CT:  
            if (!is_flow_key_valid(key)) {  
                err = ovs_flow_key_update(skb, key);  
                if (err)  
                    return err;  
            }  

            err = ovs_ct_execute(ovs_dp_get_net(dp), skb, key,  
                         nla_data(a));  

            /* Hide stolen IP fragments from user space. */  
            if (err)  
                return err == -EINPROGRESS ? 0 : err;  
            break;  
        }  

        if (unlikely(err)) {  
            kfree_skb(skb);  
            return err;  
        }  
    }  

    if (prev_port != -1)  
        do_output(dp, skb, prev_port, key);  
    else  
        consume_skb(skb);  

    return 0;  
}

8、do_output函数

static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port,  
              struct sw_flow_key *key)  
{  
    struct vport *vport = ovs_vport_rcu(dp, out_port);           //获取出端口的vport对象  

    if (likely(vport)) {  
        u16 mru = OVS_CB(skb)->mru;  

        if (likely(!mru || (skb->len <= mru + ETH_HLEN))) {     
            ovs_vport_send(vport, skb);                   //发送报文  
        } else if (mru <= vport->dev->mtu) {  
            __be16 ethertype = key->eth.type;  

            if (!is_flow_key_valid(key)) {  
                if (eth_p_mpls(skb->protocol))  
                    ethertype = ovs_skb_get_inner_protocol(skb);  
                else  
                    ethertype = vlan_get_protocol(skb);  
            }  

            ovs_fragment(vport, skb, mru, ethertype);  
        } else {  
            OVS_NLERR(true, "Cannot fragment IP frames");  
            kfree_skb(skb);  
        }  
    } else {  
        kfree_skb(skb);  
    }  
}

9、ovs_vport_send函数

void ovs_vport_send(struct vport *vport, struct sk_buff *skb)  
{  
    int mtu = vport->dev->mtu;  

    if (unlikely(packet_length(skb) > mtu && !skb_is_gso(skb))) {  
        net_warn_ratelimited("%s: dropped over-mtu packet: %d > %d\n",  
                     vport->dev->name,  
                     packet_length(skb), mtu);  
        vport->dev->stats.tx_errors++;  
        goto drop;  
    }  

    skb->dev = vport->dev;      //skb的dev设备，设置成vport关联的netdev设备  
    vport->ops->send(skb);      //调用vport对应vport_ops的send函数，如果是ovs_netdev_vport_ops，则调用内核的dev_queue_xmit函数，其他种类的vport后续分析  
    return;  

drop:  
    kfree_skb(skb);  
}