[ovs-dev] [PATCH v4 3/4] netdev: Split up netdev offloading to separate module.

Roi Dayan roid at mellanox.com
Tue May 21 16:48:38 UTC 2019



On 15/05/2019 18:36, Ilya Maximets wrote:
> New module 'netdev-offload' created to manage different flow API
> implementations. All the generic and provider independent code moved
> there from the 'netdev' module.
> 
> Flow API providers further encapsulated.
> 
> The only function that was changed is 'netdev_any_oor'.
> Now it uses offloading related hmap instead of common 'netdev_shash'.
> 
> Signed-off-by: Ilya Maximets <i.maximets at samsung.com>
> Acked-by: Ben Pfaff <blp at ovn.org>
> ---
>  lib/automake.mk               |   3 +-
>  lib/dpdk.c                    |   4 +-
>  lib/dpif-netdev.c             |   1 +
>  lib/dpif-netlink.c            |   1 +
>  lib/netdev-dpdk.c             |   1 -
>  lib/netdev-offload-provider.h |   6 +
>  lib/netdev-offload.c          | 658 ++++++++++++++++++++++++++++++++++
>  lib/netdev-offload.h          | 126 +++++++
>  lib/netdev-provider.h         |  21 +-
>  lib/netdev-rte-offloads.c     |   8 +-
>  lib/netdev-rte-offloads.h     |  22 --
>  lib/netdev.c                  | 594 +-----------------------------
>  lib/netdev.h                  |  57 ---
>  vswitchd/bridge.c             |   1 +
>  14 files changed, 801 insertions(+), 702 deletions(-)
>  create mode 100644 lib/netdev-offload.c
>  create mode 100644 lib/netdev-offload.h
>  delete mode 100644 lib/netdev-rte-offloads.h
> 
> diff --git a/lib/automake.mk b/lib/automake.mk
> index c70fda3f8..b2d90da4e 100644
> --- a/lib/automake.mk
> +++ b/lib/automake.mk
> @@ -137,9 +137,10 @@ lib_libopenvswitch_la_SOURCES = \
>  	lib/namemap.c \
>  	lib/netdev-dpdk.h \
>  	lib/netdev-dummy.c \
> +	lib/netdev-offload.c \
> +	lib/netdev-offload.h \
>  	lib/netdev-offload-provider.h \
>  	lib/netdev-provider.h \
> -	lib/netdev-rte-offloads.h \
>  	lib/netdev-vport.c \
>  	lib/netdev-vport.h \
>  	lib/netdev-vport-private.h \
> diff --git a/lib/dpdk.c b/lib/dpdk.c
> index 6c6298635..a253bef71 100644
> --- a/lib/dpdk.c
> +++ b/lib/dpdk.c
> @@ -34,7 +34,7 @@
>  #include "dirs.h"
>  #include "fatal-signal.h"
>  #include "netdev-dpdk.h"
> -#include "netdev-rte-offloads.h"
> +#include "netdev-offload-provider.h"
>  #include "openvswitch/dynamic-string.h"
>  #include "openvswitch/vlog.h"
>  #include "ovs-numa.h"
> @@ -443,7 +443,7 @@ dpdk_init__(const struct smap *ovs_other_config)
>  
>      /* Finally, register the dpdk classes */
>      netdev_dpdk_register();
> -    netdev_dpdk_flow_api_register();
> +    netdev_register_flow_api_provider(&netdev_dpdk_offloads);
>      return true;
>  }
>  
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index 5a6f2abac..4ad04fdd3 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -50,6 +50,7 @@
>  #include "ipf.h"
>  #include "latch.h"
>  #include "netdev.h"
> +#include "netdev-offload.h"
>  #include "netdev-provider.h"
>  #include "netdev-vport.h"
>  #include "netlink.h"
> diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c
> index c554666ac..ba80a0079 100644
> --- a/lib/dpif-netlink.c
> +++ b/lib/dpif-netlink.c
> @@ -38,6 +38,7 @@
>  #include "fat-rwlock.h"
>  #include "flow.h"
>  #include "netdev-linux.h"
> +#include "netdev-offload.h"
>  #include "netdev-provider.h"
>  #include "netdev-vport.h"
>  #include "netdev.h"
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index c06c9ef81..6ea618951 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -47,7 +47,6 @@
>  #include "dpif-netdev.h"
>  #include "fatal-signal.h"
>  #include "netdev-provider.h"
> -#include "netdev-rte-offloads.h"
>  #include "netdev-vport.h"
>  #include "odp-util.h"
>  #include "openvswitch/dynamic-string.h"
> diff --git a/lib/netdev-offload-provider.h b/lib/netdev-offload-provider.h
> index ceeaa50b0..2634efca0 100644
> --- a/lib/netdev-offload-provider.h
> +++ b/lib/netdev-offload-provider.h
> @@ -19,6 +19,8 @@
>  #define NETDEV_FLOW_API_PROVIDER_H 1
>  
>  #include "flow.h"
> +#include "netdev-offload.h"
> +#include "openvswitch/netdev.h"
>  #include "openvswitch/types.h"
>  #include "packets.h"
>  
> @@ -92,6 +94,10 @@ int netdev_unregister_flow_api_provider(const char *type);
>  extern const struct netdev_flow_api netdev_tc_offloads;
>  #endif
>  
> +#ifdef DPDK_NETDEV
> +extern const struct netdev_flow_api netdev_dpdk_offloads;
> +#endif
> +
>  #ifdef  __cplusplus
>  }
>  #endif
> diff --git a/lib/netdev-offload.c b/lib/netdev-offload.c
> new file mode 100644
> index 000000000..e4279f6b9
> --- /dev/null
> +++ b/lib/netdev-offload.c
> @@ -0,0 +1,658 @@
> +/*
> + * Copyright (c) 2008 - 2014, 2016, 2017 Nicira, Inc.
> + * Copyright (c) 2019 Samsung Electronics Co.,Ltd.
> + *
> + * Licensed under the Apache License, Version 2.0 (the "License");
> + * you may not use this file except in compliance with the License.
> + * You may obtain a copy of the License at:
> + *
> + *     https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.apache.org%2Flicenses%2FLICENSE-2.0&amp;data=02%7C01%7Croid%40mellanox.com%7Caf61e39fa12b40deabd208d6d94b12bf%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636935313803067807&amp;sdata=xV0mnyhiWaV2sQ0an0FnlHJlNkfqPfTtA2J04vnQXI4%3D&amp;reserved=0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +#include <config.h>
> +#include "netdev-offload.h"
> +
> +#include <errno.h>
> +#include <inttypes.h>
> +#include <sys/types.h>
> +#include <netinet/in.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <unistd.h>
> +
> +#include "cmap.h"
> +#include "coverage.h"
> +#include "dpif.h"
> +#include "dp-packet.h"
> +#include "openvswitch/dynamic-string.h"
> +#include "fatal-signal.h"
> +#include "hash.h"
> +#include "openvswitch/list.h"
> +#include "netdev-offload-provider.h"
> +#include "netdev-provider.h"
> +#include "netdev-vport.h"
> +#include "odp-netlink.h"
> +#include "openflow/openflow.h"
> +#include "packets.h"
> +#include "openvswitch/ofp-print.h"
> +#include "openvswitch/poll-loop.h"
> +#include "seq.h"
> +#include "openvswitch/shash.h"
> +#include "smap.h"
> +#include "socket-util.h"
> +#include "sset.h"
> +#include "svec.h"
> +#include "openvswitch/vlog.h"
> +#include "flow.h"
> +#include "util.h"
> +#ifdef __linux__
> +#include "tc.h"
> +#endif
> +
> +VLOG_DEFINE_THIS_MODULE(netdev_offload);
> +
> +
> +static bool netdev_flow_api_enabled = false;
> +
> +/* Protects 'netdev_flow_apis'.  */
> +static struct ovs_mutex netdev_flow_api_provider_mutex = OVS_MUTEX_INITIALIZER;
> +
> +/* Contains 'struct netdev_registered_flow_api's. */
> +static struct cmap netdev_flow_apis = CMAP_INITIALIZER;
> +
> +struct netdev_registered_flow_api {
> +    struct cmap_node cmap_node; /* In 'netdev_flow_apis', by flow_api->type. */
> +    const struct netdev_flow_api *flow_api;
> +
> +    /* Number of references: one for the flow_api itself and one for every
> +     * instance of the netdev that uses it. */
> +    struct ovs_refcount refcnt;
> +};
> +
> +static struct netdev_registered_flow_api *
> +netdev_lookup_flow_api(const char *type)
> +{
> +    struct netdev_registered_flow_api *rfa;
> +    CMAP_FOR_EACH_WITH_HASH (rfa, cmap_node, hash_string(type, 0),
> +                             &netdev_flow_apis) {
> +        if (!strcmp(type, rfa->flow_api->type)) {
> +            return rfa;
> +        }
> +    }
> +    return NULL;
> +}
> +
> +/* Registers a new netdev flow api provider. */
> +int
> +netdev_register_flow_api_provider(const struct netdev_flow_api *new_flow_api)
> +    OVS_EXCLUDED(netdev_flow_api_provider_mutex)
> +{
> +    int error = 0;
> +
> +    if (!new_flow_api->init_flow_api) {
> +        VLOG_WARN("attempted to register invalid flow api provider: %s",
> +                   new_flow_api->type);
> +        error = EINVAL;
> +    }
> +
> +    ovs_mutex_lock(&netdev_flow_api_provider_mutex);
> +    if (netdev_lookup_flow_api(new_flow_api->type)) {
> +        VLOG_WARN("attempted to register duplicate flow api provider: %s",
> +                   new_flow_api->type);
> +        error = EEXIST;
> +    } else {
> +        struct netdev_registered_flow_api *rfa;
> +
> +        rfa = xmalloc(sizeof *rfa);
> +        cmap_insert(&netdev_flow_apis, &rfa->cmap_node,
> +                    hash_string(new_flow_api->type, 0));
> +        rfa->flow_api = new_flow_api;
> +        ovs_refcount_init(&rfa->refcnt);
> +        VLOG_DBG("netdev: flow API '%s' registered.", new_flow_api->type);
> +    }
> +    ovs_mutex_unlock(&netdev_flow_api_provider_mutex);
> +
> +    return error;
> +}
> +
> +/* Unregisters a netdev flow api provider.  'type' must have been previously
> + * registered and not currently be in use by any netdevs.  After unregistration
> + * netdev flow api of that type cannot be used for netdevs.  (However, the
> + * provider may still be accessible from other threads until the next RCU grace
> + * period, so the caller must not free or re-register the same netdev_flow_api
> + * until that has passed.) */
> +int
> +netdev_unregister_flow_api_provider(const char *type)
> +    OVS_EXCLUDED(netdev_flow_api_provider_mutex)
> +{
> +    struct netdev_registered_flow_api *rfa;
> +    int error;
> +
> +    ovs_mutex_lock(&netdev_flow_api_provider_mutex);
> +    rfa = netdev_lookup_flow_api(type);
> +    if (!rfa) {
> +        VLOG_WARN("attempted to unregister a flow api provider that is not "
> +                  "registered: %s", type);
> +        error = EAFNOSUPPORT;
> +    } else if (ovs_refcount_unref(&rfa->refcnt) != 1) {
> +        ovs_refcount_ref(&rfa->refcnt);
> +        VLOG_WARN("attempted to unregister in use flow api provider: %s",
> +                  type);
> +        error = EBUSY;
> +    } else  {
> +        cmap_remove(&netdev_flow_apis, &rfa->cmap_node,
> +                    hash_string(rfa->flow_api->type, 0));
> +        ovsrcu_postpone(free, rfa);
> +        error = 0;
> +    }
> +    ovs_mutex_unlock(&netdev_flow_api_provider_mutex);
> +
> +    return error;
> +}
> +
> +static int
> +netdev_assign_flow_api(struct netdev *netdev)
> +{
> +    struct netdev_registered_flow_api *rfa;
> +
> +    CMAP_FOR_EACH (rfa, cmap_node, &netdev_flow_apis) {
> +        if (!rfa->flow_api->init_flow_api(netdev)) {
> +            ovs_refcount_ref(&rfa->refcnt);
> +            ovsrcu_set(&netdev->flow_api, rfa->flow_api);
> +            VLOG_INFO("%s: Assigned flow API '%s'.",
> +                      netdev_get_name(netdev), rfa->flow_api->type);
> +            return 0;
> +        }
> +        VLOG_DBG("%s: flow API '%s' is not suitable.",
> +                 netdev_get_name(netdev), rfa->flow_api->type);
> +    }
> +    VLOG_INFO("%s: No suitable flow API found.", netdev_get_name(netdev));
> +
> +    return -1;
> +}
> +
> +int
> +netdev_flow_flush(struct netdev *netdev)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_flush)
> +           ? flow_api->flow_flush(netdev)
> +           : EOPNOTSUPP;
> +}
> +
> +int
> +netdev_flow_dump_create(struct netdev *netdev, struct netdev_flow_dump **dump)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_dump_create)
> +           ? flow_api->flow_dump_create(netdev, dump)
> +           : EOPNOTSUPP;
> +}
> +
> +int
> +netdev_flow_dump_destroy(struct netdev_flow_dump *dump)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &dump->netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_dump_destroy)
> +           ? flow_api->flow_dump_destroy(dump)
> +           : EOPNOTSUPP;
> +}
> +
> +bool
> +netdev_flow_dump_next(struct netdev_flow_dump *dump, struct match *match,
> +                      struct nlattr **actions, struct dpif_flow_stats *stats,
> +                      struct dpif_flow_attrs *attrs, ovs_u128 *ufid,
> +                      struct ofpbuf *rbuffer, struct ofpbuf *wbuffer)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &dump->netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_dump_next)
> +           ? flow_api->flow_dump_next(dump, match, actions, stats, attrs,
> +                                      ufid, rbuffer, wbuffer)
> +           : false;
> +}
> +
> +int
> +netdev_flow_put(struct netdev *netdev, struct match *match,
> +                struct nlattr *actions, size_t act_len,
> +                const ovs_u128 *ufid, struct offload_info *info,
> +                struct dpif_flow_stats *stats)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_put)
> +           ? flow_api->flow_put(netdev, match, actions, act_len, ufid,
> +                                info, stats)
> +           : EOPNOTSUPP;
> +}
> +
> +int
> +netdev_flow_get(struct netdev *netdev, struct match *match,
> +                struct nlattr **actions, const ovs_u128 *ufid,
> +                struct dpif_flow_stats *stats,
> +                struct dpif_flow_attrs *attrs, struct ofpbuf *buf)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_get)
> +           ? flow_api->flow_get(netdev, match, actions, ufid,
> +                                stats, attrs, buf)
> +           : EOPNOTSUPP;
> +}
> +
> +int
> +netdev_flow_del(struct netdev *netdev, const ovs_u128 *ufid,
> +                struct dpif_flow_stats *stats)
> +{
> +    const struct netdev_flow_api *flow_api =
> +        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> +
> +    return (flow_api && flow_api->flow_del)
> +           ? flow_api->flow_del(netdev, ufid, stats)
> +           : EOPNOTSUPP;
> +}
> +
> +int
> +netdev_init_flow_api(struct netdev *netdev)
> +{
> +    if (!netdev_is_flow_api_enabled()) {
> +        return EOPNOTSUPP;
> +    }
> +
> +    if (ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api)) {
> +        return 0;
> +    }
> +
> +    if (netdev_assign_flow_api(netdev)) {
> +        return EOPNOTSUPP;
> +    }
> +
> +    return 0;
> +}
> +
> +void
> +netdev_uninit_flow_api(struct netdev *netdev)
> +{
> +    struct netdev_registered_flow_api *rfa;
> +    const struct netdev_flow_api *flow_api =
> +            ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> +
> +    if (!flow_api) {
> +        return;
> +    }
> +
> +    ovsrcu_set(&netdev->flow_api, NULL);
> +    rfa = netdev_lookup_flow_api(flow_api->type);
> +    ovs_refcount_unref(&rfa->refcnt);
> +}
> +
> +uint32_t
> +netdev_get_block_id(struct netdev *netdev)
> +{
> +    const struct netdev_class *class = netdev->netdev_class;
> +
> +    return (class->get_block_id
> +            ? class->get_block_id(netdev)
> +            : 0);
> +}
> +
> +/*
> + * Get the value of the hw info parameter specified by type.
> + * Returns the value on success (>= 0). Returns -1 on failure.
> + */
> +int
> +netdev_get_hw_info(struct netdev *netdev, int type)
> +{
> +    int val = -1;
> +
> +    switch (type) {
> +    case HW_INFO_TYPE_OOR:
> +        val = netdev->hw_info.oor;
> +        break;
> +    case HW_INFO_TYPE_PEND_COUNT:
> +        val = netdev->hw_info.pending_count;
> +        break;
> +    case HW_INFO_TYPE_OFFL_COUNT:
> +        val = netdev->hw_info.offload_count;
> +        break;
> +    default:
> +        break;
> +    }
> +
> +    return val;
> +}
> +
> +/*
> + * Set the value of the hw info parameter specified by type.
> + */
> +void
> +netdev_set_hw_info(struct netdev *netdev, int type, int val)
> +{
> +    switch (type) {
> +    case HW_INFO_TYPE_OOR:
> +        if (val == 0) {
> +            VLOG_DBG("Offload rebalance: netdev: %s is not OOR", netdev->name);
> +        }
> +        netdev->hw_info.oor = val;
> +        break;
> +    case HW_INFO_TYPE_PEND_COUNT:
> +        netdev->hw_info.pending_count = val;
> +        break;
> +    case HW_INFO_TYPE_OFFL_COUNT:
> +        netdev->hw_info.offload_count = val;
> +        break;
> +    default:
> +        break;
> +    }
> +}
> +
> +/* Protects below port hashmaps. */
> +static struct ovs_mutex netdev_hmap_mutex = OVS_MUTEX_INITIALIZER;
> +
> +static struct hmap port_to_netdev OVS_GUARDED_BY(netdev_hmap_mutex)
> +    = HMAP_INITIALIZER(&port_to_netdev);
> +static struct hmap ifindex_to_port OVS_GUARDED_BY(netdev_hmap_mutex)
> +    = HMAP_INITIALIZER(&ifindex_to_port);
> +
> +struct port_to_netdev_data {
> +    struct hmap_node portno_node; /* By (dpif_class, dpif_port.port_no). */
> +    struct hmap_node ifindex_node; /* By (dpif_class, ifindex). */
> +    struct netdev *netdev;
> +    struct dpif_port dpif_port;
> +    const struct dpif_class *dpif_class;
> +    int ifindex;
> +};
> +
> +/*
> + * Find if any netdev is in OOR state. Return true if there's at least
> + * one netdev that's in OOR state; otherwise return false.
> + */
> +bool
> +netdev_any_oor(void)
> +    OVS_EXCLUDED(netdev_hmap_mutex)
> +{
> +    struct port_to_netdev_data *data;
> +    bool oor = false;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +        struct netdev *dev = data->netdev;
> +
> +        if (dev->hw_info.oor) {
> +            oor = true;
> +            break;
> +        }
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    return oor;
> +}
> +
> +bool
> +netdev_is_flow_api_enabled(void)
> +{
> +    return netdev_flow_api_enabled;
> +}
> +
> +void
> +netdev_ports_flow_flush(const struct dpif_class *dpif_class)
> +{
> +    struct port_to_netdev_data *data;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +        if (data->dpif_class == dpif_class) {
> +            netdev_flow_flush(data->netdev);
> +        }
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +}
> +
> +struct netdev_flow_dump **
> +netdev_ports_flow_dump_create(const struct dpif_class *dpif_class, int *ports)
> +{
> +    struct port_to_netdev_data *data;
> +    struct netdev_flow_dump **dumps;
> +    int count = 0;
> +    int i = 0;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +        if (data->dpif_class == dpif_class) {
> +            count++;
> +        }
> +    }
> +
> +    dumps = count ? xzalloc(sizeof *dumps * count) : NULL;
> +
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +        if (data->dpif_class == dpif_class) {
> +            if (netdev_flow_dump_create(data->netdev, &dumps[i])) {
> +                continue;
> +            }
> +
> +            dumps[i]->port = data->dpif_port.port_no;
> +            i++;
> +        }
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    *ports = i;
> +    return dumps;
> +}
> +
> +int
> +netdev_ports_flow_del(const struct dpif_class *dpif_class,
> +                      const ovs_u128 *ufid,
> +                      struct dpif_flow_stats *stats)
> +{
> +    struct port_to_netdev_data *data;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +        if (data->dpif_class == dpif_class
> +            && !netdev_flow_del(data->netdev, ufid, stats)) {
> +            ovs_mutex_unlock(&netdev_hmap_mutex);
> +            return 0;
> +        }
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    return ENOENT;
> +}
> +
> +int
> +netdev_ports_flow_get(const struct dpif_class *dpif_class, struct match *match,
> +                      struct nlattr **actions, const ovs_u128 *ufid,
> +                      struct dpif_flow_stats *stats,
> +                      struct dpif_flow_attrs *attrs, struct ofpbuf *buf)
> +{
> +    struct port_to_netdev_data *data;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +        if (data->dpif_class == dpif_class
> +            && !netdev_flow_get(data->netdev, match, actions,
> +                                ufid, stats, attrs, buf)) {
> +            ovs_mutex_unlock(&netdev_hmap_mutex);
> +            return 0;
> +        }
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +    return ENOENT;
> +}
> +
> +static uint32_t
> +netdev_ports_hash(odp_port_t port, const struct dpif_class *dpif_class)
> +{
> +    return hash_int(odp_to_u32(port), hash_pointer(dpif_class, 0));
> +}
> +
> +static struct port_to_netdev_data *
> +netdev_ports_lookup(odp_port_t port_no, const struct dpif_class *dpif_class)
> +    OVS_REQUIRES(netdev_hmap_mutex)
> +{
> +    struct port_to_netdev_data *data;
> +
> +    HMAP_FOR_EACH_WITH_HASH (data, portno_node,
> +                             netdev_ports_hash(port_no, dpif_class),
> +                             &port_to_netdev) {
> +        if (data->dpif_class == dpif_class
> +            && data->dpif_port.port_no == port_no) {
> +            return data;
> +        }
> +    }
> +    return NULL;
> +}
> +
> +int
> +netdev_ports_insert(struct netdev *netdev, const struct dpif_class *dpif_class,
> +                    struct dpif_port *dpif_port)
> +{
> +    struct port_to_netdev_data *data;
> +    int ifindex = netdev_get_ifindex(netdev);
> +
> +    if (ifindex < 0) {
> +        return ENODEV;
> +    }
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    if (netdev_ports_lookup(dpif_port->port_no, dpif_class)) {
> +        ovs_mutex_unlock(&netdev_hmap_mutex);
> +        return EEXIST;
> +    }
> +
> +    data = xzalloc(sizeof *data);
> +    data->netdev = netdev_ref(netdev);
> +    data->dpif_class = dpif_class;
> +    dpif_port_clone(&data->dpif_port, dpif_port);
> +    data->ifindex = ifindex;
> +
> +    hmap_insert(&port_to_netdev, &data->portno_node,
> +                netdev_ports_hash(dpif_port->port_no, dpif_class));
> +    hmap_insert(&ifindex_to_port, &data->ifindex_node, ifindex);
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    netdev_init_flow_api(netdev);
> +
> +    return 0;
> +}
> +
> +struct netdev *
> +netdev_ports_get(odp_port_t port_no, const struct dpif_class *dpif_class)
> +{
> +    struct port_to_netdev_data *data;
> +    struct netdev *ret = NULL;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    data = netdev_ports_lookup(port_no, dpif_class);
> +    if (data) {
> +        ret = netdev_ref(data->netdev);
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    return ret;
> +}
> +
> +int
> +netdev_ports_remove(odp_port_t port_no, const struct dpif_class *dpif_class)
> +{
> +    struct port_to_netdev_data *data;
> +    int ret = ENOENT;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +
> +    data = netdev_ports_lookup(port_no, dpif_class);
> +    if (data) {
> +        dpif_port_destroy(&data->dpif_port);
> +        netdev_close(data->netdev); /* unref and possibly close */
> +        hmap_remove(&port_to_netdev, &data->portno_node);
> +        hmap_remove(&ifindex_to_port, &data->ifindex_node);
> +        free(data);
> +        ret = 0;
> +    }
> +
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    return ret;
> +}
> +
> +odp_port_t
> +netdev_ifindex_to_odp_port(int ifindex)
> +{
> +    struct port_to_netdev_data *data;
> +    odp_port_t ret = 0;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH_WITH_HASH (data, ifindex_node, ifindex, &ifindex_to_port) {
> +        if (data->ifindex == ifindex) {
> +            ret = data->dpif_port.port_no;
> +            break;
> +        }
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +
> +    return ret;
> +}
> +
> +static bool netdev_offload_rebalance_policy = false;
> +
> +bool
> +netdev_is_offload_rebalance_policy_enabled(void)
> +{
> +    return netdev_offload_rebalance_policy;
> +}
> +
> +static void
> +netdev_ports_flow_init(void)
> +{
> +    struct port_to_netdev_data *data;
> +
> +    ovs_mutex_lock(&netdev_hmap_mutex);
> +    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> +       netdev_init_flow_api(data->netdev);
> +    }
> +    ovs_mutex_unlock(&netdev_hmap_mutex);
> +}
> +
> +void
> +netdev_set_flow_api_enabled(const struct smap *ovs_other_config)
> +{
> +    if (smap_get_bool(ovs_other_config, "hw-offload", false)) {
> +        static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
> +
> +        if (ovsthread_once_start(&once)) {
> +            netdev_flow_api_enabled = true;
> +
> +            VLOG_INFO("netdev: Flow API Enabled");
> +
> +#ifdef __linux__
> +            tc_set_policy(smap_get_def(ovs_other_config, "tc-policy",
> +                                       TC_POLICY_DEFAULT));
> +#endif
> +
> +            if (smap_get_bool(ovs_other_config, "offload-rebalance", false)) {
> +                netdev_offload_rebalance_policy = true;
> +            }
> +
> +            netdev_ports_flow_init();
> +
> +            ovsthread_once_done(&once);
> +        }
> +    }
> +}
> diff --git a/lib/netdev-offload.h b/lib/netdev-offload.h
> new file mode 100644
> index 000000000..97a500647
> --- /dev/null
> +++ b/lib/netdev-offload.h
> @@ -0,0 +1,126 @@
> +/*
> + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc.
> + * Copyright (c) 2019 Samsung Electronics Co.,Ltd.
> + *
> + * Licensed under the Apache License, Version 2.0 (the "License");
> + * you may not use this file except in compliance with the License.
> + * You may obtain a copy of the License at:
> + *
> + *     https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.apache.org%2Flicenses%2FLICENSE-2.0&amp;data=02%7C01%7Croid%40mellanox.com%7Caf61e39fa12b40deabd208d6d94b12bf%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636935313803077804&amp;sdata=lzSmYVMeOCeSt8kVm7MhBQTxVivkYo0T8Mero6t%2Fqw4%3D&amp;reserved=0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +#ifndef NETDEV_OFFLOAD_H
> +#define NETDEV_OFFLOAD_H 1
> +
> +#include "openvswitch/netdev.h"
> +#include "openvswitch/types.h"
> +#include "packets.h"
> +#include "flow.h"
> +
> +#ifdef  __cplusplus
> +extern "C" {
> +#endif
> +
> +struct dp_packet_batch;
> +struct dp_packet;
> +struct netdev_class;
> +struct netdev_rxq;
> +struct netdev_saved_flags;
> +struct ofpbuf;
> +struct in_addr;
> +struct in6_addr;
> +struct smap;
> +struct sset;
> +struct ovs_action_push_tnl;
> +
> +
> +/* Offload-capable (HW) netdev information */
> +struct netdev_hw_info {
> +    bool oor;		/* Out of Offload Resources ? */
> +    int offload_count;  /* Pending (non-offloaded) flow count */
> +    int pending_count;  /* Offloaded flow count */
> +};
> +
> +enum hw_info_type {
> +    HW_INFO_TYPE_OOR = 1,		/* OOR state */
> +    HW_INFO_TYPE_PEND_COUNT = 2,	/* Pending(non-offloaded) flow count */
> +    HW_INFO_TYPE_OFFL_COUNT = 3		/* Offloaded flow count */
> +};
> +
> +struct netdev_flow_dump {
> +    struct netdev *netdev;
> +    odp_port_t port;
> +    bool terse;
> +    struct nl_dump *nl_dump;
> +};
> +
> +/* Flow offloading. */
> +struct offload_info {
> +    const struct dpif_class *dpif_class;
> +    ovs_be16 tp_dst_port; /* Destination port for tunnel in SET action */
> +    uint8_t tunnel_csum_on; /* Tunnel header with checksum */
> +
> +    /*
> +     * The flow mark id assigened to the flow. If any pkts hit the flow,
> +     * it will be in the pkt meta data.
> +     */
> +    uint32_t flow_mark;
> +};
> +
> +int netdev_flow_flush(struct netdev *);
> +int netdev_flow_dump_create(struct netdev *, struct netdev_flow_dump **dump);
> +int netdev_flow_dump_destroy(struct netdev_flow_dump *);
> +bool netdev_flow_dump_next(struct netdev_flow_dump *, struct match *,
> +                          struct nlattr **actions, struct dpif_flow_stats *,
> +                          struct dpif_flow_attrs *, ovs_u128 *ufid,
> +                          struct ofpbuf *rbuffer, struct ofpbuf *wbuffer);
> +int netdev_flow_put(struct netdev *, struct match *, struct nlattr *actions,
> +                    size_t actions_len, const ovs_u128 *,
> +                    struct offload_info *, struct dpif_flow_stats *);
> +int netdev_flow_get(struct netdev *, struct match *, struct nlattr **actions,
> +                    const ovs_u128 *, struct dpif_flow_stats *,
> +                    struct dpif_flow_attrs *, struct ofpbuf *wbuffer);
> +int netdev_flow_del(struct netdev *, const ovs_u128 *,
> +                    struct dpif_flow_stats *);
> +int netdev_init_flow_api(struct netdev *);
> +void netdev_uninit_flow_api(struct netdev *);
> +uint32_t netdev_get_block_id(struct netdev *);
> +int netdev_get_hw_info(struct netdev *, int);
> +void netdev_set_hw_info(struct netdev *, int, int);
> +bool netdev_any_oor(void);
> +bool netdev_is_flow_api_enabled(void);
> +void netdev_set_flow_api_enabled(const struct smap *ovs_other_config);
> +bool netdev_is_offload_rebalance_policy_enabled(void);
> +
> +struct dpif_class;
> +struct dpif_port;
> +int netdev_ports_insert(struct netdev *, const struct dpif_class *,
> +                        struct dpif_port *);
> +struct netdev *netdev_ports_get(odp_port_t port, const struct dpif_class *);
> +int netdev_ports_remove(odp_port_t port, const struct dpif_class *);
> +odp_port_t netdev_ifindex_to_odp_port(int ifindex);
> +
> +struct netdev_flow_dump **netdev_ports_flow_dump_create(
> +                                        const struct dpif_class *,
> +                                        int *ports);
> +void netdev_ports_flow_flush(const struct dpif_class *);
> +int netdev_ports_flow_del(const struct dpif_class *, const ovs_u128 *ufid,
> +                          struct dpif_flow_stats *stats);
> +int netdev_ports_flow_get(const struct dpif_class *, struct match *match,
> +                          struct nlattr **actions,
> +                          const ovs_u128 *ufid,
> +                          struct dpif_flow_stats *stats,
> +                          struct dpif_flow_attrs *attrs,
> +                          struct ofpbuf *buf);
> +
> +#ifdef  __cplusplus
> +}
> +#endif
> +
> +#endif /* netdev-offload.h */
> diff --git a/lib/netdev-provider.h b/lib/netdev-provider.h
> index 653854cbd..b2e707888 100644
> --- a/lib/netdev-provider.h
> +++ b/lib/netdev-provider.h
> @@ -21,6 +21,7 @@
>  
>  #include "connectivity.h"
>  #include "netdev.h"
> +#include "netdev-offload.h"
>  #include "openvswitch/list.h"
>  #include "ovs-numa.h"
>  #include "ovs-rcu.h"
> @@ -36,19 +37,6 @@ extern "C" {
>  struct netdev_tnl_build_header_params;
>  #define NETDEV_NUMA_UNSPEC OVS_NUMA_UNSPEC
>  
> -/* Offload-capable (HW) netdev information */
> -struct netdev_hw_info {
> -    bool oor;		/* Out of Offload Resources ? */
> -    int offload_count;  /* Pending (non-offloaded) flow count */
> -    int pending_count;  /* Offloaded flow count */
> -};
> -
> -enum hw_info_type {
> -    HW_INFO_TYPE_OOR = 1,		/* OOR state */
> -    HW_INFO_TYPE_PEND_COUNT = 2,	/* Pending(non-offloaded) flow count */
> -    HW_INFO_TYPE_OFFL_COUNT = 3		/* Offloaded flow count */
> -};
> -
>  /* A network device (e.g. an Ethernet device).
>   *
>   * Network device implementations may read these members but should not modify
> @@ -139,13 +127,6 @@ struct netdev_rxq {
>  struct netdev *netdev_rxq_get_netdev(const struct netdev_rxq *);
>  
>  
> -struct netdev_flow_dump {
> -    struct netdev *netdev;
> -    odp_port_t port;
> -    bool terse;
> -    struct nl_dump *nl_dump;
> -};
> -
>  /* Network device class structure, to be defined by each implementation of a
>   * network device.
>   *
> diff --git a/lib/netdev-rte-offloads.c b/lib/netdev-rte-offloads.c
> index 683763f43..b5f16b68d 100644
> --- a/lib/netdev-rte-offloads.c
> +++ b/lib/netdev-rte-offloads.c
> @@ -15,7 +15,6 @@
>   * limitations under the License.
>   */
>  #include <config.h>
> -#include "netdev-rte-offloads.h"
>  
>  #include <rte_flow.h>
>  
> @@ -756,14 +755,9 @@ netdev_rte_offloads_init_flow_api(struct netdev *netdev)
>      return netdev_dpdk_flow_api_supported(netdev) ? 0 : EOPNOTSUPP;
>  }
>  
> -static const struct netdev_flow_api netdev_dpdk_offloads = {
> +const struct netdev_flow_api netdev_dpdk_offloads = {
>      .type = "dpdk_flow_api",
>      .flow_put = netdev_rte_offloads_flow_put,
>      .flow_del = netdev_rte_offloads_flow_del,
>      .init_flow_api = netdev_rte_offloads_init_flow_api,
>  };
> -
> -void netdev_dpdk_flow_api_register(void)
> -{
> -    netdev_register_flow_api_provider(&netdev_dpdk_offloads);
> -}
> diff --git a/lib/netdev-rte-offloads.h b/lib/netdev-rte-offloads.h
> deleted file mode 100644
> index b9b292114..000000000
> --- a/lib/netdev-rte-offloads.h
> +++ /dev/null
> @@ -1,22 +0,0 @@
> -/*
> - * Copyright (c) 2019 Mellanox Technologies, Ltd.
> - *
> - * Licensed under the Apache License, Version 2.0 (the "License");
> - * you may not use this file except in compliance with the License.
> - * You may obtain a copy of the License at:
> - *
> - *     https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.apache.org%2Flicenses%2FLICENSE-2.0&amp;data=02%7C01%7Croid%40mellanox.com%7Caf61e39fa12b40deabd208d6d94b12bf%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636935313803077804&amp;sdata=lzSmYVMeOCeSt8kVm7MhBQTxVivkYo0T8Mero6t%2Fqw4%3D&amp;reserved=0
> - *
> - * Unless required by applicable law or agreed to in writing, software
> - * distributed under the License is distributed on an "AS IS" BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> - * See the License for the specific language governing permissions and
> - * limitations under the License.
> - */
> -
> -#ifndef NETDEV_DPDK_OFFLOADS_H
> -#define NETDEV_DPDK_OFFLOADS_H 1
> -
> -void netdev_dpdk_flow_api_register(void);
> -
> -#endif /* netdev-rte-offloads.h */
> diff --git a/lib/netdev.c b/lib/netdev.c
> index de40f72d8..7f0fab8b8 100644
> --- a/lib/netdev.c
> +++ b/lib/netdev.c
> @@ -97,24 +97,6 @@ struct netdev_registered_class {
>      struct ovs_refcount refcnt;
>  };
>  
> -static bool netdev_flow_api_enabled = false;
> -
> -/* Protects 'netdev_flow_apis'.  */
> -static struct ovs_mutex netdev_flow_api_provider_mutex = OVS_MUTEX_INITIALIZER;
> -
> -/* Contains 'struct netdev_registered_flow_api's. */
> -static struct cmap netdev_flow_apis = CMAP_INITIALIZER;
> -
> -struct netdev_registered_flow_api {
> -    struct cmap_node cmap_node; /* In 'netdev_flow_apis', by flow_api->type. */
> -    const struct netdev_flow_api *flow_api;
> -
> -    /* Number of references: one for the flow_api itself and one for every
> -     * instance of the netdev that uses it. */
> -    struct ovs_refcount refcnt;
> -};
> -
> -
>  /* This is set pretty low because we probably won't learn anything from the
>   * additional log messages. */
>  static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
> @@ -298,87 +280,6 @@ netdev_unregister_provider(const char *type)
>      return error;
>  }
>  
> -static struct netdev_registered_flow_api *
> -netdev_lookup_flow_api(const char *type)
> -{
> -    struct netdev_registered_flow_api *rfa;
> -    CMAP_FOR_EACH_WITH_HASH (rfa, cmap_node, hash_string(type, 0),
> -                             &netdev_flow_apis) {
> -        if (!strcmp(type, rfa->flow_api->type)) {
> -            return rfa;
> -        }
> -    }
> -    return NULL;
> -}
> -
> -/* Registers a new netdev flow api provider. */
> -int
> -netdev_register_flow_api_provider(const struct netdev_flow_api *new_flow_api)
> -    OVS_EXCLUDED(netdev_flow_api_provider_mutex)
> -{
> -    int error = 0;
> -
> -    if (!new_flow_api->init_flow_api) {
> -        VLOG_WARN("attempted to register invalid flow api provider: %s",
> -                   new_flow_api->type);
> -        error = EINVAL;
> -    }
> -
> -    ovs_mutex_lock(&netdev_flow_api_provider_mutex);
> -    if (netdev_lookup_flow_api(new_flow_api->type)) {
> -        VLOG_WARN("attempted to register duplicate flow api provider: %s",
> -                   new_flow_api->type);
> -        error = EEXIST;
> -    } else {
> -        struct netdev_registered_flow_api *rfa;
> -
> -        rfa = xmalloc(sizeof *rfa);
> -        cmap_insert(&netdev_flow_apis, &rfa->cmap_node,
> -                    hash_string(new_flow_api->type, 0));
> -        rfa->flow_api = new_flow_api;
> -        ovs_refcount_init(&rfa->refcnt);
> -        VLOG_DBG("netdev: flow API '%s' registered.", new_flow_api->type);
> -    }
> -    ovs_mutex_unlock(&netdev_flow_api_provider_mutex);
> -
> -    return error;
> -}
> -
> -/* Unregisters a netdev flow api provider.  'type' must have been previously
> - * registered and not currently be in use by any netdevs.  After unregistration
> - * netdev flow api of that type cannot be used for netdevs.  (However, the
> - * provider may still be accessible from other threads until the next RCU grace
> - * period, so the caller must not free or re-register the same netdev_flow_api
> - * until that has passed.) */
> -int
> -netdev_unregister_flow_api_provider(const char *type)
> -    OVS_EXCLUDED(netdev_flow_api_provider_mutex)
> -{
> -    struct netdev_registered_flow_api *rfa;
> -    int error;
> -
> -    ovs_mutex_lock(&netdev_flow_api_provider_mutex);
> -    rfa = netdev_lookup_flow_api(type);
> -    if (!rfa) {
> -        VLOG_WARN("attempted to unregister a flow api provider that is not "
> -                  "registered: %s", type);
> -        error = EAFNOSUPPORT;
> -    } else if (ovs_refcount_unref(&rfa->refcnt) != 1) {
> -        ovs_refcount_ref(&rfa->refcnt);
> -        VLOG_WARN("attempted to unregister in use flow api provider: %s",
> -                  type);
> -        error = EBUSY;
> -    } else  {
> -        cmap_remove(&netdev_flow_apis, &rfa->cmap_node,
> -                    hash_string(rfa->flow_api->type, 0));
> -        ovsrcu_postpone(free, rfa);
> -        error = 0;
> -    }
> -    ovs_mutex_unlock(&netdev_flow_api_provider_mutex);
> -
> -    return error;
> -}
> -
>  /* Clears 'types' and enumerates the types of all currently registered netdev
>   * providers into it.  The caller must first initialize the sset. */
>  void
> @@ -663,10 +564,10 @@ netdev_unref(struct netdev *dev)
>      ovs_assert(dev->ref_cnt);
>      if (!--dev->ref_cnt) {
>          const struct netdev_class *class = dev->netdev_class;
> -        const struct netdev_flow_api *flow_api =
> -            ovsrcu_get(const struct netdev_flow_api *, &dev->flow_api);
>          struct netdev_registered_class *rc;
>  
> +        netdev_uninit_flow_api(dev);
> +
>          dev->netdev_class->destruct(dev);
>  
>          if (dev->node) {
> @@ -679,12 +580,6 @@ netdev_unref(struct netdev *dev)
>  
>          rc = netdev_lookup_class(class->type);
>          ovs_refcount_unref(&rc->refcnt);
> -
> -        if (flow_api) {
> -            struct netdev_registered_flow_api *rfa =
> -                                    netdev_lookup_flow_api(flow_api->type);
> -            ovs_refcount_unref(&rfa->refcnt);
> -        }
>      } else {
>          ovs_mutex_unlock(&netdev_mutex);
>      }
> @@ -2257,444 +2152,6 @@ netdev_reconfigure(struct netdev *netdev)
>              : EOPNOTSUPP);
>  }
>  
> -static int
> -netdev_assign_flow_api(struct netdev *netdev)
> -{
> -    struct netdev_registered_flow_api *rfa;
> -
> -    CMAP_FOR_EACH (rfa, cmap_node, &netdev_flow_apis) {
> -        if (!rfa->flow_api->init_flow_api(netdev)) {
> -            ovs_refcount_ref(&rfa->refcnt);
> -            ovsrcu_set(&netdev->flow_api, rfa->flow_api);
> -            VLOG_INFO("%s: Assigned flow API '%s'.",
> -                      netdev_get_name(netdev), rfa->flow_api->type);
> -            return 0;
> -        }
> -        VLOG_DBG("%s: flow API '%s' is not suitable.",
> -                 netdev_get_name(netdev), rfa->flow_api->type);
> -    }
> -    VLOG_INFO("%s: No suitable flow API found.", netdev_get_name(netdev));
> -
> -    return -1;
> -}
> -
> -int
> -netdev_flow_flush(struct netdev *netdev)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_flush)
> -           ? flow_api->flow_flush(netdev)
> -           : EOPNOTSUPP;
> -}
> -
> -int
> -netdev_flow_dump_create(struct netdev *netdev, struct netdev_flow_dump **dump)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_dump_create)
> -           ? flow_api->flow_dump_create(netdev, dump)
> -           : EOPNOTSUPP;
> -}
> -
> -int
> -netdev_flow_dump_destroy(struct netdev_flow_dump *dump)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &dump->netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_dump_destroy)
> -           ? flow_api->flow_dump_destroy(dump)
> -           : EOPNOTSUPP;
> -}
> -
> -bool
> -netdev_flow_dump_next(struct netdev_flow_dump *dump, struct match *match,
> -                      struct nlattr **actions, struct dpif_flow_stats *stats,
> -                      struct dpif_flow_attrs *attrs, ovs_u128 *ufid,
> -                      struct ofpbuf *rbuffer, struct ofpbuf *wbuffer)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &dump->netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_dump_next)
> -           ? flow_api->flow_dump_next(dump, match, actions, stats, attrs,
> -                                      ufid, rbuffer, wbuffer)
> -           : false;
> -}
> -
> -int
> -netdev_flow_put(struct netdev *netdev, struct match *match,
> -                struct nlattr *actions, size_t act_len,
> -                const ovs_u128 *ufid, struct offload_info *info,
> -                struct dpif_flow_stats *stats)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_put)
> -           ? flow_api->flow_put(netdev, match, actions, act_len, ufid,
> -                                info, stats)
> -           : EOPNOTSUPP;
> -}
> -
> -int
> -netdev_flow_get(struct netdev *netdev, struct match *match,
> -                struct nlattr **actions, const ovs_u128 *ufid,
> -                struct dpif_flow_stats *stats,
> -                struct dpif_flow_attrs *attrs, struct ofpbuf *buf)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_get)
> -           ? flow_api->flow_get(netdev, match, actions, ufid,
> -                                stats, attrs, buf)
> -           : EOPNOTSUPP;
> -}
> -
> -int
> -netdev_flow_del(struct netdev *netdev, const ovs_u128 *ufid,
> -                struct dpif_flow_stats *stats)
> -{
> -    const struct netdev_flow_api *flow_api =
> -        ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
> -
> -    return (flow_api && flow_api->flow_del)
> -           ? flow_api->flow_del(netdev, ufid, stats)
> -           : EOPNOTSUPP;
> -}
> -
> -int
> -netdev_init_flow_api(struct netdev *netdev)
> -{
> -    if (!netdev_is_flow_api_enabled()) {
> -        return EOPNOTSUPP;
> -    }
> -
> -    if (ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api)) {
> -        return 0;
> -    }
> -
> -    if (netdev_assign_flow_api(netdev)) {
> -        return EOPNOTSUPP;
> -    }
> -
> -    return 0;
> -}
> -
> -uint32_t
> -netdev_get_block_id(struct netdev *netdev)
> -{
> -    const struct netdev_class *class = netdev->netdev_class;
> -
> -    return (class->get_block_id
> -            ? class->get_block_id(netdev)
> -            : 0);
> -}
> -
> -/*
> - * Get the value of the hw info parameter specified by type.
> - * Returns the value on success (>= 0). Returns -1 on failure.
> - */
> -int
> -netdev_get_hw_info(struct netdev *netdev, int type)
> -{
> -    int val = -1;
> -
> -    switch (type) {
> -    case HW_INFO_TYPE_OOR:
> -        val = netdev->hw_info.oor;
> -        break;
> -    case HW_INFO_TYPE_PEND_COUNT:
> -        val = netdev->hw_info.pending_count;
> -        break;
> -    case HW_INFO_TYPE_OFFL_COUNT:
> -        val = netdev->hw_info.offload_count;
> -        break;
> -    default:
> -        break;
> -    }
> -
> -    return val;
> -}
> -
> -/*
> - * Set the value of the hw info parameter specified by type.
> - */
> -void
> -netdev_set_hw_info(struct netdev *netdev, int type, int val)
> -{
> -    switch (type) {
> -    case HW_INFO_TYPE_OOR:
> -        if (val == 0) {
> -            VLOG_DBG("Offload rebalance: netdev: %s is not OOR", netdev->name);
> -        }
> -        netdev->hw_info.oor = val;
> -        break;
> -    case HW_INFO_TYPE_PEND_COUNT:
> -        netdev->hw_info.pending_count = val;
> -        break;
> -    case HW_INFO_TYPE_OFFL_COUNT:
> -        netdev->hw_info.offload_count = val;
> -        break;
> -    default:
> -        break;
> -    }
> -}
> -
> -/*
> - * Find if any netdev is in OOR state. Return true if there's at least
> - * one netdev that's in OOR state; otherwise return false.
> - */
> -bool
> -netdev_any_oor(void)
> -    OVS_EXCLUDED(netdev_mutex)
> -{
> -    struct shash_node *node;
> -    bool oor = false;
> -
> -    ovs_mutex_lock(&netdev_mutex);
> -    SHASH_FOR_EACH (node, &netdev_shash) {
> -        struct netdev *dev = node->data;
> -
> -        if (dev->hw_info.oor) {
> -            oor = true;
> -            break;
> -        }
> -    }
> -    ovs_mutex_unlock(&netdev_mutex);
> -
> -    return oor;
> -}
> -
> -bool
> -netdev_is_flow_api_enabled(void)
> -{
> -    return netdev_flow_api_enabled;
> -}
> -
> -/* Protects below port hashmaps. */
> -static struct ovs_mutex netdev_hmap_mutex = OVS_MUTEX_INITIALIZER;
> -
> -static struct hmap port_to_netdev OVS_GUARDED_BY(netdev_hmap_mutex)
> -    = HMAP_INITIALIZER(&port_to_netdev);
> -static struct hmap ifindex_to_port OVS_GUARDED_BY(netdev_hmap_mutex)
> -    = HMAP_INITIALIZER(&ifindex_to_port);
> -
> -struct port_to_netdev_data {
> -    struct hmap_node portno_node; /* By (dpif_class, dpif_port.port_no). */
> -    struct hmap_node ifindex_node; /* By (dpif_class, ifindex). */
> -    struct netdev *netdev;
> -    struct dpif_port dpif_port;
> -    const struct dpif_class *dpif_class;
> -    int ifindex;
> -};
> -
> -static uint32_t
> -netdev_ports_hash(odp_port_t port, const struct dpif_class *dpif_class)
> -{
> -    return hash_int(odp_to_u32(port), hash_pointer(dpif_class, 0));
> -}
> -
> -static struct port_to_netdev_data *
> -netdev_ports_lookup(odp_port_t port_no, const struct dpif_class *dpif_class)
> -    OVS_REQUIRES(netdev_hmap_mutex)
> -{
> -    struct port_to_netdev_data *data;
> -
> -    HMAP_FOR_EACH_WITH_HASH (data, portno_node,
> -                             netdev_ports_hash(port_no, dpif_class),
> -                             &port_to_netdev) {
> -        if (data->dpif_class == dpif_class
> -            && data->dpif_port.port_no == port_no) {
> -            return data;
> -        }
> -    }
> -    return NULL;
> -}
> -
> -int
> -netdev_ports_insert(struct netdev *netdev, const struct dpif_class *dpif_class,
> -                    struct dpif_port *dpif_port)
> -{
> -    struct port_to_netdev_data *data;
> -    int ifindex = netdev_get_ifindex(netdev);
> -
> -    if (ifindex < 0) {
> -        return ENODEV;
> -    }
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    if (netdev_ports_lookup(dpif_port->port_no, dpif_class)) {
> -        ovs_mutex_unlock(&netdev_hmap_mutex);
> -        return EEXIST;
> -    }
> -
> -    data = xzalloc(sizeof *data);
> -    data->netdev = netdev_ref(netdev);
> -    data->dpif_class = dpif_class;
> -    dpif_port_clone(&data->dpif_port, dpif_port);
> -    data->ifindex = ifindex;
> -
> -    hmap_insert(&port_to_netdev, &data->portno_node,
> -                netdev_ports_hash(dpif_port->port_no, dpif_class));
> -    hmap_insert(&ifindex_to_port, &data->ifindex_node, ifindex);
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -
> -    netdev_init_flow_api(netdev);
> -
> -    return 0;
> -}
> -
> -struct netdev *
> -netdev_ports_get(odp_port_t port_no, const struct dpif_class *dpif_class)
> -{
> -    struct port_to_netdev_data *data;
> -    struct netdev *ret = NULL;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    data = netdev_ports_lookup(port_no, dpif_class);
> -    if (data) {
> -        ret = netdev_ref(data->netdev);
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -
> -    return ret;
> -}
> -
> -int
> -netdev_ports_remove(odp_port_t port_no, const struct dpif_class *dpif_class)
> -{
> -    struct port_to_netdev_data *data;
> -    int ret = ENOENT;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -
> -    data = netdev_ports_lookup(port_no, dpif_class);
> -    if (data) {
> -        dpif_port_destroy(&data->dpif_port);
> -        netdev_close(data->netdev); /* unref and possibly close */
> -        hmap_remove(&port_to_netdev, &data->portno_node);
> -        hmap_remove(&ifindex_to_port, &data->ifindex_node);
> -        free(data);
> -        ret = 0;
> -    }
> -
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -
> -    return ret;
> -}
> -
> -odp_port_t
> -netdev_ifindex_to_odp_port(int ifindex)
> -{
> -    struct port_to_netdev_data *data;
> -    odp_port_t ret = 0;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    HMAP_FOR_EACH_WITH_HASH (data, ifindex_node, ifindex, &ifindex_to_port) {
> -        if (data->ifindex == ifindex) {
> -            ret = data->dpif_port.port_no;
> -            break;
> -        }
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -
> -    return ret;
> -}
> -
> -void
> -netdev_ports_flow_flush(const struct dpif_class *dpif_class)
> -{
> -    struct port_to_netdev_data *data;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> -        if (data->dpif_class == dpif_class) {
> -            netdev_flow_flush(data->netdev);
> -        }
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -}
> -
> -struct netdev_flow_dump **
> -netdev_ports_flow_dump_create(const struct dpif_class *dpif_class, int *ports)
> -{
> -    struct port_to_netdev_data *data;
> -    struct netdev_flow_dump **dumps;
> -    int count = 0;
> -    int i = 0;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> -        if (data->dpif_class == dpif_class) {
> -            count++;
> -        }
> -    }
> -
> -    dumps = count ? xzalloc(sizeof *dumps * count) : NULL;
> -
> -    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> -        if (data->dpif_class == dpif_class) {
> -            if (netdev_flow_dump_create(data->netdev, &dumps[i])) {
> -                continue;
> -            }
> -
> -            dumps[i]->port = data->dpif_port.port_no;
> -            i++;
> -        }
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -
> -    *ports = i;
> -    return dumps;
> -}
> -
> -int
> -netdev_ports_flow_del(const struct dpif_class *dpif_class,
> -                      const ovs_u128 *ufid,
> -                      struct dpif_flow_stats *stats)
> -{
> -    struct port_to_netdev_data *data;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> -        if (data->dpif_class == dpif_class
> -            && !netdev_flow_del(data->netdev, ufid, stats)) {
> -            ovs_mutex_unlock(&netdev_hmap_mutex);
> -            return 0;
> -        }
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -
> -    return ENOENT;
> -}
> -
> -int
> -netdev_ports_flow_get(const struct dpif_class *dpif_class, struct match *match,
> -                      struct nlattr **actions, const ovs_u128 *ufid,
> -                      struct dpif_flow_stats *stats,
> -                      struct dpif_flow_attrs *attrs, struct ofpbuf *buf)
> -{
> -    struct port_to_netdev_data *data;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> -        if (data->dpif_class == dpif_class
> -            && !netdev_flow_get(data->netdev, match, actions,
> -                                ufid, stats, attrs, buf)) {
> -            ovs_mutex_unlock(&netdev_hmap_mutex);
> -            return 0;
> -        }
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -    return ENOENT;
> -}
> -
>  void
>  netdev_free_custom_stats_counters(struct netdev_custom_stats *custom_stats)
>  {
> @@ -2706,50 +2163,3 @@ netdev_free_custom_stats_counters(struct netdev_custom_stats *custom_stats)
>          }
>      }
>  }
> -
> -static bool netdev_offload_rebalance_policy = false;
> -
> -bool
> -netdev_is_offload_rebalance_policy_enabled(void)
> -{
> -    return netdev_offload_rebalance_policy;
> -}
> -
> -static void
> -netdev_ports_flow_init(void)
> -{
> -    struct port_to_netdev_data *data;
> -
> -    ovs_mutex_lock(&netdev_hmap_mutex);
> -    HMAP_FOR_EACH (data, portno_node, &port_to_netdev) {
> -       netdev_init_flow_api(data->netdev);
> -    }
> -    ovs_mutex_unlock(&netdev_hmap_mutex);
> -}
> -
> -void
> -netdev_set_flow_api_enabled(const struct smap *ovs_other_config)
> -{
> -    if (smap_get_bool(ovs_other_config, "hw-offload", false)) {
> -        static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
> -
> -        if (ovsthread_once_start(&once)) {
> -            netdev_flow_api_enabled = true;
> -
> -            VLOG_INFO("netdev: Flow API Enabled");
> -
> -#ifdef __linux__
> -            tc_set_policy(smap_get_def(ovs_other_config, "tc-policy",
> -                                       TC_POLICY_DEFAULT));
> -#endif
> -
> -            if (smap_get_bool(ovs_other_config, "offload-rebalance", false)) {
> -                netdev_offload_rebalance_policy = true;
> -            }
> -
> -            netdev_ports_flow_init();
> -
> -            ovsthread_once_done(&once);
> -        }
> -    }
> -}
> diff --git a/lib/netdev.h b/lib/netdev.h
> index d94817fb6..e6e671843 100644
> --- a/lib/netdev.h
> +++ b/lib/netdev.h
> @@ -197,63 +197,6 @@ int netdev_send(struct netdev *, int qid, struct dp_packet_batch *,
>                  bool concurrent_txq);
>  void netdev_send_wait(struct netdev *, int qid);
>  
> -/* Flow offloading. */
> -struct offload_info {
> -    const struct dpif_class *dpif_class;
> -    ovs_be16 tp_dst_port; /* Destination port for tunnel in SET action */
> -    uint8_t tunnel_csum_on; /* Tunnel header with checksum */
> -
> -    /*
> -     * The flow mark id assigened to the flow. If any pkts hit the flow,
> -     * it will be in the pkt meta data.
> -     */
> -    uint32_t flow_mark;
> -};
> -struct dpif_class;
> -struct netdev_flow_dump;
> -int netdev_flow_flush(struct netdev *);
> -int netdev_flow_dump_create(struct netdev *, struct netdev_flow_dump **dump);
> -int netdev_flow_dump_destroy(struct netdev_flow_dump *);
> -bool netdev_flow_dump_next(struct netdev_flow_dump *, struct match *,
> -                          struct nlattr **actions, struct dpif_flow_stats *,
> -                          struct dpif_flow_attrs *, ovs_u128 *ufid,
> -                          struct ofpbuf *rbuffer, struct ofpbuf *wbuffer);
> -int netdev_flow_put(struct netdev *, struct match *, struct nlattr *actions,
> -                    size_t actions_len, const ovs_u128 *,
> -                    struct offload_info *, struct dpif_flow_stats *);
> -int netdev_flow_get(struct netdev *, struct match *, struct nlattr **actions,
> -                    const ovs_u128 *, struct dpif_flow_stats *,
> -                    struct dpif_flow_attrs *, struct ofpbuf *wbuffer);
> -int netdev_flow_del(struct netdev *, const ovs_u128 *,
> -                    struct dpif_flow_stats *);
> -int netdev_init_flow_api(struct netdev *);
> -uint32_t netdev_get_block_id(struct netdev *);
> -int netdev_get_hw_info(struct netdev *, int);
> -void netdev_set_hw_info(struct netdev *, int, int);
> -bool netdev_any_oor(void);
> -bool netdev_is_flow_api_enabled(void);
> -void netdev_set_flow_api_enabled(const struct smap *ovs_other_config);
> -bool netdev_is_offload_rebalance_policy_enabled(void);
> -
> -struct dpif_port;
> -int netdev_ports_insert(struct netdev *, const struct dpif_class *,
> -                        struct dpif_port *);
> -struct netdev *netdev_ports_get(odp_port_t port, const struct dpif_class *);
> -int netdev_ports_remove(odp_port_t port, const struct dpif_class *);
> -odp_port_t netdev_ifindex_to_odp_port(int ifindex);
> -struct netdev_flow_dump **netdev_ports_flow_dump_create(
> -                                        const struct dpif_class *,
> -                                        int *ports);
> -void netdev_ports_flow_flush(const struct dpif_class *);
> -int netdev_ports_flow_del(const struct dpif_class *, const ovs_u128 *ufid,
> -                          struct dpif_flow_stats *stats);
> -int netdev_ports_flow_get(const struct dpif_class *, struct match *match,
> -                          struct nlattr **actions,
> -                          const ovs_u128 *ufid,
> -                          struct dpif_flow_stats *stats,
> -                          struct dpif_flow_attrs *attrs,
> -                          struct ofpbuf *buf);
> -
>  /* native tunnel APIs */
>  /* Structure to pass parameters required to build a tunnel header. */
>  struct netdev_tnl_build_header_params {
> diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
> index 48d8c4de1..437a120ff 100644
> --- a/vswitchd/bridge.c
> +++ b/vswitchd/bridge.c
> @@ -38,6 +38,7 @@
>  #include "mac-learning.h"
>  #include "mcast-snooping.h"
>  #include "netdev.h"
> +#include "netdev-offload.h"
>  #include "nx-match.h"
>  #include "ofproto/bond.h"
>  #include "ofproto/ofproto.h"
> 

Acked-by: Roi Dayan <roid at mellanox.com>


More information about the dev mailing list