[ovs-dev] [PATCH v3 2/4] netdev-dpdk: Properly support non pmd threads.

Ethan Jackson ethan at nicira.com
Fri May 22 18:18:25 UTC 2015


Acked-by: Ethan Jackson <ethan at nicira.com>


On Fri, May 22, 2015 at 9:14 AM, Daniele Di Proietto
<diproiettod at vmware.com> wrote:
> We used to reserve DPDK lcore 0 for non pmd operations, making it
> difficult to use core 0 for packet processing.
> DPDK 2.0 properly support non EAL threads with lcore LCORE_ID_ANY.
>
> Using non EAL threads for non pmd threads, we do not need to reserve
> any core for non pmd operations
>
> Signed-off-by: Daniele Di Proietto <diproiettod at vmware.com>
> ---
>  INSTALL.DPDK.md   |  3 ---
>  lib/dpctl.c       |  6 ++++++
>  lib/dpif-netdev.c |  4 +---
>  lib/netdev-dpdk.c | 10 +---------
>  lib/netdev-dpdk.h | 16 ++++------------
>  lib/ovs-thread.c  |  2 --
>  6 files changed, 12 insertions(+), 29 deletions(-)
>
> diff --git a/INSTALL.DPDK.md b/INSTALL.DPDK.md
> index 068d631..462ba0e 100644
> --- a/INSTALL.DPDK.md
> +++ b/INSTALL.DPDK.md
> @@ -260,9 +260,6 @@ Using the DPDK with ovs-vswitchd:
>     Note, the pmd threads on a numa node are only created if there is at least
>     one DPDK interface from the numa node that has been added to OVS.
>
> -   Note, core 0 is always reserved from non-pmd threads and should never be set
> -   in the cpu mask.
> -
>     To understand where most of the time is spent and whether the caches are
>     effective, these commands can be used:
>
> diff --git a/lib/dpctl.c b/lib/dpctl.c
> index 05c28d1..e95483e 100644
> --- a/lib/dpctl.c
> +++ b/lib/dpctl.c
> @@ -783,6 +783,12 @@ dpctl_dump_flows(int argc, const char *argv[], struct dpctl_params *dpctl_p)
>          }
>      }
>
> +    /* Make sure that these values are different. PMD_ID_NULL means that the
> +     * pmd is unspecified (e.g. because the datapath doesn't have different
> +     * pmd threads), while NON_PMD_CORE_ID refers to every non pmd threads
> +     * in the userspace datapath */
> +    BUILD_ASSERT(PMD_ID_NULL != NON_PMD_CORE_ID);
> +
>      ds_init(&ds);
>      flow_dump = dpif_flow_dump_create(dpif, false);
>      flow_dump_thread = dpif_flow_dump_thread_create(flow_dump);
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index ace5cb5..76d1003 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -580,7 +580,7 @@ pmd_info_show_stats(struct ds *reply,
>      if (pmd->numa_id != OVS_NUMA_UNSPEC) {
>          ds_put_format(reply, " numa_id %d", pmd->numa_id);
>      }
> -    if (pmd->core_id != OVS_CORE_UNSPEC) {
> +    if (pmd->core_id != OVS_CORE_UNSPEC && pmd->core_id != NON_PMD_CORE_ID) {
>          ds_put_format(reply, " core_id %u", pmd->core_id);
>      }
>      ds_put_cstr(reply, ":\n");
> @@ -829,8 +829,6 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
>      ovs_mutex_init_recursive(&dp->non_pmd_mutex);
>      ovsthread_key_create(&dp->per_pmd_key, NULL);
>
> -    /* Reserves the core NON_PMD_CORE_ID for all non-pmd threads. */
> -    ovs_numa_try_pin_core_specific(NON_PMD_CORE_ID);
>      dp_netdev_set_nonpmd(dp);
>      dp->n_dpdk_rxqs = NR_QUEUE;
>
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index a4868cc..3fe5a82 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -1927,7 +1927,7 @@ dpdk_init(int argc, char **argv)
>      }
>
>      /* We are called from the main thread here */
> -    thread_set_nonpmd();
> +    RTE_PER_LCORE(_lcore_id) = NON_PMD_CORE_ID;
>
>      return result + 1 + base;
>  }
> @@ -2012,14 +2012,6 @@ pmd_thread_setaffinity_cpu(unsigned cpu)
>      return 0;
>  }
>
> -void
> -thread_set_nonpmd(void)
> -{
> -    /* We have to use NON_PMD_CORE_ID to allow non-pmd threads to perform
> -     * certain DPDK operations, like rte_eth_dev_configure(). */
> -    RTE_PER_LCORE(_lcore_id) = NON_PMD_CORE_ID;
> -}
> -
>  static bool
>  thread_is_pmd(void)
>  {
> diff --git a/lib/netdev-dpdk.h b/lib/netdev-dpdk.h
> index 2924f23..646d3e2 100644
> --- a/lib/netdev-dpdk.h
> +++ b/lib/netdev-dpdk.h
> @@ -5,11 +5,6 @@
>
>  struct dp_packet;
>
> -/* Reserves cpu core 0 for all non-pmd threads.  Changing the value of this
> - * macro will allow pmd thread to be pinned on cpu core 0.  This may not be
> - * ideal since the core may be non-isolated. */
> -#define NON_PMD_CORE_ID 0
> -
>  #ifdef DPDK_NETDEV
>
>  #include <rte_config.h>
> @@ -25,14 +20,17 @@ struct dp_packet;
>  #include <rte_launch.h>
>  #include <rte_malloc.h>
>
> +#define NON_PMD_CORE_ID LCORE_ID_ANY
> +
>  int dpdk_init(int argc, char **argv);
>  void netdev_dpdk_register(void);
>  void free_dpdk_buf(struct dp_packet *);
>  int pmd_thread_setaffinity_cpu(unsigned cpu);
> -void thread_set_nonpmd(void);
>
>  #else
>
> +#define NON_PMD_CORE_ID UINT32_MAX
> +
>  #include "util.h"
>
>  static inline int
> @@ -62,11 +60,5 @@ pmd_thread_setaffinity_cpu(unsigned cpu OVS_UNUSED)
>      return 0;
>  }
>
> -static inline void
> -thread_set_nonpmd(void)
> -{
> -    /* Nothing */
> -}
> -
>  #endif /* DPDK_NETDEV */
>  #endif
> diff --git a/lib/ovs-thread.c b/lib/ovs-thread.c
> index 7d38c80..4161095 100644
> --- a/lib/ovs-thread.c
> +++ b/lib/ovs-thread.c
> @@ -334,8 +334,6 @@ ovsthread_wrapper(void *aux_)
>      set_subprogram_name("%s%u", aux.name, id);
>      ovsrcu_quiesce_end();
>
> -    thread_set_nonpmd();
> -
>      return aux.start(aux.arg);
>  }
>
> --
> 2.1.4
>
> _______________________________________________
> dev mailing list
> dev at openvswitch.org
> http://openvswitch.org/mailman/listinfo/dev



More information about the dev mailing list