[ovs-dev] [dpdk patch v4 1/5] netdev: Add function for configuring tx and rx queues.

Pravin Shelar pshelar at nicira.com
Mon Sep 15 17:07:07 UTC 2014


On Fri, Sep 12, 2014 at 3:04 PM, Alex Wang <alexw at nicira.com> wrote:
> This commit adds a new API to the 'struct netdev_class' which
> allows user to configure the number of tx queues and rx queues
> of 'netdev'.  Upcoming patches will use this function to set
> multiple tx/rx queues when adding the netdev to dpif-netdev.
>
> Currently, only netdev-dpdk module implements this function.
>
> Signed-off-by: Alex Wang <alexw at nicira.com>
>
> ---
> PATCH -> V4:
> - move this hidden patch forward.
> ---
>  lib/netdev-bsd.c      |    1 +
>  lib/netdev-dpdk.c     |   46 +++++++++++++++++++++++++++++++++++++---------
>  lib/netdev-dummy.c    |    1 +
>  lib/netdev-linux.c    |    1 +
>  lib/netdev-provider.h |   10 ++++++++++
>  lib/netdev-vport.c    |    1 +
>  lib/netdev.c          |   25 +++++++++++++++++++++++++
>  lib/netdev.h          |    1 +
>  8 files changed, 77 insertions(+), 9 deletions(-)
>
> diff --git a/lib/netdev-bsd.c b/lib/netdev-bsd.c
> index 32c04a3..20b4718 100644
> --- a/lib/netdev-bsd.c
> +++ b/lib/netdev-bsd.c
> @@ -1563,6 +1563,7 @@ netdev_bsd_update_flags(struct netdev *netdev_, enum netdev_flags off,
>      NULL, /* set_config */                           \
>      NULL, /* get_tunnel_config */                    \
>      NULL, /* get_numa_id */                          \
> +    NULL, /* set_multiq */                           \
>                                                       \
>      netdev_bsd_send,                                 \
>      netdev_bsd_send_wait,                            \
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index 4f9c5c2..d11c070 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -399,13 +399,14 @@ dpdk_eth_dev_init(struct netdev_dpdk *dev) OVS_REQUIRES(dpdk_mutex)
>          return ENODEV;
>      }
>
> -    diag = rte_eth_dev_configure(dev->port_id, NR_QUEUE, NR_QUEUE,  &port_conf);
> +    diag = rte_eth_dev_configure(dev->port_id, dev->up.n_rxq, dev->up.n_txq,
> +                                 &port_conf);
>      if (diag) {
>          VLOG_ERR("eth dev config error %d",diag);
>          return -diag;
>      }
>
> -    for (i = 0; i < NR_QUEUE; i++) {
> +    for (i = 0; i < dev->up.n_txq; i++) {
>          diag = rte_eth_tx_queue_setup(dev->port_id, i, NIC_PORT_TX_Q_SIZE,
>                                        dev->socket_id, &tx_conf);
>          if (diag) {
> @@ -414,7 +415,7 @@ dpdk_eth_dev_init(struct netdev_dpdk *dev) OVS_REQUIRES(dpdk_mutex)
>          }
>      }
>
> -    for (i = 0; i < NR_QUEUE; i++) {
> +    for (i = 0; i < dev->up.n_rxq; i++) {
>          diag = rte_eth_rx_queue_setup(dev->port_id, i, NIC_PORT_RX_Q_SIZE,
>                                        dev->socket_id,
>                                        &rx_conf, dev->dpdk_mp->mp);
> @@ -491,12 +492,12 @@ netdev_dpdk_init(struct netdev *netdev_, unsigned int port_no) OVS_REQUIRES(dpdk
>          goto unlock;
>      }
>
> +    netdev_->n_txq = NR_QUEUE;
> +    netdev_->n_rxq = NR_QUEUE;
>      err = dpdk_eth_dev_init(netdev);
>      if (err) {
>          goto unlock;
>      }
> -    netdev_->n_txq = NR_QUEUE;
> -    netdev_->n_rxq = NR_QUEUE;
>
>      list_push_back(&dpdk_list, &netdev->list_node);
>
> @@ -590,6 +591,26 @@ netdev_dpdk_get_numa_id(const struct netdev *netdev_)
>      return netdev->socket_id;
>  }
>
> +/* Sets the number of tx queues and rx queues for the dpdk interface.
> + * If the configuration fails, do not try restoring its old configuration
> + * and just returns the error. */
> +static int
> +netdev_dpdk_set_multiq(struct netdev *netdev_, unsigned int n_txq,
> +                       unsigned int n_rxq)
> +{
> +    struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
> +    int err = 0;
> +
> +    ovs_mutex_lock(&netdev->mutex);
> +    rte_eth_dev_stop(netdev->port_id);
> +    netdev->up.n_txq = MAX(n_txq, NR_QUEUE);
> +    netdev->up.n_rxq = MAX(n_rxq, NR_QUEUE);

MAX check can be cone for netdev_set_multiq() since all netdev
implementation which support multi queue needs it.

> +    err = dpdk_eth_dev_init(netdev);
> +    ovs_mutex_unlock(&netdev->mutex);
> +
> +    return err;
> +}
> +
>  static struct netdev_rxq *
>  netdev_dpdk_rxq_alloc(void)
>  {
> @@ -687,7 +708,11 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct dpif_packet **packets,
>      struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
>      int nb_rx;
>
> -    dpdk_queue_flush(dev, rxq_->queue_id);
> +    /* There is only one tx queue for this core.  Do not flush other
> +     * queueus. */
> +    if (rxq_->queue_id == rte_lcore_id()) {
> +        dpdk_queue_flush(dev, rxq_->queue_id);
> +    }
>
If thread changes core, we might never flush for certain queues.

>      nb_rx = rte_eth_rx_burst(rx->port_id, rxq_->queue_id,
>                               (struct rte_mbuf **) packets,
> @@ -1339,7 +1364,7 @@ unlock_dpdk:
>      return err;
>  }
>
> -#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT)              \
> +#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, MULTIQ)      \
>  {                                                             \
>      NAME,                                                     \
>      INIT,                       /* init */                    \
> @@ -1354,6 +1379,7 @@ unlock_dpdk:
>      NULL,                       /* netdev_dpdk_set_config */  \
>      NULL,                       /* get_tunnel_config */       \
>      netdev_dpdk_get_numa_id,    /* get_numa_id */             \
> +    MULTIQ,                     /* set_multiq */              \
>                                                                \
>      netdev_dpdk_send,           /* send */                    \
>      NULL,                       /* send_wait */               \
> @@ -1441,13 +1467,15 @@ const struct netdev_class dpdk_class =
>      NETDEV_DPDK_CLASS(
>          "dpdk",
>          dpdk_class_init,
> -        netdev_dpdk_construct);
> +        netdev_dpdk_construct,
> +        netdev_dpdk_set_multiq);
>
>  const struct netdev_class dpdk_ring_class =
>      NETDEV_DPDK_CLASS(
>          "dpdkr",
>          NULL,
> -        netdev_dpdk_ring_construct);
> +        netdev_dpdk_ring_construct,
> +        NULL);
>
>  void
>  netdev_dpdk_register(void)
> diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c
> index 1334a67..e0df0b9 100644
> --- a/lib/netdev-dummy.c
> +++ b/lib/netdev-dummy.c
> @@ -1045,6 +1045,7 @@ static const struct netdev_class dummy_class = {
>      netdev_dummy_set_config,
>      NULL,                       /* get_tunnel_config */
>      NULL,                       /* get_numa_id */
> +    NULL,                       /* set_multiq */
>
>      netdev_dummy_send,          /* send */
>      NULL,                       /* send_wait */
> diff --git a/lib/netdev-linux.c b/lib/netdev-linux.c
> index e311122..531999e 100644
> --- a/lib/netdev-linux.c
> +++ b/lib/netdev-linux.c
> @@ -2751,6 +2751,7 @@ netdev_linux_update_flags(struct netdev *netdev_, enum netdev_flags off,
>      NULL,                       /* set_config */                \
>      NULL,                       /* get_tunnel_config */         \
>      NULL,                       /* get_numa_id */               \
> +    NULL,                       /* set_multiq */                \
>                                                                  \
>      netdev_linux_send,                                          \
>      netdev_linux_send_wait,                                     \
> diff --git a/lib/netdev-provider.h b/lib/netdev-provider.h
> index c08ef35..c92afff 100644
> --- a/lib/netdev-provider.h
> +++ b/lib/netdev-provider.h
> @@ -258,6 +258,16 @@ struct netdev_class {
>       * such info, returns NETDEV_NUMA_UNSPEC. */
>      int (*get_numa_id)(const struct netdev *netdev);
>
> +    /* Configures the number of tx queues and rx queues of 'netdev'.
> +     * Return 0 if successful, otherwise a positive errno value.
> +     *
> +     * On error, the tx queue and rx queue configuration is indeterminant.
> +     * Caller should make decision on whether to restore the previous or
> +     * the default configuration.  Also, caller must make sure there is no
> +     * other thread accessing the queues at the same time. */
> +    int (*set_multiq)(struct netdev *netdev, unsigned int n_txq,
> +                      unsigned int n_rxq);
> +
>      /* Sends buffers on 'netdev'.
>       * Returns 0 if successful (for every buffer), otherwise a positive errno
>       * value.  Returns EAGAIN without blocking if one or more packets cannot be
> diff --git a/lib/netdev-vport.c b/lib/netdev-vport.c
> index 6ab90bf..91689e7 100644
> --- a/lib/netdev-vport.c
> +++ b/lib/netdev-vport.c
> @@ -776,6 +776,7 @@ get_stats(const struct netdev *netdev, struct netdev_stats *stats)
>      SET_CONFIG,                                             \
>      GET_TUNNEL_CONFIG,                                      \
>      NULL,                       /* get_numa_id */           \
> +    NULL,                       /* set_multiq */            \
>                                                              \
>      NULL,                       /* send */                  \
>      NULL,                       /* send_wait */             \
> diff --git a/lib/netdev.c b/lib/netdev.c
> index 0d065e7..7e5a748 100644
> --- a/lib/netdev.c
> +++ b/lib/netdev.c
> @@ -663,6 +663,31 @@ netdev_rxq_drain(struct netdev_rxq *rx)
>              : 0);
>  }
>
> +/* Configures the number of tx queues and rx queues of 'netdev'.
> + * Return 0 if successful, otherwise a positive errno value.
> + *
> + * On error, the tx queue and rx queue configuration is indeterminant.
> + * Caller should make decision on whether to restore the previous or
> + * the default configuration.  Also, caller must make sure there is no
> + * other thread accessing the queues at the same time. */
> +int
> +netdev_set_multiq(struct netdev *netdev, unsigned int n_txq,
> +                  unsigned int n_rxq)
> +{
> +    int error;
> +
> +    error = (netdev->netdev_class->set_multiq
> +             ? netdev->netdev_class->set_multiq(netdev, n_txq, n_rxq)
> +             : EOPNOTSUPP);
> +
> +    if (error != EOPNOTSUPP) {
> +        VLOG_DBG_RL(&rl, "failed to set tx/rx queue for network device %s:"
> +                    "%s", netdev_get_name(netdev), ovs_strerror(error));
> +    }
> +
It should check if existing number of queues are same as new ones.

> +    return error;
> +}
> +
>  /* Sends 'buffers' on 'netdev'.  Returns 0 if successful (for every packet),
>   * otherwise a positive errno value.  Returns EAGAIN without blocking if
>   * at least one the packets cannot be queued immediately.  Returns EMSGSIZE
> diff --git a/lib/netdev.h b/lib/netdev.h
> index c6249c4..33668b4 100644
> --- a/lib/netdev.h
> +++ b/lib/netdev.h
> @@ -162,6 +162,7 @@ const char *netdev_get_type_from_name(const char *);
>  int netdev_get_mtu(const struct netdev *, int *mtup);
>  int netdev_set_mtu(const struct netdev *, int mtu);
>  int netdev_get_ifindex(const struct netdev *);
> +int netdev_set_multiq(struct netdev *, unsigned int n_txq, unsigned int n_rxq);
>
>  /* Packet reception. */
>  int netdev_rxq_open(struct netdev *, struct netdev_rxq **, int id);
> --
> 1.7.9.5
>



More information about the dev mailing list