[ovs-dev] [PATCH v2] netdev-dpdk: Allow configurable queue sizes for 'dpdk' ports

Ciara Loftus ciara.loftus at intel.com
Tue Sep 20 08:28:08 UTC 2016


The other_config:dpdk-rxq-size and dpdk-txq-size fields allow for an
integer between 1 and 4096 that reflects the number of rx/tx descriptors
to initialise 'dpdk' devices with. If no value is specified, they
default to 2048. 'dpdk-*xq-size' fields must be set before launching the
daemon as changing the queue size requires the NIC to restart.

Signed-off-by: Ciara Loftus <ciara.loftus at intel.com>
---
v2
* Rebase

 INSTALL.DPDK-ADVANCED.md | 16 ++++++++++++++--
 NEWS                     |  3 +++
 lib/netdev-dpdk.c        | 35 +++++++++++++++++++++++++++++++----
 vswitchd/vswitch.xml     | 26 ++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/INSTALL.DPDK-ADVANCED.md b/INSTALL.DPDK-ADVANCED.md
index d7b9873..08aba81 100644
--- a/INSTALL.DPDK-ADVANCED.md
+++ b/INSTALL.DPDK-ADVANCED.md
@@ -257,7 +257,19 @@ needs to be affinitized accordingly.
   The rx queues are assigned to pmd threads on the same NUMA node in a
   round-robin fashion.
 
-### 4.4 Exact Match Cache
+### 4.4 DPDK Physical Port Queue Sizes
+  `ovs-vsctl set Open_vSwitch . other_config:dpdk-rxq-size=<n-rxq-desc>`
+  `ovs-vsctl set Open_vSwitch . other_config:dpdk-txq-size=<n-txq-desc>`
+
+  The command above sets the number of rx/tx descriptors that the NICs
+  associated with 'dpdk' ports will be initialised with.
+
+  Different 'dpdk-rxq-size' and 'dpdk-txq-size' configurations yield different
+  benefits in terms of throughput and latency for different scenarios.
+  Generally, smaller queue sizes can have a positive impact for latency at the
+  expense of throughput. The opposite is often true for larger queue sizes.
+
+### 4.5 Exact Match Cache
 
   Each pmd thread contains one EMC. After initial flow setup in the
   datapath, the EMC contains a single table and provides the lowest level
@@ -274,7 +286,7 @@ needs to be affinitized accordingly.
   avoiding datapath classifier lookups is to have multiple pmd threads
   running. This can be done as described in section 4.2.
 
-### 4.5 Rx Mergeable buffers
+### 4.6 Rx Mergeable buffers
 
   Rx Mergeable buffers is a virtio feature that allows chaining of multiple
   virtio descriptors to handle large packet sizes. As such, large packets
diff --git a/NEWS b/NEWS
index 21ab538..2d28bd3 100644
--- a/NEWS
+++ b/NEWS
@@ -125,6 +125,9 @@ v2.6.0 - xx xxx xxxx
      * Remove dpdkvhostcuse port type.
      * OVS client mode for vHost and vHost reconnect (Requires QEMU 2.7)
      * 'dpdkvhostuserclient' port type.
+     * New 'other_config:dpdk-rxq-size' and 'other_config:dpdk-txq-size' fields
+       that specify the number of rxq and txq descriptors to initialise DPDK
+       NICs with.
    - Increase number of registers to 16.
    - ovs-benchmark: This utility has been removed due to lack of use and
      bitrot.
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 89bdc4d..fdda313 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -132,8 +132,9 @@ BUILD_ASSERT_DECL((MAX_NB_MBUF / ROUND_DOWN_POW2(MAX_NB_MBUF/MIN_NB_MBUF))
 
 #define SOCKET0              0
 
-#define NIC_PORT_RX_Q_SIZE 2048  /* Size of Physical NIC RX Queue, Max (n+32<=4096)*/
-#define NIC_PORT_TX_Q_SIZE 2048  /* Size of Physical NIC TX Queue, Max (n+32<=4096)*/
+#define NIC_PORT_DEFAULT_RXQ_SIZE 2048 /* Default size of Physical NIC RXQ */
+#define NIC_PORT_DEFAULT_TXQ_SIZE 2048 /* Default size of Physical NIC TXQ */
+#define NIC_PORT_MAX_Q_SIZE 4096       /* Maximum size of Physical NIC Queue */
 
 #define OVS_VHOST_MAX_QUEUE_NUM 1024  /* Maximum number of vHost TX queues. */
 #define OVS_VHOST_QUEUE_MAP_UNKNOWN (-1) /* Mapping not initialized. */
@@ -142,6 +143,9 @@ BUILD_ASSERT_DECL((MAX_NB_MBUF / ROUND_DOWN_POW2(MAX_NB_MBUF/MIN_NB_MBUF))
 
 static char *vhost_sock_dir = NULL;   /* Location of vhost-user sockets */
 
+static int dpdk_rxq_size = 0;    /* Configured size of Physical NIC RX Queue */
+static int dpdk_txq_size = 0;    /* Configured size of Physical NIC TX Queue */
+
 #define VHOST_ENQ_RETRY_NUM 8
 #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
 
@@ -646,7 +650,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int n_rxq, int n_txq)
         }
 
         for (i = 0; i < n_txq; i++) {
-            diag = rte_eth_tx_queue_setup(dev->port_id, i, NIC_PORT_TX_Q_SIZE,
+            diag = rte_eth_tx_queue_setup(dev->port_id, i, dpdk_txq_size,
                                           dev->socket_id, NULL);
             if (diag) {
                 VLOG_INFO("Interface %s txq(%d) setup error: %s",
@@ -662,7 +666,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int n_rxq, int n_txq)
         }
 
         for (i = 0; i < n_rxq; i++) {
-            diag = rte_eth_rx_queue_setup(dev->port_id, i, NIC_PORT_RX_Q_SIZE,
+            diag = rte_eth_rx_queue_setup(dev->port_id, i, dpdk_rxq_size,
                                           dev->socket_id, NULL,
                                           dev->dpdk_mp->mp);
             if (diag) {
@@ -3132,6 +3136,23 @@ process_vhost_flags(char *flag, char *default_val, int size,
     return changed;
 }
 
+static void
+process_queue_size_flags(const struct smap *ovs_other_config, char *flag,
+                         int default_size, int *new_size)
+{
+    int queue_size;
+
+    queue_size = smap_get_int(ovs_other_config, flag, 0);
+    if (queue_size > 0 && queue_size <= NIC_PORT_MAX_Q_SIZE) {
+        *new_size = queue_size;
+    } else {
+        *new_size = default_size;
+    }
+
+    VLOG_INFO("'dpdk' ports will be configured with a %s of %i",
+              flag, *new_size);
+}
+
 static char **
 grow_argv(char ***argv, size_t cur_siz, size_t grow_by)
 {
@@ -3374,6 +3395,12 @@ dpdk_init__(const struct smap *ovs_other_config)
         vhost_sock_dir = sock_dir_subcomponent;
     }
 
+    /* Determine the queue sizes to specify when initializing 'dpdk' ports */
+    process_queue_size_flags(ovs_other_config, "dpdk-rxq-size",
+                             NIC_PORT_DEFAULT_RXQ_SIZE, &dpdk_rxq_size);
+    process_queue_size_flags(ovs_other_config, "dpdk-txq-size",
+                             NIC_PORT_DEFAULT_TXQ_SIZE, &dpdk_txq_size);
+
     argv = grow_argv(&argv, 0, 1);
     argc = 1;
     argv[0] = xstrdup(ovs_get_program_name());
diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index e73023d..2ffcbdf 100644
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -299,6 +299,32 @@
         </p>
       </column>
 
+      <column name="other_config" key="dpdk-rxq-size"
+              type='{"type": "integer", "minInteger": 1, "maxInteger": 4096}'>
+        <p>
+          Specifies the queue size (number rx descriptors) of dpdk ports.
+          Ensure that your NIC(s) can support the particular value before
+          modifying 'dpdk-rxq-size'.
+        </p>
+        <p>
+          Defaults to 2048. Maximum value is 4096. Changing this value requires
+          restarting the daemon.
+        </p>
+      </column>
+
+      <column name="other_config" key="dpdk-txq-size"
+              type='{"type": "integer", "minInteger": 1, "maxInteger": 4096}'>
+        <p>
+          Specifies the queue size (number tx descriptors) of dpdk ports.
+          Ensure that your NIC(s) can support the particular value before
+          modifying 'dpdk-txq-size'.
+        </p>
+        <p>
+          Defaults to 2048. Maximum value is 4096. Changing this value requires
+          restarting the daemon.
+        </p>
+      </column>
+
       <column name="other_config" key="n-handler-threads"
               type='{"type": "integer", "minInteger": 1}'>
         <p>
-- 
2.4.3




More information about the dev mailing list