[ovs-dev] [PATCH] Detailed packet drop statistics per dpdk and vhostuser ports

Sriram Vatala sriram.v at altencalsoftlabs.com
Fri Jun 7 12:39:21 UTC 2019


OVS may be unable to transmit packets for multiple reasons and
today there is a single counter to track packets dropped due to
any of those reasons. The most common reason is that a VM is
unable to read packets fast enough causing the vhostuser port
transmit queue on the OVS side to become full. This manifests
as a problem with VNFs not receiving all packets. Having a
separate drop counter to track packets dropped because the
transmit queue is full will clearly indicate that the problem
is on the VM side and not in OVS. Similarly maintaining separate
counters for all possible drops helps in indicating sensible
cause for packet drops.

This patch adds counters to track packets dropped due to all possible
reasons and display them when "--details" optional flag is passed  to
“ovs-appctl dpctl/show -s” . The detailed stats will be available
for both dpdk and vhostuser ports.

cmd Usage : "ovs-appctl dpctl/show -s --details" (OR)
            "ovs-appctl dpctl/show --statistics --details"

Following are the details of the new counters :

queue_full : These are the packets dropped due to Transmit queue overrun.
mtu_exceeded : These are the packets dropped due to MTU mismatch.
               (i.e Pkt len > Max Dev MTU)
qos : These are the packets dropped due to transmission/reception rate
      exceeding the configured Egress/Ingress policer rate on the interface.

Signed-off-by: Sriram Vatala <sriram.v at altencalsoftlabs.com>
---
 include/openvswitch/netdev.h |  8 ++++++++
 lib/dpctl.c                  | 26 ++++++++++++++++++++++-
 lib/dpctl.h                  |  5 +++++
 lib/dpctl.man                |  8 ++++++--
 lib/netdev-dpdk.c            | 49 +++++++++++++++++++++++++++++++++++++-------
 5 files changed, 86 insertions(+), 10 deletions(-)

diff --git a/include/openvswitch/netdev.h b/include/openvswitch/netdev.h
index 0c10f7b..69480a4 100644
--- a/include/openvswitch/netdev.h
+++ b/include/openvswitch/netdev.h
@@ -61,6 +61,14 @@ struct netdev_stats {
     uint64_t tx_heartbeat_errors;
     uint64_t tx_window_errors;
 
+    /* Detailed receive drops. */
+    uint64_t rx_qos_drops;
+
+    /* Detailed transmit drops. */
+    uint64_t tx_qfull_drops;
+    uint64_t tx_qos_drops;
+    uint64_t tx_mtu_drops;
+
     /* Extended statistics based on RFC2819. */
     uint64_t rx_1_to_64_packets;
     uint64_t rx_65_to_127_packets;
diff --git a/lib/dpctl.c b/lib/dpctl.c
index 9c4eb65..cdbf740 100644
--- a/lib/dpctl.c
+++ b/lib/dpctl.c
@@ -38,6 +38,7 @@
 #include "flow.h"
 #include "openvswitch/match.h"
 #include "netdev.h"
+#include "netdev-provider.h"
 #include "netlink.h"
 #include "odp-util.h"
 #include "openvswitch/ofpbuf.h"
@@ -683,6 +684,12 @@ show_dpif(struct dpif *dpif, struct dpctl_params *dpctl_p)
                 print_stat(dpctl_p, "    RX packets:", s.rx_packets);
                 print_stat(dpctl_p, " errors:", s.rx_errors);
                 print_stat(dpctl_p, " dropped:", s.rx_dropped);
+                if (dpctl_p->print_detailed_stats &&
+                    netdev->netdev_class->is_pmd) {
+                    dpctl_print(dpctl_p, " (");
+                    print_stat(dpctl_p, "qos:", s.rx_qos_drops);
+                    dpctl_print(dpctl_p, ") ");
+                }
                 print_stat(dpctl_p, " overruns:", s.rx_over_errors);
                 print_stat(dpctl_p, " frame:", s.rx_frame_errors);
                 dpctl_print(dpctl_p, "\n");
@@ -690,6 +697,14 @@ show_dpif(struct dpif *dpif, struct dpctl_params *dpctl_p)
                 print_stat(dpctl_p, "    TX packets:", s.tx_packets);
                 print_stat(dpctl_p, " errors:", s.tx_errors);
                 print_stat(dpctl_p, " dropped:", s.tx_dropped);
+                if (dpctl_p->print_detailed_stats &&
+                    netdev->netdev_class->is_pmd) {
+                    dpctl_print(dpctl_p, " (");
+                    print_stat(dpctl_p, "queue_full:", s.tx_qfull_drops);
+                    print_stat(dpctl_p, " mtu_exceeded:", s.tx_mtu_drops);
+                    print_stat(dpctl_p, " qos:", s.tx_qos_drops);
+                    dpctl_print(dpctl_p, ") ");
+                }
                 print_stat(dpctl_p, " aborted:", s.tx_aborted_errors);
                 print_stat(dpctl_p, " carrier:", s.tx_carrier_errors);
                 dpctl_print(dpctl_p, "\n");
@@ -2414,7 +2429,8 @@ static const struct dpctl_command all_commands[] = {
     { "del-if", "dp iface...", 2, INT_MAX, dpctl_del_if, DP_RW },
     { "set-if", "dp iface...", 2, INT_MAX, dpctl_set_if, DP_RW },
     { "dump-dps", "", 0, 0, dpctl_dump_dps, DP_RO },
-    { "show", "[dp...]", 0, INT_MAX, dpctl_show, DP_RO },
+    { "show", "[-s | --statistics [--details]] [dp...]", 0, \
+      INT_MAX, dpctl_show, DP_RO },
     { "dump-flows", "[dp] [filter=..] [type=..]",
       0, 3, dpctl_dump_flows, DP_RO },
     { "add-flow", "[dp] flow actions", 2, 3, dpctl_add_flow, DP_RW },
@@ -2545,6 +2561,14 @@ dpctl_unixctl_handler(struct unixctl_conn *conn, int argc, const char *argv[],
             } else if (!strcmp(arg, "--no-names")) {
                 dpctl_p.names = false;
                 set_names = true;
+            } else if (!strcmp(arg, "--details")) {
+                if (dpctl_p.print_statistics) {
+                    dpctl_p.print_detailed_stats = true;
+                } else {
+                    ds_put_format(&ds, "--details should be "
+                                  "preceded by --statistics");
+                    error = true;
+                }
             } else {
                 ds_put_format(&ds, "Unrecognized option %s", argv[1]);
                 error = true;
diff --git a/lib/dpctl.h b/lib/dpctl.h
index 9d00521..1d3720b 100644
--- a/lib/dpctl.h
+++ b/lib/dpctl.h
@@ -27,6 +27,11 @@ struct dpctl_params {
     /* -s, --statistics: Print port/flow statistics? */
     bool print_statistics;
 
+    /* --details: Print detailed port/flow statistics?
+     * optional arument for --statistics
+     */
+    bool print_detailed_stats;
+
     /* --clear: Reset existing statistics to zero when modifying a flow? */
     bool zero_statistics;
 
diff --git a/lib/dpctl.man b/lib/dpctl.man
index 1ff3511..ab7a66a 100644
--- a/lib/dpctl.man
+++ b/lib/dpctl.man
@@ -58,12 +58,16 @@ Removes each \fInetdev\fR from the list of network devices datapath
 Prints the name of each configured datapath on a separate line.
 .
 .TP
-.DO "[\fB\-s\fR | \fB\-\-statistics\fR]" "\*(DX\fBshow" "\fR[\fIdp\fR...]"
+.DO "[\fB\-s\fR | \fB\-\-statistics\fR [\fB\-\-details\fR]]" \
+"\*(DX\fBshow" "\fR[\fIdp\fR...]"
 Prints a summary of configured datapaths, including their datapath
 numbers and a list of ports connected to each datapath.  (The local
 port is identified as port 0.)  If \fB\-s\fR or \fB\-\-statistics\fR
 is specified, then packet and byte counters are also printed for each
-port.
+port. When dpctl is invoked with ovs-appctl (i.e for netdev datapath),
+if \fB\-\-details\fR is specified after \fB\-s\fR or \fB\-\-statistics\fR,
+then details of packet drop counters are also printed. The detailed
+stats will be available for both dpdk and vhostuser ports.
 .IP
 The datapath numbers consists of flow stats and mega flow mask stats.
 .IP
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 3498b32..6c2eb38 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -2124,6 +2124,7 @@ netdev_dpdk_vhost_update_rx_counters(struct netdev_stats *stats,
 
     stats->rx_packets += count;
     stats->rx_dropped += dropped;
+    stats->rx_qos_drops += dropped;
     for (i = 0; i < count; i++) {
         packet = packets[i];
         packet_size = dp_packet_size(packet);
@@ -2236,6 +2237,7 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq, struct dp_packet_batch *batch,
     if (OVS_UNLIKELY(dropped)) {
         rte_spinlock_lock(&dev->stats_lock);
         dev->stats.rx_dropped += dropped;
+        dev->stats.rx_qos_drops += dropped;
         rte_spinlock_unlock(&dev->stats_lock);
     }
 
@@ -2319,6 +2321,9 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
     struct rte_mbuf **cur_pkts = (struct rte_mbuf **) pkts;
     unsigned int total_pkts = cnt;
     unsigned int dropped = 0;
+    unsigned int mtu_drops;
+    unsigned int qos_drops;
+    unsigned int qfull_drops;
     int i, retries = 0;
     int vid = netdev_dpdk_get_vid(dev);
 
@@ -2335,9 +2340,11 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
     rte_spinlock_lock(&dev->tx_q[qid].tx_lock);
 
     cnt = netdev_dpdk_filter_packet_len(dev, cur_pkts, cnt);
+    mtu_drops = total_pkts - cnt;
+    qos_drops = cnt;
     /* Check has QoS has been configured for the netdev */
     cnt = netdev_dpdk_qos_run(dev, cur_pkts, cnt, true);
-    dropped = total_pkts - cnt;
+    qos_drops -= cnt;
 
     do {
         int vhost_qid = qid * VIRTIO_QNUM + VIRTIO_RXQ;
@@ -2357,9 +2364,14 @@ __netdev_dpdk_vhost_send(struct netdev *netdev, int qid,
 
     rte_spinlock_unlock(&dev->tx_q[qid].tx_lock);
 
+    qfull_drops = cnt;
+    dropped = mtu_drops + qos_drops + qfull_drops;
     rte_spinlock_lock(&dev->stats_lock);
     netdev_dpdk_vhost_update_tx_counters(&dev->stats, pkts, total_pkts,
-                                         cnt + dropped);
+                                         dropped);
+    dev->stats.tx_mtu_drops += mtu_drops;
+    dev->stats.tx_qos_drops += qos_drops;
+    dev->stats.tx_qfull_drops += qfull_drops;
     rte_spinlock_unlock(&dev->stats_lock);
 
 out:
@@ -2384,12 +2396,15 @@ dpdk_do_tx_copy(struct netdev *netdev, int qid, struct dp_packet_batch *batch)
     struct rte_mbuf *pkts[PKT_ARRAY_SIZE];
     uint32_t cnt = batch_cnt;
     uint32_t dropped = 0;
+    uint32_t mtu_drops = 0;
+    uint32_t qos_drops = 0;
+    uint32_t qfull_drops = 0;
 
     if (dev->type != DPDK_DEV_VHOST) {
         /* Check if QoS has been configured for this netdev. */
         cnt = netdev_dpdk_qos_run(dev, (struct rte_mbuf **) batch->packets,
                                   batch_cnt, false);
-        dropped += batch_cnt - cnt;
+        qos_drops = batch_cnt - cnt;
     }
 
     uint32_t txcnt = 0;
@@ -2402,7 +2417,7 @@ dpdk_do_tx_copy(struct netdev *netdev, int qid, struct dp_packet_batch *batch)
             VLOG_WARN_RL(&rl, "Too big size %u max_packet_len %d",
                          size, dev->max_packet_len);
 
-            dropped++;
+            mtu_drops++;
             continue;
         }
 
@@ -2425,13 +2440,17 @@ dpdk_do_tx_copy(struct netdev *netdev, int qid, struct dp_packet_batch *batch)
             __netdev_dpdk_vhost_send(netdev, qid, (struct dp_packet **) pkts,
                                      txcnt);
         } else {
-            dropped += netdev_dpdk_eth_tx_burst(dev, qid, pkts, txcnt);
+            qfull_drops = netdev_dpdk_eth_tx_burst(dev, qid, pkts, txcnt);
         }
     }
 
+    dropped += mtu_drops + qos_drops + qfull_drops;
     if (OVS_UNLIKELY(dropped)) {
         rte_spinlock_lock(&dev->stats_lock);
         dev->stats.tx_dropped += dropped;
+        dev->stats.tx_mtu_drops += mtu_drops;
+        dev->stats.tx_qos_drops += qos_drops;
+        dev->stats.tx_qfull_drops += qfull_drops;
         rte_spinlock_unlock(&dev->stats_lock);
     }
 }
@@ -2473,18 +2492,25 @@ netdev_dpdk_send__(struct netdev_dpdk *dev, int qid,
         dp_packet_delete_batch(batch, true);
     } else {
         int tx_cnt, dropped;
+        int mtu_drops, qfull_drops, qos_drops;
         int batch_cnt = dp_packet_batch_size(batch);
         struct rte_mbuf **pkts = (struct rte_mbuf **) batch->packets;
 
         tx_cnt = netdev_dpdk_filter_packet_len(dev, pkts, batch_cnt);
+        mtu_drops = batch_cnt - tx_cnt;
+        qos_drops = tx_cnt;
         tx_cnt = netdev_dpdk_qos_run(dev, pkts, tx_cnt, true);
-        dropped = batch_cnt - tx_cnt;
+        qos_drops -= tx_cnt;
 
-        dropped += netdev_dpdk_eth_tx_burst(dev, qid, pkts, tx_cnt);
+        qfull_drops = netdev_dpdk_eth_tx_burst(dev, qid, pkts, tx_cnt);
 
+        dropped = mtu_drops + qos_drops + qfull_drops;
         if (OVS_UNLIKELY(dropped)) {
             rte_spinlock_lock(&dev->stats_lock);
             dev->stats.tx_dropped += dropped;
+            dev->stats.tx_mtu_drops += mtu_drops;
+            dev->stats.tx_qos_drops += qos_drops;
+            dev->stats.tx_qfull_drops += qfull_drops;
             rte_spinlock_unlock(&dev->stats_lock);
         }
     }
@@ -2598,6 +2624,11 @@ netdev_dpdk_vhost_get_stats(const struct netdev *netdev,
     stats->rx_errors = dev->stats.rx_errors;
     stats->rx_length_errors = dev->stats.rx_length_errors;
 
+    stats->tx_mtu_drops = dev->stats.tx_mtu_drops;
+    stats->tx_qos_drops = dev->stats.tx_qos_drops;
+    stats->tx_qfull_drops = dev->stats.tx_qfull_drops;
+    stats->rx_qos_drops = dev->stats.rx_qos_drops;
+
     stats->rx_1_to_64_packets = dev->stats.rx_1_to_64_packets;
     stats->rx_65_to_127_packets = dev->stats.rx_65_to_127_packets;
     stats->rx_128_to_255_packets = dev->stats.rx_128_to_255_packets;
@@ -2733,6 +2764,10 @@ out:
     rte_spinlock_lock(&dev->stats_lock);
     stats->tx_dropped = dev->stats.tx_dropped;
     stats->rx_dropped = dev->stats.rx_dropped;
+    stats->tx_mtu_drops = dev->stats.tx_mtu_drops;
+    stats->tx_qos_drops = dev->stats.tx_qos_drops;
+    stats->tx_qfull_drops = dev->stats.tx_qfull_drops;
+    stats->rx_qos_drops = dev->stats.rx_qos_drops;
     rte_spinlock_unlock(&dev->stats_lock);
 
     /* These are the available DPDK counters for packets not received due to
-- 
2.7.4



More information about the dev mailing list