[ovs-dev] [PATCH] netdev-dpdk: Remove access to DPDK internals.

David Marchand david.marchand at redhat.com
Tue Sep 7 08:23:43 UTC 2021


Instead of dereferencing DPDK ethdev internals, we can list "sibling"
ports [1]: such ports share the underlying rte_device object.

This API was experimental so far, but it will go to stable in 21.11
(see [2]) as it underwent no change since introduction in v19.05.
For the time being, waive the warning for this experimental API.

1: https://git.dpdk.org/dpdk/commit/?id=7f98942886bc
2: https://patchwork.dpdk.org/project/dpdk/list/?series=18705

Signed-off-by: David Marchand <david.marchand at redhat.com>
---
Note: I sent this against the master branch for discussion, but I am
fine with re-sending this against dpdk-latest without the #pragma trick.

---
 lib/netdev-dpdk.c | 95 ++++++++++++++++++++++++++---------------------
 1 file changed, 53 insertions(+), 42 deletions(-)

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index ca92c947a2..2347de7f5b 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -1279,26 +1279,6 @@ common_construct(struct netdev *netdev, dpdk_port_t port_no,
     return 0;
 }
 
-/* Get the number of OVS interfaces which have the same DPDK
- * rte device (e.g. same pci bus address).
- * FIXME: avoid direct access to DPDK internal array rte_eth_devices.
- */
-static int
-netdev_dpdk_get_num_ports(struct rte_device *device)
-    OVS_REQUIRES(dpdk_mutex)
-{
-    struct netdev_dpdk *dev;
-    int count = 0;
-
-    LIST_FOR_EACH (dev, list_node, &dpdk_list) {
-        if (rte_eth_devices[dev->port_id].device == device
-        && rte_eth_devices[dev->port_id].state != RTE_ETH_DEV_UNUSED) {
-            count++;
-        }
-    }
-    return count;
-}
-
 static int
 vhost_common_construct(struct netdev *netdev)
     OVS_REQUIRES(dpdk_mutex)
@@ -1452,8 +1432,6 @@ static void
 netdev_dpdk_destruct(struct netdev *netdev)
 {
     struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
-    struct rte_device *rte_dev;
-    struct rte_eth_dev *eth_dev;
 
     ovs_mutex_lock(&dpdk_mutex);
 
@@ -1461,20 +1439,46 @@ netdev_dpdk_destruct(struct netdev *netdev)
     dev->started = false;
 
     if (dev->attached) {
-        /* Retrieve eth device data before closing it.
-         * FIXME: avoid direct access to DPDK internal array rte_eth_devices.
-         */
-        eth_dev = &rte_eth_devices[dev->port_id];
-        rte_dev = eth_dev->device;
+        bool dpdk_resources_still_used = false;
+        struct rte_eth_dev_info dev_info;
+        dpdk_port_t sibling_port_id;
+
+        /* Check if this netdev has siblings (i.e. shares DPDK resources) among
+         * other OVS netdevs. */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+        RTE_ETH_FOREACH_DEV_SIBLING (sibling_port_id, dev->port_id) {
+#pragma GCC diagnostic pop
+            struct netdev_dpdk *sibling;
+
+            /* RTE_ETH_FOREACH_DEV_SIBLING lists dev->port_id as part of the
+             * loop. */
+            if (sibling_port_id == dev->port_id) {
+                continue;
+            }
+            LIST_FOR_EACH (sibling, list_node, &dpdk_list) {
+                if (sibling->port_id != sibling_port_id) {
+                    continue;
+                }
+                dpdk_resources_still_used = true;
+                break;
+            }
+            if (dpdk_resources_still_used) {
+                break;
+            }
+        }
+
+        /* Retrieve eth device data before closing it. */
+        rte_eth_dev_info_get(dev->port_id, &dev_info);
 
         /* Remove the eth device. */
         rte_eth_dev_close(dev->port_id);
 
-        /* Remove this rte device and all its eth devices if all the eth
-         * devices belonging to the rte device are closed.
-         */
-        if (!netdev_dpdk_get_num_ports(rte_dev)) {
-            int ret = rte_dev_remove(rte_dev);
+        /* Remove the rte device if no associated eth device is used by OVS.
+         * Note: any remaining eth devices associated to this rte device are
+         * closed by DPDK ethdev layer. */
+        if (!dpdk_resources_still_used) {
+            int ret = rte_dev_remove(dev_info.device);
 
             if (ret < 0) {
                 VLOG_ERR("Device '%s' can not be detached: %s.",
@@ -3793,12 +3797,12 @@ static void
 netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
                    const char *argv[], void *aux OVS_UNUSED)
 {
-    char *response;
-    dpdk_port_t port_id;
-    struct netdev_dpdk *dev;
-    struct rte_device *rte_dev;
     struct ds used_interfaces = DS_EMPTY_INITIALIZER;
+    struct rte_eth_dev_info dev_info;
+    dpdk_port_t sibling_port_id;
+    dpdk_port_t port_id;
     bool used = false;
+    char *response;
 
     ovs_mutex_lock(&dpdk_mutex);
 
@@ -3808,18 +3812,24 @@ netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
         goto error;
     }
 
-    rte_dev = rte_eth_devices[port_id].device;
     ds_put_format(&used_interfaces,
                   "Device '%s' is being used by the following interfaces:",
                   argv[1]);
 
-    LIST_FOR_EACH (dev, list_node, &dpdk_list) {
-        /* FIXME: avoid direct access to DPDK array rte_eth_devices. */
-        if (rte_eth_devices[dev->port_id].device == rte_dev
-            && rte_eth_devices[dev->port_id].state != RTE_ETH_DEV_UNUSED) {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+    RTE_ETH_FOREACH_DEV_SIBLING (sibling_port_id, port_id) {
+#pragma GCC diagnostic pop
+        struct netdev_dpdk *dev;
+
+        LIST_FOR_EACH (dev, list_node, &dpdk_list) {
+            if (dev->port_id != sibling_port_id) {
+                continue;
+            }
             used = true;
             ds_put_format(&used_interfaces, " %s",
                           netdev_get_name(&dev->up));
+            break;
         }
     }
 
@@ -3831,8 +3841,9 @@ netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
     }
     ds_destroy(&used_interfaces);
 
+    rte_eth_dev_info_get(port_id, &dev_info);
     rte_eth_dev_close(port_id);
-    if (rte_dev_remove(rte_dev) < 0) {
+    if (rte_dev_remove(dev_info.device) < 0) {
         response = xasprintf("Device '%s' can not be detached", argv[1]);
         goto error;
     }
-- 
2.23.0



More information about the dev mailing list