[ovs-dev] [PATCH 1/2] dpdk: support non-contiguous NUMA nodes for IBM POWER systems

David Christensen drc at linux.vnet.ibm.com
Wed May 12 16:27:54 UTC 2021


NUMA nodes are not guaranteed to be continguous on IBM POWER systems,
nor are all NUMA nodes guaranteed to have CPUs attached to them.  For
example:

$ lscpu
Architecture:        ppc64le
Byte Order:          Little Endian
CPU(s):              128
On-line CPU(s) list: 0-127
Thread(s) per core:  4
Core(s) per socket:  16
Socket(s):           2
NUMA node(s):        6
Model:               2.3 (pvr 004e 1203)
Model name:          POWER9, altivec supported
CPU max MHz:         3800.0000
CPU min MHz:         2300.0000
L1d cache:           32K
L1i cache:           32K
L2 cache:            512K
L3 cache:            10240K
NUMA node0 CPU(s):   0-63
NUMA node8 CPU(s):   64-127
NUMA node252 CPU(s):
NUMA node253 CPU(s):
NUMA node254 CPU(s):
NUMA node255 CPU(s):

Modify NUMA node detection logic to allow non-contiguous NUMA nodes.

Signed-off-by: David Christensen <drc at linux.vnet.ibm.com>
---
 lib/dpdk.c     | 27 ++++++++++++++++++---------
 lib/ovs-numa.c |  4 +---
 lib/ovs-numa.h |  1 +
 3 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/lib/dpdk.c b/lib/dpdk.c
index 319540394..dd1816427 100644
--- a/lib/dpdk.c
+++ b/lib/dpdk.c
@@ -133,18 +133,27 @@ static char *
 construct_dpdk_socket_mem(void)
 {
     const char *def_value = "1024";
-    int numa, numa_nodes = ovs_numa_get_n_numas();
-    struct ds dpdk_socket_mem = DS_EMPTY_INITIALIZER;
-
-    if (numa_nodes == 0 || numa_nodes == OVS_NUMA_UNSPEC) {
-        numa_nodes = 1;
-    }
+    struct ovs_numa_dump *dump;
+    int last_node = 0;
 
+    /* Build a list of all numa nodes with at least one core */
+    dump = ovs_numa_dump_n_cores_per_numa(1);
+    struct ds dpdk_socket_mem = DS_EMPTY_INITIALIZER;
     ds_put_cstr(&dpdk_socket_mem, def_value);
-    for (numa = 1; numa < numa_nodes; ++numa) {
-        ds_put_format(&dpdk_socket_mem, ",%s", def_value);
-    }
 
+    const struct ovs_numa_info_numa *node;
+    FOR_EACH_NUMA_ON_DUMP(node,dump) {
+        while (node->numa_id > last_node+1 &&
+               node->numa_id != OVS_NUMA_UNSPEC &&
+               node->numa_id <= MAX_NUMA_NODES){
+                ds_put_format(&dpdk_socket_mem, ",%s", "0");
+                ++last_node;
+        }
+        if (node->numa_id != 0) {
+                ds_put_format(&dpdk_socket_mem, ",%s", def_value);
+        }
+    }
+    ovs_numa_dump_destroy(dump);
     return ds_cstr(&dpdk_socket_mem);
 }
 
diff --git a/lib/ovs-numa.c b/lib/ovs-numa.c
index 6d0a68522..6f85d7023 100644
--- a/lib/ovs-numa.c
+++ b/lib/ovs-numa.c
@@ -56,8 +56,6 @@ VLOG_DEFINE_THIS_MODULE(ovs_numa);
  * TODO: Fix ovs-numa when cpu hotplug is used.
  */
 
-#define MAX_NUMA_NODES 128
-
 /* numa node. */
 struct numa_node {
     struct hmap_node hmap_node;     /* In the 'all_numa_nodes'. */
@@ -229,7 +227,7 @@ discover_numa_and_core(void)
         }
 
         free(path);
-        if (!dir || !numa_supported) {
+        if (!numa_supported) {
             break;
         }
     }
diff --git a/lib/ovs-numa.h b/lib/ovs-numa.h
index 8f2ea3430..7d4b5af4c 100644
--- a/lib/ovs-numa.h
+++ b/lib/ovs-numa.h
@@ -25,6 +25,7 @@
 
 #define OVS_CORE_UNSPEC INT_MAX
 #define OVS_NUMA_UNSPEC INT_MAX
+#define MAX_NUMA_NODES 128
 
 /* Dump of a list of 'struct ovs_numa_info'. */
 struct ovs_numa_dump {
-- 
2.27.0



More information about the dev mailing list