[ovs-dev] [PATCH] OVS-DPDK: Change "dpdk-socket-mem" default value.

[Billy O'Mahony]

From: Marcin Rybka <marcinx.rybka at intel.com>

When "dpdk-socket-mem" and "dpdk-alloc-mem" are not specified,
"dpdk-socket-mem" will be set to allocate 1024MB on each NUMA node.
This change will prevent OVS from failing when NIC is attached on
NUMA node 1 and higher. Patch contains documentation update.

Signed-off-by: Marcin Rybka <marcinx.rybka at intel.com>
Signed-off-by: Billy O'Mahony <billy.o.mahony at intel.com>
---
 Documentation/intro/install/dpdk.rst |  3 ++-
 lib/dpdk.c                           | 35 ++++++++++++++++++++++++++++++++++-
 vswitchd/vswitch.xml                 |  7 ++++---
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/Documentation/intro/install/dpdk.rst b/Documentation/intro/install/dpdk.rst
index fea4890..b68438d 100644
--- a/Documentation/intro/install/dpdk.rst
+++ b/Documentation/intro/install/dpdk.rst
@@ -228,7 +228,8 @@ listed below. Defaults will be provided for all values not explicitly set.
 
 ``dpdk-socket-mem``
   Comma separated list of memory to pre-allocate from hugepages on specific
-  sockets.
+  sockets. If not specified, 1024 MB will be set for each numa node by
+  default.
 
 ``dpdk-hugepage-dir``
   Directory where hugetlbfs is mounted
diff --git a/lib/dpdk.c b/lib/dpdk.c
index 00dd974..125178e 100644
--- a/lib/dpdk.c
+++ b/lib/dpdk.c
@@ -35,6 +35,7 @@
 #include "netdev-dpdk.h"
 #include "openvswitch/dynamic-string.h"
 #include "openvswitch/vlog.h"
+#include "ovs-numa.h"
 #include "smap.h"
 
 VLOG_DEFINE_THIS_MODULE(dpdk);
@@ -163,6 +164,29 @@ construct_dpdk_options(const struct smap *ovs_other_config,
     return ret;
 }
 
+static char *
+construct_dpdk_socket_mem(void)
+{
+    int numa = 0;
+    const char *def_value = "1024";
+    int numa_nodes = ovs_numa_get_n_numas();
+
+    if (numa_nodes == 0 || numa_nodes == OVS_NUMA_UNSPEC) {
+        numa_nodes = 1;
+    }
+
+    /* Allocate enough memory for digits, comma-sep and terminator. */
+    char *dpdk_socket_mem = xzalloc(numa_nodes * (strlen(def_value) + 1));
+
+    strcat(dpdk_socket_mem, def_value);
+    for (numa = 1; numa < numa_nodes; ++numa) {
+        strcat(dpdk_socket_mem, ",");
+        strcat(dpdk_socket_mem, def_value);
+    }
+
+    return dpdk_socket_mem;
+}
+
 #define MAX_DPDK_EXCL_OPTS 10
 
 static int
@@ -170,6 +194,7 @@ construct_dpdk_mutex_options(const struct smap *ovs_other_config,
                              char ***argv, const int initial_size,
                              char **extra_args, const size_t extra_argc)
 {
+    char *default_dpdk_socket_mem = construct_dpdk_socket_mem();
     struct dpdk_exclusive_options_map {
         const char *category;
         const char *ovs_dpdk_options[MAX_DPDK_EXCL_OPTS];
@@ -180,7 +205,7 @@ construct_dpdk_mutex_options(const struct smap *ovs_other_config,
         {"memory type",
          {"dpdk-alloc-mem", "dpdk-socket-mem", NULL,},
          {"-m",             "--socket-mem",    NULL,},
-         "1024,0", 1
+         default_dpdk_socket_mem, 1
         },
     };
 
@@ -227,6 +252,8 @@ construct_dpdk_mutex_options(const struct smap *ovs_other_config,
         }
     }
 
+    free(default_dpdk_socket_mem);
+
     return ret;
 }
 
@@ -420,6 +447,12 @@ dpdk_init__(const struct smap *ovs_other_config)
         argv_to_release[argc_tmp] = argv[argc_tmp];
     }
 
+    {
+        int i;
+        for (i = 0; i <= argc; i++) {
+            VLOG_WARN("BOM %d '%s'", argc, argv[i]);
+        }
+    }
     /* Make sure things are initialized ... */
     result = rte_eal_init(argc, argv);
     if (result < 0) {
diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index 9c2a826..d74f28b 100644
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -302,9 +302,10 @@
         </p>
         <p>
           If dpdk-socket-mem and dpdk-alloc-mem are not specified, dpdk-socket-mem
-          will be used and the default value is 1024,0. If dpdk-socket-mem and
-          dpdk-alloc-mem are specified at same time, dpdk-socket-mem will be
-          used as default. Changing this value requires restarting the daemon.
+          will be used and the default value is 1024 for each numa node. If
+          dpdk-socket-mem and dpdk-alloc-mem are specified at same time,
+          dpdk-socket-mem will be used as default. Changing this value
+          requires restarting the daemon.
         </p>
       </column>
 
-- 
2.7.4