[ovs-dev] [PATCH] ovs-numa: Add ovs-numa.{c, h} for extracting and storing cpu socket and cpu core info.

Alex Wang alexw at nicira.com
Wed Aug 13 19:04:19 UTC 2014


Signed-off-by: Alex Wang <alexw at nicira.com>

---
Changes since first post:
- use readdir() for the buffer safety.
- only compile ovs-numa.c in linux.
- make ovs_numa_get_n_sockets/cores() return *UNSPEC if there
  is no sockets or cores extracted.
- refine code based on Ben's review.
- Address review comments from Thomas Graf.
- Add dummy interface for WIN32 case.
---
 lib/automake.mk         |    2 +
 lib/ovs-numa.c          |  289 +++++++++++++++++++++++++++++++++++++++++++++++
 lib/ovs-numa.h          |  111 ++++++++++++++++++
 tests/ofproto-macros.at |    1 +
 vswitchd/bridge.c       |    2 +
 5 files changed, 405 insertions(+)
 create mode 100644 lib/ovs-numa.c
 create mode 100644 lib/ovs-numa.h

diff --git a/lib/automake.mk b/lib/automake.mk
index e95191d..d46613f 100644
--- a/lib/automake.mk
+++ b/lib/automake.mk
@@ -310,6 +310,8 @@ lib_libopenvswitch_la_SOURCES += \
 	lib/netlink-protocol.h \
 	lib/netlink-socket.c \
 	lib/netlink-socket.h \
+	lib/ovs-numa.c \
+	lib/ovs-numa.h \
 	lib/rtnetlink-link.c \
 	lib/rtnetlink-link.h \
 	lib/route-table.c \
diff --git a/lib/ovs-numa.c b/lib/ovs-numa.c
new file mode 100644
index 0000000..bcf9b68
--- /dev/null
+++ b/lib/ovs-numa.c
@@ -0,0 +1,289 @@
+/*
+ * Copyright (c) 2014 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* On non-Linux, these functions are defined inline in ovs-numa.h. */
+#ifdef __linux__
+
+#include <config.h>
+#include "ovs-numa.h"
+
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stddef.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "hash.h"
+#include "hmap.h"
+#include "list.h"
+#include "ovs-thread.h"
+#include "vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(ovs_numa);
+
+#define MAX_CPU_SOCKETS 128
+
+/* Cpu socket. */
+struct cpu_socket {
+    struct hmap_node hmap_node;     /* In the 'all_cpu_sockets'. */
+    struct list cores;              /* List of cpu cores on the socket. */
+    int socket_id;                  /* Socket id. */
+};
+
+/* Cpu core on a cpu socket. */
+struct cpu_core {
+    struct hmap_node hmap_node;/* In the 'all_cpu_cores'. */
+    struct list list_node;     /* In 'cpu_socket->cores' list. */
+    struct cpu_socket *socket; /* Socket containing the core. */
+    int core_id;               /* Core id. */
+    bool pinned;               /* If a thread has been pinned to the core. */
+};
+
+/* Contains all 'struct cpu_socket's. */
+static struct hmap all_cpu_sockets = HMAP_INITIALIZER(&all_cpu_sockets);
+/* Contains all 'struct cpu_core's. */
+static struct hmap all_cpu_cores = HMAP_INITIALIZER(&all_cpu_cores);
+/* True if socket and core info are correctly extracted. */
+static bool found_sockets_and_cores;
+
+/* Returns true if 'str' contains all digits.  Returns false otherwise. */
+static bool
+contain_all_digits(const char *str)
+{
+    return str[strspn(str, "0123456789")] == '\0';
+}
+
+/* Discovers all cpu sockets and the corresponding cpu cores for each socket.
+ * Constructs the 'struct cpu_socket' and 'struct cpu_core'. */
+static void
+discover_sockets_and_cores(void)
+{
+    int n_cpus = 0;
+    int i;
+
+    for (i = 0; i < MAX_CPU_SOCKETS; i++) {
+        DIR *dir;
+        char* path;
+
+        /* Constructs the path to node /sys/devices/system/nodeX. */
+        path = xasprintf("/sys/devices/system/node/node%d", i);
+        dir = opendir(path);
+
+        /* Creates 'struct cpu_socket' if the 'dir' is non-null. */
+        if (dir) {
+            struct cpu_socket *s = xzalloc(sizeof *s);
+            struct dirent *subdir;
+
+            hmap_insert(&all_cpu_sockets, &s->hmap_node, hash_int(i, 0));
+            list_init(&s->cores);
+            s->socket_id = i;
+
+            while ((subdir = readdir(dir)) != NULL) {
+                if (!strncmp(subdir->d_name, "cpu", 3)
+                    && contain_all_digits(subdir->d_name + 3)){
+                    struct cpu_core *c = xzalloc(sizeof *c);
+                    uint32_t core_id;
+
+                    core_id = strtoul(subdir->d_name + 3, NULL, 10);
+                    hmap_insert(&all_cpu_cores, &c->hmap_node,
+                                hash_int(core_id, 0));
+                    list_insert(&s->cores, &c->list_node);
+                    c->core_id = core_id;
+                    n_cpus++;
+                }
+            }
+            VLOG_INFO("Discovered %"PRIu64" CPU cores on CPU socket %d",
+                      list_size(&s->cores), s->socket_id);
+            free(path);
+            closedir(dir);
+        } else {
+            if (errno != ENOENT) {
+                VLOG_WARN("opendir(%s) failed (%s)", path,
+                          ovs_strerror(errno));
+            }
+            free(path);
+            break;
+        }
+    }
+
+    VLOG_INFO("Discovered %"PRIu64" CPU Sockets and %d CPU cores",
+               hmap_count(&all_cpu_sockets), n_cpus);
+    if (hmap_count(&all_cpu_sockets) && hmap_count(&all_cpu_cores)) {
+        found_sockets_and_cores = true;
+    }
+}
+
+/* Extracts the numa node and core info from the 'sysfs'. */
+void
+ovs_numa_init(void)
+{
+    static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
+
+    if (ovsthread_once_start(&once)) {
+        discover_sockets_and_cores();
+        ovsthread_once_done(&once);
+    }
+}
+
+bool
+ovs_numa_cpu_socket_id_is_valid(int sid)
+{
+    return sid < ovs_numa_get_n_sockets();
+}
+
+bool
+ovs_numa_cpu_core_id_is_valid(int cid)
+{
+    return cid < ovs_numa_get_n_cores();
+}
+
+/* Returns the number of cpu sockets. */
+int
+ovs_numa_get_n_sockets(void)
+{
+    return found_sockets_and_cores ? hmap_count(&all_cpu_sockets)
+                                   : OVS_SOCKET_UNSPEC;
+}
+
+/* Returns the number of cpu cores. */
+int
+ovs_numa_get_n_cores(void)
+{
+    return found_sockets_and_cores ? hmap_count(&all_cpu_cores)
+                                   : OVS_CORE_UNSPEC;
+}
+
+/* Returns the number of cpu cores on socket. */
+int
+ovs_numa_get_n_cores_on_socket(int socket_id)
+{
+    if (found_sockets_and_cores) {
+        struct cpu_socket *socket;
+
+        ovs_assert(ovs_numa_cpu_socket_id_is_valid(socket_id));
+        socket = CONTAINER_OF(hmap_first_with_hash(&all_cpu_sockets,
+                                                   hash_int(socket_id, 0)),
+                              struct cpu_socket, hmap_node);
+
+        return list_size(&socket->cores);
+    }
+
+    return OVS_CORE_UNSPEC;
+}
+
+/* Returns the number of unpinned cpu cores on socket. */
+int
+ovs_numa_get_n_unpinned_cores_on_socket(int socket_id)
+{
+    if (found_sockets_and_cores) {
+        struct cpu_socket *socket;
+        struct cpu_core *core;
+        int count = 0;
+
+        ovs_assert(ovs_numa_cpu_socket_id_is_valid(socket_id));
+        socket = CONTAINER_OF(hmap_first_with_hash(&all_cpu_sockets,
+                                                   hash_int(socket_id, 0)),
+                              struct cpu_socket, hmap_node);
+        LIST_FOR_EACH(core, list_node, &socket->cores) {
+            if (!core->pinned) {
+                count++;
+            }
+        }
+
+        return count;
+    }
+
+    return OVS_CORE_UNSPEC;
+}
+
+/* Given 'core_id', tries to pin that core.  Returns true, if succeeds.
+ * False, if the core has already been pinned. */
+bool
+ovs_numa_try_pin_core_specific(int core_id)
+{
+    struct cpu_core *core;
+
+    ovs_assert(ovs_numa_cpu_core_id_is_valid(core_id));
+
+    core = CONTAINER_OF(hmap_first_with_hash(&all_cpu_cores,
+                                             hash_int(core_id, 0)),
+                        struct cpu_core, hmap_node);
+    if (!core->pinned) {
+        core->pinned = true;
+        return true;
+    }
+
+    return false;
+}
+
+/* Searches through all cores for an unpinned core.  Returns the core_id
+ * if found and set the 'core->pinned' to true.  Otherwise, returns -1. */
+int
+ovs_numa_get_unpinned_core_any(void)
+{
+    struct cpu_core *core;
+
+    HMAP_FOR_EACH(core, hmap_node, &all_cpu_cores) {
+        if (!core->pinned) {
+            core->pinned = true;
+            return core->core_id;
+        }
+    }
+
+    return OVS_CORE_UNSPEC;
+}
+
+/* Searches through all cores on socket with 'socket_id' for an unpinned core.
+ * Returns the core_id if found and sets the 'core->pinned' to true.
+ * Otherwise, returns -1. */
+int
+ovs_numa_get_unpinned_core_on_socket(int socket_id)
+{
+    struct cpu_socket *socket;
+    struct cpu_core *core;
+
+    ovs_assert(ovs_numa_cpu_socket_id_is_valid(socket_id));
+
+    socket = CONTAINER_OF(hmap_first_with_hash(&all_cpu_sockets,
+                                               hash_int(socket_id, 0)),
+                          struct cpu_socket, hmap_node);
+    LIST_FOR_EACH(core, list_node, &socket->cores) {
+        if (!core->pinned) {
+            core->pinned = true;
+            return core->core_id;
+        }
+    }
+
+    return OVS_CORE_UNSPEC;
+}
+
+/* Resets the 'core->pinned' for the core with 'core_id'. */
+void
+ovs_numa_unpin_core(int core_id)
+{
+    struct cpu_core *core;
+
+    ovs_assert(ovs_numa_cpu_core_id_is_valid(core_id));
+
+    core = CONTAINER_OF(hmap_first_with_hash(&all_cpu_cores,
+                                             hash_int(core_id, 0)),
+                        struct cpu_core, hmap_node);
+    core->pinned = false;
+}
+
+#endif /* __linux__ */
diff --git a/lib/ovs-numa.h b/lib/ovs-numa.h
new file mode 100644
index 0000000..95884c5
--- /dev/null
+++ b/lib/ovs-numa.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2014 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef OVS_NUMA_H
+#define OVS_NUMA_H 1
+
+#include <limits.h>
+#include <stdbool.h>
+
+#include "compiler.h"
+
+#define OVS_CORE_UNSPEC INT_MAX
+#define OVS_SOCKET_UNSPEC INT_MAX
+
+#ifdef __linux__
+
+void ovs_numa_init(void);
+bool ovs_numa_cpu_socket_id_is_valid(int sid);
+bool ovs_numa_cpu_core_id_is_valid(int cid);
+int ovs_numa_get_n_sockets(void);
+int ovs_numa_get_n_cores(void);
+int ovs_numa_get_n_cores_on_socket(int socket_id);
+int ovs_numa_get_n_unpinned_cores_on_socket(int socket_id);
+bool ovs_numa_try_pin_core_specific(int core_id);
+int ovs_numa_get_unpinned_core_any(void);
+int ovs_numa_get_unpinned_core_on_socket(int socket_id);
+void ovs_numa_unpin_core(int core_id);
+
+#else
+
+static inline void
+ovs_numa_init(void)
+{
+    /* Nothing */
+}
+
+static inline bool
+ovs_numa_cpu_socket_id_is_valid(int sid OVS_UNUSED)
+{
+    return false;
+}
+
+static inline bool
+ovs_numa_cpu_core_id_is_valid(int cid OVS_UNUSED)
+{
+    return false;
+}
+
+static inline int
+ovs_numa_get_n_sockets(void)
+{
+    return OVS_SOCKET_UNSPEC;
+}
+
+static inline int
+ovs_numa_get_n_cores(void)
+{
+    return OVS_CORE_UNSPEC;
+}
+
+static inline int
+ovs_numa_get_n_cores_on_socket(int socket_id OVS_UNUSED)
+{
+    return OVS_CORE_UNSPEC;
+}
+
+static inline int
+ovs_numa_get_n_unpinned_cores_on_socket(int socket_id OVS_UNUSED)
+{
+    return OVS_CORE_UNSPEC;
+}
+
+static inline bool
+ovs_numa_try_pin_core_specific(int core_id OVS_UNUSED)
+{
+    return false;
+}
+
+static inline int
+ovs_numa_get_unpinned_core_any(void)
+{
+    return OVS_CORE_UNSPEC;
+}
+
+static inline int
+ovs_numa_get_unpinned_core_on_socket(int socket_id OVS_UNUSED)
+{
+    return OVS_CORE_UNSPEC;
+}
+
+static inline void
+ovs_numa_unpin_core(int core_id OVS_UNUSED)
+{
+    /* Nothing */
+}
+
+#endif /* __linux__ */
+#endif /* ovs-thead.h */
diff --git a/tests/ofproto-macros.at b/tests/ofproto-macros.at
index 85ecc5c..77b9b39 100644
--- a/tests/ofproto-macros.at
+++ b/tests/ofproto-macros.at
@@ -76,6 +76,7 @@ m4_define([OVS_VSWITCHD_START],
    AT_CHECK([ovs-vswitchd --detach --no-chdir --pidfile --enable-dummy$3 --disable-system --log-file -vvconn -vofproto_dpif], [0], [], [stderr])
    AT_CAPTURE_FILE([ovs-vswitchd.log])
    AT_CHECK([[sed < stderr '
+/ovs_numa|INFO|Discovered /d
 /vlog|INFO|opened log file/d
 /vswitchd|INFO|ovs-vswitchd (Open vSwitch)/d
 /reconnect|INFO|/d
diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
index d8ea4dd..42fc0ec 100644
--- a/vswitchd/bridge.c
+++ b/vswitchd/bridge.c
@@ -42,6 +42,7 @@
 #include "ofpbuf.h"
 #include "ofproto/bond.h"
 #include "ofproto/ofproto.h"
+#include "ovs-numa.h"
 #include "poll-loop.h"
 #include "seq.h"
 #include "sha1.h"
@@ -441,6 +442,7 @@ bridge_init(const char *remote)
     lacp_init();
     bond_init();
     cfm_init();
+    ovs_numa_init();
     stp_init();
 }
 
-- 
1.7.9.5




More information about the dev mailing list