[ovs-dev] [RFC PATCH 15/26] netdev-offload: Add multi-thread API
Gaetan Rivet
grive at u256.net
Sat Dec 5 14:22:10 UTC 2020
Expose functions reporting user configuration of offloading threads, as
well as utility functions for multithreading.
This will only expose the configuration knob to the user, while no
datapath will implement the multiple thread request.
This will allow implementations to use this API for offload thread
management in relevant layers before enabling the actual dataplane
implementation.
The offload thread ID is lazily allocated and can as such be in a
different order than the offload thread start sequence.
The RCU thread will sometime access hardware-offload objects from
a provider for reclamation purposes. In such case, it will get
a default offload thread ID of 0. Care must be taken that using
this thread ID is safe concurrently with the offload threads.
Signed-off-by: Gaetan Rivet <grive at u256.net>
---
lib/netdev-offload-dpdk.c | 4 +-
lib/netdev-offload-provider.h | 6 ++-
lib/netdev-offload.c | 81 +++++++++++++++++++++++++++++++++--
lib/netdev-offload.h | 21 ++++++++-
vswitchd/vswitch.xml | 16 +++++++
5 files changed, 120 insertions(+), 8 deletions(-)
diff --git a/lib/netdev-offload-dpdk.c b/lib/netdev-offload-dpdk.c
index b29c1188f..7648a0ccd 100644
--- a/lib/netdev-offload-dpdk.c
+++ b/lib/netdev-offload-dpdk.c
@@ -1656,12 +1656,12 @@ out:
static int
netdev_offload_dpdk_hw_offload_stats_get(struct netdev *netdev,
- uint64_t *counter)
+ uint64_t *counters)
{
struct netdev_offload_dpdk_data *data;
data = netdev->hw_info.offload_data;
- *counter = data->rte_flow_counter;
+ *counters = data->rte_flow_counter;
return 0;
}
diff --git a/lib/netdev-offload-provider.h b/lib/netdev-offload-provider.h
index fd38cea66..2ac73cd36 100644
--- a/lib/netdev-offload-provider.h
+++ b/lib/netdev-offload-provider.h
@@ -83,8 +83,10 @@ struct netdev_flow_api {
int (*flow_del)(struct netdev *, const ovs_u128 *ufid,
struct dpif_flow_stats *);
- /* Queries an offload provider hardware statistics. */
- int (*hw_offload_stats_get)(struct netdev *netdev, uint64_t *counter);
+ /* Queries an offload provider hardware statistics.
+ * One counter per offload thread is expected.
+ */
+ int (*hw_offload_stats_get)(struct netdev *netdev, uint64_t *counters);
/* Initializies the netdev flow api.
* Return 0 if successful, otherwise returns a positive errno value. */
diff --git a/lib/netdev-offload.c b/lib/netdev-offload.c
index 4a8403ead..17b8cbc5a 100644
--- a/lib/netdev-offload.c
+++ b/lib/netdev-offload.c
@@ -60,6 +60,12 @@ VLOG_DEFINE_THIS_MODULE(netdev_offload);
static bool netdev_flow_api_enabled = false;
+#define DEFAULT_OFFLOAD_THREAD_NB 1
+#define MAX_OFFLOAD_THREAD_NB 10
+
+static unsigned int offload_thread_nb = DEFAULT_OFFLOAD_THREAD_NB;
+DEFINE_EXTERN_PER_THREAD_DATA(netdev_offload_thread_id, OVSTHREAD_ID_UNSET);
+
/* Protects 'netdev_flow_apis'. */
static struct ovs_mutex netdev_flow_api_provider_mutex = OVS_MUTEX_INITIALIZER;
@@ -281,13 +287,13 @@ netdev_flow_del(struct netdev *netdev, const ovs_u128 *ufid,
}
int
-netdev_hw_offload_stats_get(struct netdev *netdev, uint64_t *counter)
+netdev_hw_offload_stats_get(struct netdev *netdev, uint64_t *counters)
{
const struct netdev_flow_api *flow_api =
ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
return (flow_api && flow_api->hw_offload_stats_get)
- ? flow_api->hw_offload_stats_get(netdev, counter)
+ ? flow_api->hw_offload_stats_get(netdev, counters)
: EOPNOTSUPP;
}
@@ -436,6 +442,64 @@ netdev_is_flow_api_enabled(void)
return netdev_flow_api_enabled;
}
+unsigned int
+netdev_offload_thread_nb(void)
+{
+ return offload_thread_nb;
+}
+
+unsigned int
+netdev_offload_ufid_to_thread_id(const ovs_u128 ufid)
+{
+ uint32_t ufid_hash;
+
+ if (netdev_offload_thread_nb() == 1) {
+ return 0;
+ }
+
+ ufid_hash = hash_words64_inline(
+ (const uint64_t [2]){ ufid.u64.lo,
+ ufid.u64.hi }, 2, 1);
+ return ufid_hash % netdev_offload_thread_nb();
+}
+
+unsigned int
+netdev_offload_thread_init(void)
+{
+ static atomic_count next_id = ATOMIC_COUNT_INIT(0);
+ bool thread_is_hw_offload;
+ bool thread_is_rcu;
+
+ thread_is_hw_offload = !strncmp(get_subprogram_name(),
+ "hw_offload", strlen("hw_offload"));
+ thread_is_rcu = !strncmp(get_subprogram_name(), "urcu", strlen("urcu"));
+
+ /* Panic if any other thread besides offload and RCU tries
+ * to initialize their thread ID. */
+ ovs_assert(thread_is_hw_offload || thread_is_rcu);
+
+ if (*netdev_offload_thread_id_get() == OVSTHREAD_ID_UNSET) {
+ unsigned int id;
+
+ if (thread_is_rcu) {
+ /* RCU will compete with other threads for shared object access.
+ * Reclamation functions using a thread ID must be thread-safe.
+ * For that end, and because RCU must consider all potential shared
+ * objects anyway, its thread-id can be whichever, so return 0.
+ */
+ id = 0;
+ } else {
+ /* Only the actual offload threads have their own ID. */
+ id = atomic_count_inc(&next_id);
+ }
+ /* Panic if any offload thread is getting a spurious ID. */
+ ovs_assert(id < netdev_offload_thread_nb());
+ return *netdev_offload_thread_id_get() = id;
+ } else {
+ return *netdev_offload_thread_id_get();
+ }
+}
+
void
netdev_ports_flow_flush(const char *dpif_type)
{
@@ -664,7 +728,18 @@ netdev_set_flow_api_enabled(const struct smap *ovs_other_config)
if (ovsthread_once_start(&once)) {
netdev_flow_api_enabled = true;
- VLOG_INFO("netdev: Flow API Enabled");
+ offload_thread_nb = smap_get_ullong(ovs_other_config,
+ "hw-offload-thread-nb",
+ DEFAULT_OFFLOAD_THREAD_NB);
+ if (offload_thread_nb > MAX_OFFLOAD_THREAD_NB) {
+ VLOG_WARN("netdev: Invalid number of threads requested: %u",
+ offload_thread_nb);
+ offload_thread_nb = DEFAULT_OFFLOAD_THREAD_NB;
+ }
+
+ VLOG_INFO("netdev: Flow API Enabled, using %u thread%s",
+ offload_thread_nb,
+ offload_thread_nb > 1 ? "s" : "");
#ifdef __linux__
tc_set_policy(smap_get_def(ovs_other_config, "tc-policy",
diff --git a/lib/netdev-offload.h b/lib/netdev-offload.h
index 5ed561d13..e60539706 100644
--- a/lib/netdev-offload.h
+++ b/lib/netdev-offload.h
@@ -20,6 +20,7 @@
#include "openvswitch/netdev.h"
#include "openvswitch/types.h"
+#include "ovs-thread.h"
#include "packets.h"
#include "flow.h"
@@ -79,6 +80,24 @@ struct offload_info {
* to delete the original flow. */
};
+DECLARE_EXTERN_PER_THREAD_DATA(unsigned int, netdev_offload_thread_id);
+
+unsigned int netdev_offload_thread_nb(void);
+unsigned int netdev_offload_thread_init(void);
+unsigned int netdev_offload_ufid_to_thread_id(const ovs_u128 ufid);
+
+static inline unsigned int
+netdev_offload_thread_id(void)
+{
+ unsigned int id = *netdev_offload_thread_id_get();
+
+ if (OVS_UNLIKELY(id == OVSTHREAD_ID_UNSET)) {
+ id = netdev_offload_thread_init();
+ }
+
+ return id;
+}
+
int netdev_flow_flush(struct netdev *);
int netdev_flow_dump_create(struct netdev *, struct netdev_flow_dump **dump,
bool terse);
@@ -95,7 +114,7 @@ int netdev_flow_get(struct netdev *, struct match *, struct nlattr **actions,
struct dpif_flow_attrs *, struct ofpbuf *wbuffer);
int netdev_flow_del(struct netdev *, const ovs_u128 *,
struct dpif_flow_stats *);
-int netdev_hw_offload_stats_get(struct netdev *, uint64_t *counter);
+int netdev_hw_offload_stats_get(struct netdev *, uint64_t *counters);
int netdev_init_flow_api(struct netdev *);
void netdev_uninit_flow_api(struct netdev *);
uint32_t netdev_get_block_id(struct netdev *);
diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index 89a876796..63f99299e 100644
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -247,6 +247,22 @@
</p>
</column>
+ <column name="other_config" key="hw-offload-thread-nb"
+ type='{"type": "integer", "minInteger": 1, "maxInteger": 10}'>
+ <p>
+ Set this value to the number of threads created to manage hardware
+ offloads.
+ </p>
+ <p>
+ The default value is <code>1</code>. Changing this value requires
+ restarting the daemon.
+ </p>
+ <p>
+ This is only relevant if
+ <ref column="other_config" key="hw-offload"/> is enabled.
+ </p>
+ </column>
+
<column name="other_config" key="tc-policy"
type='{"type": "string",
"enum": ["set", ["none", "skip_sw", "skip_hw"]]}'>
--
2.29.2
More information about the dev
mailing list