[ovs-dev] [PATCH v1] flow: miniflow_extract metadata branchless optimization

Yanqin Wei Yanqin.Wei at arm.com
Thu Aug 22 09:30:00 UTC 2019


"miniflow_extract" is a branch heavy implementation for packet header and
metadata parsing. There is a lot of meta data handling for all traffic.
But this should not be applicable for packets from interface.
This patch adds a layer of inline encapsulation to miniflow_extract and
introduces constant "md_valid" input parameter as a branch condition.
The new branch will be removed by the compiler at compile time. Two
instances of miniflow_extract with different branches will be generated.

This patch is tested on an arm64 platform. It improves more than 3.5%
performance in P2P forwarding cases.

Change-Id: I5d606afb52bfa68e8afa6f886d69b9665cdad51a
Reviewed-by: Gavin Hu <Gavin.Hu at arm.com>
Signed-off-by: Yanqin Wei <Yanqin.Wei at arm.com>
---
 lib/dpif-netdev.c |  13 +++---
 lib/flow.c        | 116 ++++++++++++++++++++++++++++++++----------------------
 lib/flow.h        |   2 +
 3 files changed, 79 insertions(+), 52 deletions(-)

diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index d0a1c58..6686b14 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -6508,12 +6508,15 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
             }
         }
 
-        miniflow_extract(packet, &key->mf);
+        if (!md_is_valid) {
+            miniflow_extract_firstpass(packet, &key->mf);
+            key->hash =
+                dpif_netdev_packet_get_rss_hash_orig_pkt(packet, &key->mf);
+        } else {
+            miniflow_extract(packet, &key->mf);
+            key->hash = dpif_netdev_packet_get_rss_hash(packet, &key->mf);
+        }
         key->len = 0; /* Not computed yet. */
-        key->hash =
-                (md_is_valid == false)
-                ? dpif_netdev_packet_get_rss_hash_orig_pkt(packet, &key->mf)
-                : dpif_netdev_packet_get_rss_hash(packet, &key->mf);
 
         /* If EMC is disabled skip emc_lookup */
         flow = (cur_min != 0) ? emc_lookup(&cache->emc_cache, key) : NULL;
diff --git a/lib/flow.c b/lib/flow.c
index e54fd2e..e5b554b 100644
--- a/lib/flow.c
+++ b/lib/flow.c
@@ -707,7 +707,8 @@ ipv6_sanity_check(const struct ovs_16aligned_ip6_hdr *nh, size_t size)
 }
 
 /* Initializes 'dst' from 'packet' and 'md', taking the packet type into
- * account.  'dst' must have enough space for FLOW_U64S * 8 bytes.
+ * account.  'dst' must have enough space for FLOW_U64S * 8 bytes. Metadata
+ * initialization should be bypassed if "md_valid" is false.
  *
  * Initializes the layer offsets as follows:
  *
@@ -732,8 +733,9 @@ ipv6_sanity_check(const struct ovs_16aligned_ip6_hdr *nh, size_t size)
  *      present and the packet has at least the content used for the fields
  *      of interest for the flow, otherwise UINT16_MAX.
  */
-void
-miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
+static inline ALWAYS_INLINE void
+miniflow_extract__(struct dp_packet *packet, struct miniflow *dst,
+                    const bool md_valid)
 {
     /* Add code to this function (or its callees) to extract new fields. */
     BUILD_ASSERT_DECL(FLOW_WC_SEQ == 41);
@@ -752,54 +754,60 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
     ovs_be16 ct_tp_src = 0, ct_tp_dst = 0;
 
     /* Metadata. */
-    if (flow_tnl_dst_is_set(&md->tunnel)) {
-        miniflow_push_words(mf, tunnel, &md->tunnel,
-                            offsetof(struct flow_tnl, metadata) /
-                            sizeof(uint64_t));
-
-        if (!(md->tunnel.flags & FLOW_TNL_F_UDPIF)) {
-            if (md->tunnel.metadata.present.map) {
-                miniflow_push_words(mf, tunnel.metadata, &md->tunnel.metadata,
-                                    sizeof md->tunnel.metadata /
-                                    sizeof(uint64_t));
-            }
-        } else {
-            if (md->tunnel.metadata.present.len) {
-                miniflow_push_words(mf, tunnel.metadata.present,
-                                    &md->tunnel.metadata.present, 1);
-                miniflow_push_words(mf, tunnel.metadata.opts.gnv,
-                                    md->tunnel.metadata.opts.gnv,
-                                    DIV_ROUND_UP(md->tunnel.metadata.present.len,
-                                                 sizeof(uint64_t)));
+    if (md_valid) {
+        if (flow_tnl_dst_is_set(&md->tunnel)) {
+            miniflow_push_words(mf, tunnel, &md->tunnel,
+                                offsetof(struct flow_tnl, metadata) /
+                                sizeof(uint64_t));
+
+            if (!(md->tunnel.flags & FLOW_TNL_F_UDPIF)) {
+                if (md->tunnel.metadata.present.map) {
+                    miniflow_push_words(mf, tunnel.metadata,
+                                        &md->tunnel.metadata,
+                                        sizeof md->tunnel.metadata /
+                                        sizeof(uint64_t));
+                }
+            } else {
+                if (md->tunnel.metadata.present.len) {
+                    miniflow_push_words(mf, tunnel.metadata.present,
+                                        &md->tunnel.metadata.present, 1);
+                    miniflow_push_words(mf, tunnel.metadata.opts.gnv,
+                                        md->tunnel.metadata.opts.gnv,
+                                        DIV_ROUND_UP(
+                                               md->tunnel.metadata.present.len,
+                                               sizeof(uint64_t)));
+                }
             }
         }
-    }
-    if (md->skb_priority || md->pkt_mark) {
-        miniflow_push_uint32(mf, skb_priority, md->skb_priority);
-        miniflow_push_uint32(mf, pkt_mark, md->pkt_mark);
-    }
-    miniflow_push_uint32(mf, dp_hash, md->dp_hash);
-    miniflow_push_uint32(mf, in_port, odp_to_u32(md->in_port.odp_port));
-    if (md->ct_state) {
-        miniflow_push_uint32(mf, recirc_id, md->recirc_id);
-        miniflow_push_uint8(mf, ct_state, md->ct_state);
-        ct_nw_proto_p = miniflow_pointer(mf, ct_nw_proto);
-        miniflow_push_uint8(mf, ct_nw_proto, 0);
-        miniflow_push_uint16(mf, ct_zone, md->ct_zone);
-    } else if (md->recirc_id) {
-        miniflow_push_uint32(mf, recirc_id, md->recirc_id);
-        miniflow_pad_to_64(mf, recirc_id);
-    }
-
-    if (md->ct_state) {
-        miniflow_push_uint32(mf, ct_mark, md->ct_mark);
-        miniflow_push_be32(mf, packet_type, packet_type);
-
-        if (!ovs_u128_is_zero(md->ct_label)) {
-            miniflow_push_words(mf, ct_label, &md->ct_label,
-                                sizeof md->ct_label / sizeof(uint64_t));
+        if (md->skb_priority || md->pkt_mark) {
+            miniflow_push_uint32(mf, skb_priority, md->skb_priority);
+            miniflow_push_uint32(mf, pkt_mark, md->pkt_mark);
+        }
+        miniflow_push_uint32(mf, dp_hash, md->dp_hash);
+        miniflow_push_uint32(mf, in_port, odp_to_u32(md->in_port.odp_port));
+        if (md->ct_state) {
+            miniflow_push_uint32(mf, recirc_id, md->recirc_id);
+            miniflow_push_uint8(mf, ct_state, md->ct_state);
+            ct_nw_proto_p = miniflow_pointer(mf, ct_nw_proto);
+            miniflow_push_uint8(mf, ct_nw_proto, 0);
+            miniflow_push_uint16(mf, ct_zone, md->ct_zone);
+        } else if (md->recirc_id) {
+            miniflow_push_uint32(mf, recirc_id, md->recirc_id);
+            miniflow_pad_to_64(mf, recirc_id);
+        }
+
+        if (md->ct_state) {
+            miniflow_push_uint32(mf, ct_mark, md->ct_mark);
+            miniflow_push_be32(mf, packet_type, packet_type);
+
+            if (!ovs_u128_is_zero(md->ct_label)) {
+                miniflow_push_words(mf, ct_label, &md->ct_label,
+                                    sizeof md->ct_label / sizeof(uint64_t));
+            }
         }
     } else {
+        miniflow_push_uint32(mf, dp_hash, md->dp_hash);
+        miniflow_push_uint32(mf, in_port, odp_to_u32(md->in_port.odp_port));
         miniflow_pad_from_64(mf, packet_type);
         miniflow_push_be32(mf, packet_type, packet_type);
     }
@@ -865,6 +873,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
 
         /* Push both source and destination address at once. */
         miniflow_push_words(mf, nw_src, &nh->ip_src, 1);
+
         if (ct_nw_proto_p && !md->ct_orig_tuple_ipv6) {
             *ct_nw_proto_p = md->ct_orig_tuple.ipv4.ipv4_proto;
             if (*ct_nw_proto_p) {
@@ -900,6 +909,7 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
                             sizeof nh->ip6_src / 8);
         miniflow_push_words(mf, ipv6_dst, &nh->ip6_dst,
                             sizeof nh->ip6_dst / 8);
+
         if (ct_nw_proto_p && md->ct_orig_tuple_ipv6) {
             *ct_nw_proto_p = md->ct_orig_tuple.ipv6.ipv6_proto;
             if (*ct_nw_proto_p) {
@@ -1076,6 +1086,18 @@ miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
     dst->map = mf.map;
 }
 
+void
+miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
+{
+    miniflow_extract__(packet, dst, true);
+}
+
+void
+miniflow_extract_firstpass(struct dp_packet *packet, struct miniflow *dst)
+{
+    miniflow_extract__(packet, dst, false);
+}
+
 ovs_be16
 parse_dl_type(const struct eth_header *data_, size_t size)
 {
diff --git a/lib/flow.h b/lib/flow.h
index 7298c71..2d38574 100644
--- a/lib/flow.h
+++ b/lib/flow.h
@@ -541,6 +541,8 @@ struct pkt_metadata;
  * 'dst->map' is ignored on input and set on output to indicate which fields
  * were extracted. */
 void miniflow_extract(struct dp_packet *packet, struct miniflow *dst);
+void miniflow_extract_firstpass(struct dp_packet *packet,
+                                struct miniflow *dst);
 void miniflow_map_init(struct miniflow *, const struct flow *);
 void flow_wc_map(const struct flow *, struct flowmap *);
 size_t miniflow_alloc(struct miniflow *dsts[], size_t n,
-- 
2.7.4



More information about the dev mailing list