[ovs-dev] [PATCH v2 1/6] dpif-netdev/mfex: Add AVX512 basic ipv6 traffic profiles

Kumar Amber kumar.amber at intel.com
Tue Aug 24 14:27:58 UTC 2021


Add AVX512 IPv6 optimized profile for IPv6/UDP and
IPv6/TCP.

MFEX autovalidaton test-case already has the IPv6 support for
validating against the scalar mfex.

Signed-off-by: Kumar Amber <kumar.amber at intel.com>
Signed-off-by: Harry van Haaren <harry.van.haaren at intel.com>
Co-authored-by: Harry van Haaren <harry.van.haaren at intel.com>

---
v2:
- Fix CI build error
- Fix check-patch sign-offs
---
 NEWS                              |   3 +
 lib/automake.mk                   |   1 +
 lib/dpif-netdev-extract-avx512.c  | 140 +++++++++++++++++++++++++++++-
 lib/dpif-netdev-private-extract.c |  28 +++++-
 lib/dpif-netdev-private-extract.h |   6 ++
 tests/pcap/mfex_test.pcap         | Bin 416 -> 632 bytes
 6 files changed, 176 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 1f2adf718..f18e2c572 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,9 @@ Post-v2.16.0
        by default.  'other_config:dpdk-socket-limit' can be set equal to
        the 'other_config:dpdk-socket-mem' to preserve the legacy memory
        limiting behavior.
+   - Userspace datapath:
+     * Add AVX512 optimized profiles to miniflow extract for IPv6/UDP and
+       IPv6/TCP.
 
 
 v2.16.0 - 16 Aug 2021
diff --git a/lib/automake.mk b/lib/automake.mk
index 8ac138f71..245c0886c 100644
--- a/lib/automake.mk
+++ b/lib/automake.mk
@@ -33,6 +33,7 @@ lib_libopenvswitchavx512_la_CFLAGS = \
 	-mavx512f \
 	-mavx512bw \
 	-mavx512dq \
+	-mavx512vl \
 	-mbmi \
 	-mbmi2 \
 	-fPIC \
diff --git a/lib/dpif-netdev-extract-avx512.c b/lib/dpif-netdev-extract-avx512.c
index ec64419e3..3384a8dba 100644
--- a/lib/dpif-netdev-extract-avx512.c
+++ b/lib/dpif-netdev-extract-avx512.c
@@ -49,6 +49,8 @@
 #include "dpif-netdev-private-extract.h"
 #include "dpif-netdev-private-flow.h"
 
+#define plen ip6_ctlun.ip6_un1.ip6_un1_plen
+
 /* AVX512-BW level permutex2var_epi8 emulation. */
 static inline __m512i
 __attribute__((target("avx512bw")))
@@ -137,6 +139,7 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a)
 #define PATTERN_ETHERTYPE_MASK PATTERN_ETHERTYPE_GEN(0xFF, 0xFF)
 #define PATTERN_ETHERTYPE_IPV4 PATTERN_ETHERTYPE_GEN(0x08, 0x00)
 #define PATTERN_ETHERTYPE_DT1Q PATTERN_ETHERTYPE_GEN(0x81, 0x00)
+#define PATTERN_ETHERTYPE_IPV6 PATTERN_ETHERTYPE_GEN(0x86, 0xDD)
 
 /* VLAN (Dot1Q) patterns and masks. */
 #define PATTERN_DT1Q_MASK                                               \
@@ -192,6 +195,25 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a)
   NU, NU, NU, NU, NU, NU, NU, NU, 38, 39, 40, 41, NU, NU, NU, NU, /* TCP */   \
   NU, NU, NU, NU, NU, NU, NU, NU, /* Unused. */
 
+/* Generator for checking IPv6 ver. */
+#define PATTERN_IPV6_GEN(VER_TRC, PROTO)                                      \
+  VER_TRC,     /* Version: 4bits and Traffic class: 4bits. */                 \
+  0, 0, 0,     /* Traffic class: 4bits and Flow Label: 24bits. */             \
+  0, 0,        /* Payload length 16bits. */                                   \
+  PROTO, 0,    /* Next Header 8bits and Hop limit 8bits. */                   \
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Src IP: 128bits. */      \
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Dst IP: 128bits. */
+
+#define PATTERN_IPV6_MASK PATTERN_IPV6_GEN(0xF0, 0xFF)
+#define PATTERN_IPV6_UDP PATTERN_IPV6_GEN(0x60, 0x11)
+#define PATTERN_IPV6_TCP PATTERN_IPV6_GEN(0x60, 0x06)
+
+#define PATTERN_IPV6_SHUFFLE                                                  \
+   0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, NU, NU, /* Ether */ \
+  22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, /* IPv6 */  \
+  38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, /* IPv6 */  \
+  NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, /* Unused */
+
 /* Generation of K-mask bitmask values, to zero out data in result. Note that
  * these correspond 1:1 to the above "*_SHUFFLE" values, and bit used must be
  * set in this K-mask, and "NU" values must be zero in the k-mask. Each mask
@@ -204,6 +226,8 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a)
 #define KMASK_IPV4      0xF0FFULL
 #define KMASK_UDP       0x000FULL
 #define KMASK_TCP       0x0F00ULL
+#define KMASK_IPV6      0xFFFFULL
+#define KMASK_ETHER_IPV6     0x3FFFULL
 
 #define PATTERN_IPV4_UDP_KMASK \
     (KMASK_ETHER | (KMASK_IPV4 << 16) | (KMASK_UDP << 32))
@@ -217,6 +241,9 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a)
 #define PATTERN_DT1Q_IPV4_TCP_KMASK \
     (KMASK_ETHER | (KMASK_DT1Q << 16) | (KMASK_IPV4 << 24) | (KMASK_TCP << 40))
 
+#define PATTERN_IPV6_KMASK \
+    (KMASK_ETHER_IPV6 | (KMASK_IPV6 << 16) | (KMASK_IPV6 << 32))
+
 /* This union allows initializing static data as u8, but easily loading it
  * into AVX512 registers too. The union ensures proper alignment for the zmm.
  */
@@ -295,6 +322,8 @@ enum MFEX_PROFILES {
     PROFILE_ETH_IPV4_TCP,
     PROFILE_ETH_VLAN_IPV4_UDP,
     PROFILE_ETH_VLAN_IPV4_TCP,
+    PROFILE_ETH_IPV6_UDP,
+    PROFILE_ETH_IPV6_TCP,
     PROFILE_COUNT,
 };
 
@@ -368,8 +397,84 @@ static const struct mfex_profile mfex_profiles[PROFILE_COUNT] =
         },
         .dp_pkt_min_size = 46,
     },
+
+    [PROFILE_ETH_IPV6_UDP] = {
+        .probe_mask.u8_data = { PATTERN_ETHERTYPE_MASK PATTERN_IPV6_MASK },
+        .probe_data.u8_data = { PATTERN_ETHERTYPE_IPV6 PATTERN_IPV6_UDP },
+
+        .store_shuf.u8_data = { PATTERN_IPV6_SHUFFLE },
+        .store_kmsk = PATTERN_IPV6_KMASK,
+
+        .mf_bits = { 0x18a0000000000000, 0x000000000004043c},
+        .dp_pkt_offs = {
+            0, UINT16_MAX, 14, 54,
+        },
+        .dp_pkt_min_size = 54,
+    },
+
+    [PROFILE_ETH_IPV6_TCP] = {
+        .probe_mask.u8_data = { PATTERN_ETHERTYPE_MASK PATTERN_IPV6_MASK },
+        .probe_data.u8_data = { PATTERN_ETHERTYPE_IPV6 PATTERN_IPV6_TCP },
+
+        .store_shuf.u8_data = { PATTERN_IPV6_SHUFFLE },
+        .store_kmsk = PATTERN_IPV6_KMASK,
+
+        .mf_bits = { 0x18a0000000000000, 0x000000000004443c},
+        .dp_pkt_offs = {
+            0, UINT16_MAX, 14, 54,
+        },
+        .dp_pkt_min_size = 54,
+    },
+
 };
 
+/* IPv6 header helper function to fix TC, flow label and next header. */
+static inline void ALWAYS_INLINE
+mfex_handle_ipv6_hdr_block(const uint8_t *ipv6, uint64_t *block)
+{
+    static const uint8_t data_shuf[16] = {
+        0, 1, 2, 3, /* copy IPv6 label in place, it is masked later. */
+        1, 0,       /* Byte-swap TC fields for LE usage. */
+        7, 6,       /* Move TTL and next proto to MF required locations. */
+    };
+
+    /* BE mask for IPv6 label, and mask to strip away unwanted TC bits. */
+    const uint64_t mask = 0xffff0f00 | (UINT64_MAX << 40);
+    uint64_t mask_data[2] = { mask, mask };
+
+    /* Load constant data. Is lifted to occur 1x per burst, not per packet. */
+    __m128i ipv6_hdr = _mm_loadu_si128((void *) ipv6);
+    __m128i v_mask = _mm_loadu_si128((void *) mask_data);
+    __m128i v_shuf_mask = _mm_loadu_si128((void *) data_shuf);
+
+    /* Shuffle data layout, shift 16-bits to get TC fixed, mask to cleanup. */
+    __m128i v_ipv6 = _mm_shuffle_epi8(ipv6_hdr, v_shuf_mask);
+    __m128i v_tc_shift = _mm_mask_slli_epi16(v_ipv6, 0b100, v_ipv6, 4);
+    __m128i v_ipv6_m = _mm_and_si128(v_tc_shift, v_mask);
+
+    *block = _mm_extract_epi64(v_ipv6_m, 0);
+}
+
+/* IPv6 Protocol specific helper functions, for handling L4 UDP/TCP. */
+static inline void
+mfex_handle_ipv6_l4(const uint8_t *ports, uint64_t *block)
+{
+    void *ptr_ports = (void *) ports;
+    uint32_t l4_ports = *(uint32_t *) ptr_ports;
+    *block = l4_ports;
+}
+
+/* IPv6 specific helper functions, for calculating offsets/lengths. */
+static void
+mfex_ipv6_set_l2_pad_size(struct dp_packet *pkt,
+                          struct ovs_16aligned_ip6_hdr *nh,
+                          uint32_t payload_size_ipv6)
+{
+    /* Handle dynamic l2_pad_size. */
+    payload_size_ipv6 = payload_size_ipv6 - IPV6_HEADER_LEN;
+    uint16_t p_len =  ntohs(nh->plen);
+    dp_packet_set_l2_pad_size(pkt, payload_size_ipv6 - p_len);
+}
 
 /* Protocol specific helper functions, for calculating offsets/lenghts. */
 static int32_t
@@ -539,6 +644,38 @@ mfex_avx512_process(struct dp_packet_batch *packets,
                 }
 
             } break;
+
+        case PROFILE_ETH_IPV6_UDP: {
+                /* Handle dynamic l2_pad_size. */
+                uint32_t payload_size_ipv6 = size - sizeof(struct eth_header);
+                struct ovs_16aligned_ip6_hdr *nh = (void *)&pkt[sizeof
+                                                   (struct eth_header)];
+                mfex_ipv6_set_l2_pad_size(packet, nh, payload_size_ipv6);
+
+                /* Process IPv6 header for TC, flow Label and next header. */
+                mfex_handle_ipv6_hdr_block(&pkt[ETH_HEADER_LEN], &blocks[8]);
+
+                /* Process UDP header. */
+                mfex_handle_ipv6_l4((void *)&pkt[54], &blocks[9]);
+
+            } break;
+
+        case PROFILE_ETH_IPV6_TCP: {
+                /* Handle dynamic l2_pad_size. */
+                uint32_t payload_size_ipv6 = size - sizeof(struct eth_header);
+                struct ovs_16aligned_ip6_hdr *nh = (void *)&pkt[sizeof
+                                                   (struct eth_header)];
+                mfex_ipv6_set_l2_pad_size(packet, nh, payload_size_ipv6);
+
+                /* Process IPv6 header for TC, flow Label and next header. */
+                mfex_handle_ipv6_hdr_block(&pkt[ETH_HEADER_LEN], &blocks[8]);
+
+                /* Process TCP header. */
+                mfex_handle_ipv6_l4((void *)&pkt[54], &blocks[10]);
+                const struct tcp_header *tcp = (void *)&pkt[54];
+                mfex_handle_tcp_flags(tcp, &blocks[9]);
+
+            } break;
         default:
             break;
         };
@@ -584,7 +721,8 @@ DECLARE_MFEX_FUNC(ip_udp, PROFILE_ETH_IPV4_UDP)
 DECLARE_MFEX_FUNC(ip_tcp, PROFILE_ETH_IPV4_TCP)
 DECLARE_MFEX_FUNC(dot1q_ip_udp, PROFILE_ETH_VLAN_IPV4_UDP)
 DECLARE_MFEX_FUNC(dot1q_ip_tcp, PROFILE_ETH_VLAN_IPV4_TCP)
-
+DECLARE_MFEX_FUNC(ipv6_udp, PROFILE_ETH_IPV6_UDP)
+DECLARE_MFEX_FUNC(ipv6_tcp, PROFILE_ETH_IPV6_TCP)
 
 static int32_t
 avx512_isa_probe(uint32_t needs_vbmi)
diff --git a/lib/dpif-netdev-private-extract.c b/lib/dpif-netdev-private-extract.c
index 7a06dbf6f..0b665bced 100644
--- a/lib/dpif-netdev-private-extract.c
+++ b/lib/dpif-netdev-private-extract.c
@@ -93,7 +93,33 @@ static struct dpif_miniflow_extract_impl mfex_impls[] = {
     [MFEX_IMPL_DOT1Q_IPv4_TCP] = {
         .probe = mfex_avx512_probe,
         .extract_func = mfex_avx512_dot1q_ip_tcp,
-        .name = "avx512_dot1q_ipv4_tcp", },
+        .name = "avx512_dot1q_ipv4_tcp",
+    },
+
+    [MFEX_IMPL_VMBI_IPv6_UDP] = {
+        .probe = mfex_avx512_vbmi_probe,
+        .extract_func = mfex_avx512_vbmi_ipv6_udp,
+        .name = "avx512_vbmi_ipv6_udp",
+    },
+
+    [MFEX_IMPL_IPv6_UDP] = {
+        .probe = mfex_avx512_probe,
+        .extract_func = mfex_avx512_ipv6_udp,
+        .name = "avx512_ipv6_udp",
+    },
+
+    [MFEX_IMPL_VMBI_IPv6_TCP] = {
+        .probe = mfex_avx512_vbmi_probe,
+        .extract_func = mfex_avx512_vbmi_ipv6_tcp,
+        .name = "avx512_vbmi_ipv6_tcp",
+    },
+
+    [MFEX_IMPL_IPv6_TCP] = {
+        .probe = mfex_avx512_probe,
+        .extract_func = mfex_avx512_ipv6_tcp,
+        .name = "avx512_ipv6_tcp",
+    },
+
 #endif
 };
 
diff --git a/lib/dpif-netdev-private-extract.h b/lib/dpif-netdev-private-extract.h
index f9a757ba4..4bbe70f26 100644
--- a/lib/dpif-netdev-private-extract.h
+++ b/lib/dpif-netdev-private-extract.h
@@ -90,6 +90,10 @@ enum dpif_miniflow_extract_impl_idx {
     MFEX_IMPL_DOT1Q_IPv4_UDP,
     MFEX_IMPL_VMBI_DOT1Q_IPv4_TCP,
     MFEX_IMPL_DOT1Q_IPv4_TCP,
+    MFEX_IMPL_VMBI_IPv6_UDP,
+    MFEX_IMPL_IPv6_UDP,
+    MFEX_IMPL_VMBI_IPv6_TCP,
+    MFEX_IMPL_IPv6_TCP,
 #endif
     MFEX_IMPL_MAX
 };
@@ -197,6 +201,8 @@ DECLARE_AVX512_MFEX_PROTOTYPE(ip_udp);
 DECLARE_AVX512_MFEX_PROTOTYPE(ip_tcp);
 DECLARE_AVX512_MFEX_PROTOTYPE(dot1q_ip_udp);
 DECLARE_AVX512_MFEX_PROTOTYPE(dot1q_ip_tcp);
+DECLARE_AVX512_MFEX_PROTOTYPE(ipv6_udp);
+DECLARE_AVX512_MFEX_PROTOTYPE(ipv6_tcp);
 
 #endif /* __x86_64__ */
 
diff --git a/tests/pcap/mfex_test.pcap b/tests/pcap/mfex_test.pcap
index 1aac67b8d643ecb016c758cba4cc32212a80f52a..66c78021fa42345f6866a0e2f51da53ba482ac51 100644
GIT binary patch
literal 632
zcmca|c+)~A1{MYw`2U}Qff2|NR*_3o>*ix{0<uB)|B?Nd%tT+un5zBGYXmB3yPE(M
zW#ABW=!{;u`L|!=(XW0b9(CHe+A~kJnd!CbY&kgZ!IzvX3aJdHK*- at A2{X|Ts11zY
z>&HJ>a!-E8{-bx1OcY^rSQ8o2RlbBh-mkXj{L$JlM%&*F%BB~DC7a$!#T=aTW(9)?
zLjcHf5C~vWU^r%saIYOu41~ot9$d2|`2LQUk6XhKZhhqxVS8Qn+1t;R|CfK-ZfVZ5
z<L%|&Y|@XH=^iccUp-X^;?~;NFt>UEO#)$!l_%FJE(xf+7CToOY8ud;$GO<|ou3yn
zP5Q)~%eP-R1_x<`-95B}=bb~HhezxEQfG8`o)Sj5(+p at T2+KcMSog$3!h6pftuGu5
zu0X3m!SBGfEVGzvfdl)No$2VN?3O^7q6IVogc~%LE>9O*ac=kYJ}a;(GGJ2#W!sA_
z{m&O^Xh=eQwnz)+Gb^Aj5Z?S?@r6&%7rauLq&fi<3&?I*k+s*$$mz$<S90iX*ei)}
pgC5Wn5Efm at oSbspm6^}zLp$6gs5|yw at k{m#=dlrDgSf+`0sxz0v&8 at a

literal 416
zcmca|c+)~A1{MYw`2U}Qff2}Q<eHVR>K`M68ITRa|G at yFii5$Gfk6YL%z>@uY&}o|
z2s4N<1VH2&7y^V87$)XGOtD~MV$cFgfG~zBGGJ2#YtF$<F=a4i;9x8Q*<ZrSM6Ufz
xK>KST_NTIwYriok6N4Vm)gX-Q@<yO<!C`>c^{cp<7_5LgK^UuU{2>VS0RZ!RQ+EIW

-- 
2.25.1



More information about the dev mailing list