[ovs-dev] [PATCH v6 1/5] userspace: Switching of L3 packets in L2 pipeline
Zoltán Balogh
zoltan.balogh at ericsson.com
Fri May 12 11:07:38 UTC 2017
From: Jan Scheurich <jan.scheurich at ericsson.com>
Ports have a new layer3 attribute if they send/receive L3 packets.
The packet_type included in structs dp_packet and flow is considered in
ofproto-dpif. The classical L2 match fields (dl_src, dl_dst, dl_type, and
vlan_tci, vlan_vid, vlan_pcp) now have Ethernet as pre-requisite.
A dummy ethernet header is pushed to L3 packets received from L3 ports
before the the pipeline processing starts. The ethernet header is popped
before sending a packet to a L3 port.
For datapath ports that can receive L2 or L3 packets, the packet_type
becomes part of the flow key for datapath flows and is handled
appropriately in dpif-netdev.
In the 'else' branch in flow_put_on_pmd() function, the additional check
flow_equal(&match.flow, &netdev_flow->flow) was removed, as a) the dpcls
lookup is sufficient to uniquely identify a flow and b) it caused false
negatives because the flow in netdev->flow may not properly masked.
In dpif_netdev_flow_put() we now use the same method for constructing the
netdev_flow_key as the one used when adding the flow to the dplcs to make sure
these always match. The function netdev_flow_key_from_flow() used so far was
not only inefficient but sometimes caused mismatches and subsequent flow
update failures.
Signed-off-by: Lorand Jakab <lojakab at cisco.com>
Signed-off-by: Simon Horman <simon.horman at netronome.com>
Signed-off-by: Jiri Benc <jbenc at redhat.com>
Signed-off-by: Yi Yang <yi.y.yang at intel.com>
Signed-off-by: Jan Scheurich <jan.scheurich at ericsson.com>
Co-authored-by: Zoltan Balogh <zoltan.balogh at ericsson.com>
---
build-aux/extract-ofp-fields | 1 +
datapath/linux/compat/include/linux/openvswitch.h | 2 +
include/openvswitch/match.h | 1 +
include/openvswitch/meta-flow.h | 15 +-
lib/dpif-netdev.c | 46 +++---
lib/dpif-netlink.c | 2 +-
lib/dpif.c | 2 +-
lib/match.c | 25 +++
lib/meta-flow.c | 2 +
lib/netdev-vport.c | 8 +-
lib/netdev.h | 1 +
lib/odp-execute.c | 2 +
lib/odp-util.c | 184 ++++++++++++++++++----
lib/odp-util.h | 6 +-
lib/packets.h | 1 -
ofproto/ofproto-dpif-sflow.c | 1 +
ofproto/ofproto-dpif-upcall.c | 4 +-
ofproto/ofproto-dpif-xlate.c | 55 ++++++-
ofproto/ofproto-dpif.c | 6 +-
ofproto/tunnel.c | 3 +
tests/tunnel-push-pop-ipv6.at | 10 +-
tests/tunnel-push-pop.at | 12 +-
tests/tunnel.at | 28 ++--
23 files changed, 304 insertions(+), 113 deletions(-)
diff --git a/build-aux/extract-ofp-fields b/build-aux/extract-ofp-fields
index af7c69b..d5b8a82 100755
--- a/build-aux/extract-ofp-fields
+++ b/build-aux/extract-ofp-fields
@@ -39,6 +39,7 @@ FORMATTING = {"decimal": ("MFS_DECIMAL", 1, 8),
"TCP flags": ("MFS_TCP_FLAGS", 2, 2)}
PREREQS = {"none": "MFP_NONE",
+ "Ethernet": "MFP_ETHERNET",
"ARP": "MFP_ARP",
"VLAN VID": "MFP_VLAN_VID",
"IPv4": "MFP_IPV4",
diff --git a/datapath/linux/compat/include/linux/openvswitch.h b/datapath/linux/compat/include/linux/openvswitch.h
index d22102e..7990638 100644
--- a/datapath/linux/compat/include/linux/openvswitch.h
+++ b/datapath/linux/compat/include/linux/openvswitch.h
@@ -363,6 +363,8 @@ enum ovs_key_attr {
/* Only used within kernel data path. */
OVS_KEY_ATTR_TUNNEL_INFO, /* struct ovs_tunnel_info */
#endif
+
+ OVS_KEY_ATTR_PACKET_TYPE, /* be32 packet type */
__OVS_KEY_ATTR_MAX
};
diff --git a/include/openvswitch/match.h b/include/openvswitch/match.h
index 06fa04c..ce06919 100644
--- a/include/openvswitch/match.h
+++ b/include/openvswitch/match.h
@@ -115,6 +115,7 @@ void match_set_ct_ipv6_dst(struct match *, const struct in6_addr *);
void match_set_ct_ipv6_dst_masked(struct match *, const struct in6_addr *,
const struct in6_addr *);
+void match_set_packet_type(struct match *, ovs_be32 packet_type);
void match_set_skb_priority(struct match *, uint32_t skb_priority);
void match_set_dl_type(struct match *, ovs_be16);
void match_set_dl_src(struct match *, const struct eth_addr );
diff --git a/include/openvswitch/meta-flow.h b/include/openvswitch/meta-flow.h
index 11852d2..c284ec6 100644
--- a/include/openvswitch/meta-flow.h
+++ b/include/openvswitch/meta-flow.h
@@ -985,7 +985,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: MAC.
* Maskable: bitwise.
* Formatting: Ethernet.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read/write.
* NXM: NXM_OF_ETH_SRC(2) since v1.1.
* OXM: OXM_OF_ETH_SRC(4) since OF1.2 and v1.7.
@@ -1001,7 +1001,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: MAC.
* Maskable: bitwise.
* Formatting: Ethernet.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read/write.
* NXM: NXM_OF_ETH_DST(1) since v1.1.
* OXM: OXM_OF_ETH_DST(3) since OF1.2 and v1.7.
@@ -1020,7 +1020,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: be16.
* Maskable: no.
* Formatting: hexadecimal.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read-only.
* NXM: NXM_OF_ETH_TYPE(3) since v1.1.
* OXM: OXM_OF_ETH_TYPE(5) since OF1.2 and v1.7.
@@ -1050,7 +1050,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: be16.
* Maskable: bitwise.
* Formatting: hexadecimal.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read/write.
* NXM: NXM_OF_VLAN_TCI(4) since v1.1.
* OXM: none.
@@ -1066,7 +1066,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: be16 (low 12 bits).
* Maskable: no.
* Formatting: decimal.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read/write.
* NXM: none.
* OXM: none.
@@ -1084,7 +1084,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: be16 (low 12 bits).
* Maskable: bitwise.
* Formatting: decimal.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read/write.
* NXM: none.
* OXM: OXM_OF_VLAN_VID(6) since OF1.2 and v1.7.
@@ -1100,7 +1100,7 @@ enum OVS_PACKED_ENUM mf_field_id {
* Type: u8 (low 3 bits).
* Maskable: no.
* Formatting: decimal.
- * Prerequisites: none.
+ * Prerequisites: Ethernet.
* Access: read/write.
* NXM: none.
* OXM: none.
@@ -1808,6 +1808,7 @@ enum OVS_PACKED_ENUM mf_prereqs {
MFP_NONE,
/* L2 requirements. */
+ MFP_ETHERNET,
MFP_ARP,
MFP_VLAN_VID,
MFP_IPV4,
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index d215156..773f4dc 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -1864,24 +1864,6 @@ netdev_flow_key_clone(struct netdev_flow_key *dst,
offsetof(struct netdev_flow_key, mf) + src->len);
}
-/* Slow. */
-static void
-netdev_flow_key_from_flow(struct netdev_flow_key *dst,
- const struct flow *src)
-{
- struct dp_packet packet;
- uint64_t buf_stub[512 / 8];
-
- dp_packet_use_stub(&packet, buf_stub, sizeof buf_stub);
- pkt_metadata_from_flow(&packet.md, src);
- flow_compose(&packet, src);
- miniflow_extract(&packet, &dst->mf);
- dp_packet_uninit(&packet);
-
- dst->len = netdev_flow_key_size(miniflow_n_values(&dst->mf));
- dst->hash = 0; /* Not computed yet. */
-}
-
/* Initialize a netdev_flow_key 'mask' from 'match'. */
static inline void
netdev_flow_mask_init(struct netdev_flow_key *mask,
@@ -2357,7 +2339,7 @@ dp_netdev_flow_add(struct dp_netdev_pmd_thread *pmd,
cmap_insert(&pmd->flow_table, CONST_CAST(struct cmap_node *, &flow->node),
dp_netdev_flow_hash(&flow->ufid));
- if (OVS_UNLIKELY(VLOG_IS_DBG_ENABLED())) {
+ if (OVS_UNLIKELY(!VLOG_DROP_DBG((&upcall_rl)))) {
struct ds ds = DS_EMPTY_INITIALIZER;
struct ofpbuf key_buf, mask_buf;
struct odp_flow_key_parms odp_parms = {
@@ -2382,10 +2364,21 @@ dp_netdev_flow_add(struct dp_netdev_pmd_thread *pmd,
ds_put_cstr(&ds, ", actions:");
format_odp_actions(&ds, actions, actions_len);
- VLOG_DBG_RL(&upcall_rl, "%s", ds_cstr(&ds));
+ VLOG_DBG("%s", ds_cstr(&ds));
ofpbuf_uninit(&key_buf);
ofpbuf_uninit(&mask_buf);
+
+ /* Add a printout of the actual match installed. */
+ struct match m;
+ ds_clear(&ds);
+ ds_put_cstr(&ds, "flow match: ");
+ miniflow_expand(&flow->cr.flow.mf, &m.flow);
+ miniflow_expand(&flow->cr.mask->mf, &m.wc.masks);
+ match_format(&m, &ds, OFP_DEFAULT_PRIORITY);
+
+ VLOG_DBG("%s", ds_cstr(&ds));
+
ds_destroy(&ds);
}
@@ -2422,8 +2415,7 @@ flow_put_on_pmd(struct dp_netdev_pmd_thread *pmd,
error = ENOENT;
}
} else {
- if (put->flags & DPIF_FP_MODIFY
- && flow_equal(&match->flow, &netdev_flow->flow)) {
+ if (put->flags & DPIF_FP_MODIFY) {
struct dp_netdev_actions *new_actions;
struct dp_netdev_actions *old_actions;
@@ -2465,7 +2457,7 @@ static int
dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
{
struct dp_netdev *dp = get_dp_netdev(dpif);
- struct netdev_flow_key key;
+ struct netdev_flow_key key, mask;
struct dp_netdev_pmd_thread *pmd;
struct match match;
ovs_u128 ufid;
@@ -2494,9 +2486,10 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
}
/* Must produce a netdev_flow_key for lookup.
- * This interface is no longer performance critical, since it is not used
- * for upcall processing any more. */
- netdev_flow_key_from_flow(&key, &match.flow);
+ * Use the same method as employed to create the key when adding
+ * the flow to the dplcs to make sure they match. */
+ netdev_flow_mask_init(&mask, &match);
+ netdev_flow_key_init_masked(&key, &match.flow, &mask);
if (put->pmd_id == PMD_ID_NULL) {
if (cmap_count(&dp->poll_threads) == 0) {
@@ -4525,6 +4518,7 @@ emc_processing(struct dp_netdev_pmd_thread *pmd,
key->hash = dpif_netdev_packet_get_rss_hash(packet, &key->mf);
flow = emc_lookup(flow_cache, key);
+
if (OVS_LIKELY(flow)) {
dp_netdev_queue_batches(packet, flow, &key->mf, batches,
n_batches);
diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c
index 319808f..5c3cebd 100644
--- a/lib/dpif-netlink.c
+++ b/lib/dpif-netlink.c
@@ -1570,7 +1570,7 @@ dpif_netlink_encode_execute(int dp_ifindex, const struct dpif_execute *d_exec,
dp_packet_size(d_exec->packet));
key_ofs = nl_msg_start_nested(buf, OVS_PACKET_ATTR_KEY);
- odp_key_from_pkt_metadata(buf, &d_exec->packet->md);
+ odp_key_from_dp_packet(buf, d_exec->packet);
nl_msg_end_nested(buf, key_ofs);
nl_msg_put_unspec(buf, OVS_PACKET_ATTR_ACTIONS,
diff --git a/lib/dpif.c b/lib/dpif.c
index 44f63f8..fe6a986 100644
--- a/lib/dpif.c
+++ b/lib/dpif.c
@@ -1765,7 +1765,7 @@ log_execute_message(struct dpif *dpif, const struct dpif_execute *execute,
packet = ofp_packet_to_string(dp_packet_data(execute->packet),
dp_packet_size(execute->packet),
execute->packet->packet_type);
- odp_key_from_pkt_metadata(&md, &execute->packet->md);
+ odp_key_from_dp_packet(&md, execute->packet);
ds_put_format(&ds, "%s: %sexecute ",
dpif_name(dpif),
(subexecute ? "sub-"
diff --git a/lib/match.c b/lib/match.c
index 96b1850..4855c74 100644
--- a/lib/match.c
+++ b/lib/match.c
@@ -477,6 +477,13 @@ match_set_ct_ipv6_dst_masked(struct match *match, const struct in6_addr *dst,
}
void
+match_set_packet_type(struct match *match, ovs_be32 packet_type)
+{
+ match->flow.packet_type = packet_type;
+ match->wc.masks.packet_type = OVS_BE32_MAX;
+}
+
+void
match_set_dl_type(struct match *match, ovs_be16 dl_type)
{
match->wc.masks.dl_type = OVS_BE16_MAX;
@@ -1246,6 +1253,22 @@ match_format(const struct match *match, struct ds *s, int priority)
format_be16_masked(s, "ct_tp_dst", f->ct_tp_dst, wc->masks.ct_tp_dst);
}
+ if (wc->masks.packet_type) {
+ if (pt_ns_type_be(wc->masks.packet_type) == 0) {
+ ds_put_format(s, "packet_type=(%u,*),",
+ pt_ns(f->packet_type));
+ } else if (pt_ns_type_be(wc->masks.packet_type) == OVS_BE16_MAX) {
+ ds_put_format(s, "packet_type=(%u,0x%"PRIx16"),",
+ pt_ns(f->packet_type),
+ pt_ns_type(f->packet_type));
+ } else{
+ ds_put_format(s, "packet_type=(%u,0x%"PRIx16"/0x%"PRIx16"),",
+ pt_ns(f->packet_type),
+ pt_ns_type(f->packet_type),
+ pt_ns_type(wc->masks.packet_type));
+ }
+ }
+
if (wc->masks.dl_type) {
skip_type = true;
if (f->dl_type == htons(ETH_TYPE_IP)) {
@@ -1358,8 +1381,10 @@ match_format(const struct match *match, struct ds *s, int priority)
ntohs(wc->masks.vlans[i].tci));
}
}
+
format_eth_masked(s, "dl_src", f->dl_src, wc->masks.dl_src);
format_eth_masked(s, "dl_dst", f->dl_dst, wc->masks.dl_dst);
+
if (!skip_type && wc->masks.dl_type) {
ds_put_format(s, "%sdl_type=%s0x%04"PRIx16",",
colors.param, colors.end, ntohs(f->dl_type));
diff --git a/lib/meta-flow.c b/lib/meta-flow.c
index 6b97794..a963cce 100644
--- a/lib/meta-flow.c
+++ b/lib/meta-flow.c
@@ -404,6 +404,8 @@ mf_are_prereqs_ok__(const struct mf_field *mf, const struct flow *flow,
switch (mf->prereqs) {
case MFP_NONE:
return true;
+ case MFP_ETHERNET:
+ return is_ethernet(flow, wc);
case MFP_ARP:
return (flow->dl_type == htons(ETH_TYPE_ARP) ||
flow->dl_type == htons(ETH_TYPE_RARP));
diff --git a/lib/netdev-vport.c b/lib/netdev-vport.c
index 39093e8..84b9be3 100644
--- a/lib/netdev-vport.c
+++ b/lib/netdev-vport.c
@@ -97,9 +97,13 @@ netdev_vport_is_patch(const struct netdev *netdev)
bool
netdev_vport_is_layer3(const struct netdev *dev)
{
- const char *type = netdev_get_type(dev);
+ if (is_vport_class(netdev_get_class(dev))) {
+ struct netdev_vport *vport = netdev_vport_cast(dev);
+
+ return vport->tnl_cfg.is_layer3;
+ }
- return (!strcmp("lisp", type));
+ return false;
}
static bool
diff --git a/lib/netdev.h b/lib/netdev.h
index d6c07c1..416d2b7 100644
--- a/lib/netdev.h
+++ b/lib/netdev.h
@@ -100,6 +100,7 @@ struct netdev_tunnel_config {
bool csum;
bool dont_fragment;
+ bool is_layer3;
};
void netdev_run(void);
diff --git a/lib/odp-execute.c b/lib/odp-execute.c
index 08bc50a..d656334 100644
--- a/lib/odp-execute.c
+++ b/lib/odp-execute.c
@@ -375,6 +375,7 @@ odp_execute_set_action(struct dp_packet *packet, const struct nlattr *a)
break;
case OVS_KEY_ATTR_UNSPEC:
+ case OVS_KEY_ATTR_PACKET_TYPE:
case OVS_KEY_ATTR_ENCAP:
case OVS_KEY_ATTR_ETHERTYPE:
case OVS_KEY_ATTR_IN_PORT:
@@ -473,6 +474,7 @@ odp_execute_masked_set_action(struct dp_packet *packet,
break;
case OVS_KEY_ATTR_TUNNEL: /* Masked data not supported for tunnel. */
+ case OVS_KEY_ATTR_PACKET_TYPE:
case OVS_KEY_ATTR_UNSPEC:
case OVS_KEY_ATTR_CT_STATE:
case OVS_KEY_ATTR_CT_ZONE:
diff --git a/lib/odp-util.c b/lib/odp-util.c
index e24bd74..4d07d1c 100644
--- a/lib/odp-util.c
+++ b/lib/odp-util.c
@@ -173,6 +173,7 @@ ovs_key_attr_to_string(enum ovs_key_attr attr, char *namebuf, size_t bufsize)
case OVS_KEY_ATTR_MPLS: return "mpls";
case OVS_KEY_ATTR_DP_HASH: return "dp_hash";
case OVS_KEY_ATTR_RECIRC_ID: return "recirc_id";
+ case OVS_KEY_ATTR_PACKET_TYPE: return "packet_type";
case __OVS_KEY_ATTR_MAX:
default:
@@ -1947,6 +1948,7 @@ static const struct attr_len_tbl ovs_flow_key_attr_lens[OVS_KEY_ATTR_MAX + 1] =
[OVS_KEY_ATTR_CT_LABELS] = { .len = sizeof(struct ovs_key_ct_labels) },
[OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4] = { .len = sizeof(struct ovs_key_ct_tuple_ipv4) },
[OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6] = { .len = sizeof(struct ovs_key_ct_tuple_ipv6) },
+ [OVS_KEY_ATTR_PACKET_TYPE] = { .len = 4 },
};
/* Returns the correct length of the payload for a flow key attribute of the
@@ -2951,6 +2953,27 @@ format_odp_key_attr(const struct nlattr *a, const struct nlattr *ma,
}
break;
+ case OVS_KEY_ATTR_PACKET_TYPE: {
+ ovs_be32 packet_type = nl_attr_get_be32(a);
+ uint16_t ns = pt_ns(packet_type);
+ uint16_t ns_type = pt_ns_type(packet_type);
+
+ if (!is_exact) {
+ ovs_be32 mask = nl_attr_get_be32(ma);
+ uint16_t mask_ns_type = pt_ns_type(mask);
+
+ if (mask == 0) {
+ ds_put_format(ds, "ns=%u,id=*", ns);
+ } else {
+ ds_put_format(ds, "ns=%u,id=0x%"PRIx16"/0x%"PRIx16,
+ ns, ns_type, mask_ns_type);
+ }
+ } else {
+ ds_put_format(ds, "ns=%u,id=0x%"PRIx16, ns, ns_type);
+ }
+ break;
+ }
+
case OVS_KEY_ATTR_ETHERNET: {
const struct ovs_key_ethernet *mask = ma ? nl_attr_get(ma) : NULL;
const struct ovs_key_ethernet *key = nl_attr_get(a);
@@ -4417,7 +4440,8 @@ odp_flow_key_from_flow__(const struct odp_flow_key_parms *parms,
size_t encap[FLOW_MAX_VLAN_HEADERS] = {0};
size_t max_vlans;
const struct flow *flow = parms->flow;
- const struct flow *data = export_mask ? parms->mask : parms->flow;
+ const struct flow *mask = parms->mask;
+ const struct flow *data = export_mask ? mask : flow;
nl_msg_put_u32(buf, OVS_KEY_ATTR_PRIORITY, data->skb_priority);
@@ -4476,36 +4500,44 @@ odp_flow_key_from_flow__(const struct odp_flow_key_parms *parms,
nl_msg_put_odp_port(buf, OVS_KEY_ATTR_IN_PORT, data->in_port.odp_port);
}
- eth_key = nl_msg_put_unspec_uninit(buf, OVS_KEY_ATTR_ETHERNET,
- sizeof *eth_key);
- get_ethernet_key(data, eth_key);
+ if (export_mask || flow->packet_type != htonl(PT_ETH)) {
+ nl_msg_put_be32(buf, OVS_KEY_ATTR_PACKET_TYPE, data->packet_type);
+ }
if (OVS_UNLIKELY(parms->probe)) {
max_vlans = FLOW_MAX_VLAN_HEADERS;
} else {
max_vlans = MIN(parms->support.max_vlan_headers, flow_vlan_limit);
}
- for (int encaps = 0; encaps < max_vlans; encaps++) {
- ovs_be16 tpid = flow->vlans[encaps].tpid;
- if (flow->vlans[encaps].tci == htons(0)) {
- if (eth_type_vlan(flow->dl_type)) {
- /* If VLAN was truncated the tpid is in dl_type */
- tpid = flow->dl_type;
- } else {
- break;
+ /* Conditionally add L2 attributes for Ethernet packets */
+ if (flow->packet_type == htonl(PT_ETH)) {
+ eth_key = nl_msg_put_unspec_uninit(buf, OVS_KEY_ATTR_ETHERNET,
+ sizeof *eth_key);
+ get_ethernet_key(data, eth_key);
+
+ for (int encaps = 0; encaps < max_vlans; encaps++) {
+ ovs_be16 tpid = flow->vlans[encaps].tpid;
+
+ if (flow->vlans[encaps].tci == htons(0)) {
+ if (eth_type_vlan(flow->dl_type)) {
+ /* If VLAN was truncated the tpid is in dl_type */
+ tpid = flow->dl_type;
+ } else {
+ break;
+ }
}
- }
- if (export_mask) {
- nl_msg_put_be16(buf, OVS_KEY_ATTR_ETHERTYPE, OVS_BE16_MAX);
- } else {
- nl_msg_put_be16(buf, OVS_KEY_ATTR_ETHERTYPE, tpid);
- }
- nl_msg_put_be16(buf, OVS_KEY_ATTR_VLAN, data->vlans[encaps].tci);
- encap[encaps] = nl_msg_start_nested(buf, OVS_KEY_ATTR_ENCAP);
- if (flow->vlans[encaps].tci == htons(0)) {
- goto unencap;
+ if (export_mask) {
+ nl_msg_put_be16(buf, OVS_KEY_ATTR_ETHERTYPE, OVS_BE16_MAX);
+ } else {
+ nl_msg_put_be16(buf, OVS_KEY_ATTR_ETHERTYPE, tpid);
+ }
+ nl_msg_put_be16(buf, OVS_KEY_ATTR_VLAN, data->vlans[encaps].tci);
+ encap[encaps] = nl_msg_start_nested(buf, OVS_KEY_ATTR_ENCAP);
+ if (flow->vlans[encaps].tci == htons(0)) {
+ goto unencap;
+ }
}
}
@@ -4658,8 +4690,10 @@ odp_flow_key_from_mask(const struct odp_flow_key_parms *parms,
/* Generate ODP flow key from the given packet metadata */
void
-odp_key_from_pkt_metadata(struct ofpbuf *buf, const struct pkt_metadata *md)
+odp_key_from_dp_packet(struct ofpbuf *buf, const struct dp_packet *packet)
{
+ const struct pkt_metadata *md = &packet->md;
+
nl_msg_put_u32(buf, OVS_KEY_ATTR_PRIORITY, md->skb_priority);
if (flow_tnl_dst_is_set(&md->tunnel)) {
@@ -4701,18 +4735,29 @@ odp_key_from_pkt_metadata(struct ofpbuf *buf, const struct pkt_metadata *md)
if (md->in_port.odp_port != ODPP_NONE) {
nl_msg_put_odp_port(buf, OVS_KEY_ATTR_IN_PORT, md->in_port.odp_port);
}
+
+ /* Add OVS_KEY_ATTR_ETHERNET for non-Ethernet packets */
+ if (pt_ns(packet->packet_type) == OFPHTN_ETHERTYPE) {
+ nl_msg_put_be16(buf, OVS_KEY_ATTR_ETHERTYPE,
+ pt_ns_type_be(packet->packet_type));
+ }
}
/* Generate packet metadata from the given ODP flow key. */
void
-odp_key_to_pkt_metadata(const struct nlattr *key, size_t key_len,
- struct pkt_metadata *md)
+odp_key_to_dp_packet(const struct nlattr *key, size_t key_len,
+ struct dp_packet *packet)
{
+ static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
const struct nlattr *nla;
+ struct pkt_metadata *md = &packet->md;
+ ovs_be32 packet_type = htonl(PT_UNKNOWN);
+ ovs_be16 ethertype = 0;
size_t left;
uint32_t wanted_attrs = 1u << OVS_KEY_ATTR_PRIORITY |
1u << OVS_KEY_ATTR_SKB_MARK | 1u << OVS_KEY_ATTR_TUNNEL |
- 1u << OVS_KEY_ATTR_IN_PORT;
+ 1u << OVS_KEY_ATTR_IN_PORT | 1u << OVS_KEY_ATTR_ETHERTYPE |
+ 1u << OVS_KEY_ATTR_ETHERNET;
pkt_metadata_init(md, ODPP_NONE);
@@ -4792,14 +4837,32 @@ odp_key_to_pkt_metadata(const struct nlattr *key, size_t key_len,
md->in_port.odp_port = nl_attr_get_odp_port(nla);
wanted_attrs &= ~(1u << OVS_KEY_ATTR_IN_PORT);
break;
+ case OVS_KEY_ATTR_ETHERNET:
+ /* Presence of OVS_KEY_ATTR_ETHERNET indicates Ethernet packet. */
+ packet_type = htonl(PT_ETH);
+ wanted_attrs &= ~(1u << OVS_KEY_ATTR_ETHERNET);
+ break;
+ case OVS_KEY_ATTR_ETHERTYPE:
+ ethertype = nl_attr_get_be16(nla);
+ wanted_attrs &= ~(1u << OVS_KEY_ATTR_ETHERTYPE);
+ break;
default:
break;
}
if (!wanted_attrs) {
- return; /* Have everything. */
+ break; /* Have everything. */
}
}
+
+ if (packet_type == htonl(PT_ETH)){
+ packet->packet_type = htonl(PT_ETH);
+ } else if (packet_type == htonl(PT_UNKNOWN) && ethertype != 0) {
+ packet->packet_type = PACKET_TYPE_BE(OFPHTN_ETHERTYPE,
+ ntohs(ethertype));
+ } else {
+ VLOG_ERR_RL(&rl, "Packet without ETHERTYPE. Unknown packet_type.\n");
+ }
}
uint32_t
@@ -4963,7 +5026,21 @@ parse_ethertype(const struct nlattr *attrs[OVS_KEY_ATTR_MAX + 1],
*expected_attrs |= UINT64_C(1) << OVS_KEY_ATTR_ETHERTYPE;
} else {
if (!is_mask) {
- flow->dl_type = htons(FLOW_DL_TYPE_NONE);
+ /* Default ethertype for well-known L3 packets. */
+ /* XXX: Needed??? */
+ if (present_attrs & (UINT64_C(1) << OVS_KEY_ATTR_IPV4)) {
+ flow->dl_type = htons(ETH_TYPE_IP);
+ } else if (present_attrs & (UINT64_C(1) << OVS_KEY_ATTR_IPV6)) {
+ flow->dl_type = htons(ETH_TYPE_IPV6);
+ } else if (present_attrs & (UINT64_C(1) << OVS_KEY_ATTR_MPLS)) {
+ flow->dl_type = htons(ETH_TYPE_MPLS);
+ } else {
+ flow->dl_type = htons(FLOW_DL_TYPE_NONE);
+ }
+ } else if (src_flow->packet_type != htonl(PT_ETH)) {
+ /* dl_type is mandatory for non-Ethernet packets */
+ /* XXX: Needed??? */
+ flow->dl_type = htons(0xffff);
} else if (ntohs(src_flow->dl_type) < ETH_TYPE_MIN) {
/* See comments in odp_flow_key_from_flow__(). */
VLOG_ERR_RL(&rl, "mask expected for non-Ethernet II frame");
@@ -5405,23 +5482,37 @@ odp_flow_key_to_flow__(const struct nlattr *key, size_t key_len,
flow->in_port.odp_port = ODPP_NONE;
}
- /* Ethernet header. */
+ if (present_attrs & (UINT64_C(1) << OVS_KEY_ATTR_PACKET_TYPE)) {
+ flow->packet_type
+ = nl_attr_get_be32(attrs[OVS_KEY_ATTR_PACKET_TYPE]);
+ expected_attrs |= UINT64_C(1) << OVS_KEY_ATTR_PACKET_TYPE;
+ } else if (!is_mask) {
+ flow->packet_type = htonl(PT_ETH);
+ }
+
+ /* Check for Ethernet header. */
if (present_attrs & (UINT64_C(1) << OVS_KEY_ATTR_ETHERNET)) {
const struct ovs_key_ethernet *eth_key;
eth_key = nl_attr_get(attrs[OVS_KEY_ATTR_ETHERNET]);
put_ethernet_key(eth_key, flow);
- if (is_mask) {
- expected_attrs |= UINT64_C(1) << OVS_KEY_ATTR_ETHERNET;
+ if (!is_mask) {
+ flow->packet_type = htonl(PT_ETH);
}
- }
- if (!is_mask) {
expected_attrs |= UINT64_C(1) << OVS_KEY_ATTR_ETHERNET;
}
+ else if (present_attrs & (UINT64_C(1) << OVS_KEY_ATTR_ETHERTYPE)) {
+ ovs_be16 ethertype = nl_attr_get_be16(attrs[OVS_KEY_ATTR_ETHERTYPE]);
+ if (!is_mask) {
+ flow->packet_type = PACKET_TYPE_BE(OFPHTN_ETHERTYPE,
+ ntohs(ethertype));
+ }
+ expected_attrs |= UINT64_C(1) << OVS_KEY_ATTR_ETHERTYPE;
+ }
/* Get Ethertype or 802.1Q TPID or FLOW_DL_TYPE_NONE. */
if (!parse_ethertype(attrs, present_attrs, &expected_attrs, flow,
- src_flow)) {
+ src_flow)) {
return ODP_FIT_ERROR;
}
@@ -5712,6 +5803,29 @@ commit_set_ether_addr_action(const struct flow *flow, struct flow *base_flow,
}
static void
+commit_ether_action(const struct flow *flow, struct flow *base_flow,
+ struct ofpbuf *odp_actions, struct flow_wildcards *wc,
+ bool use_masked)
+{
+ if (flow->packet_type == htonl(PT_ETH)) {
+ if (base_flow->packet_type != htonl(PT_ETH)) {
+ odp_put_push_eth_action(odp_actions, &flow->dl_src, &flow->dl_dst);
+ base_flow->packet_type = flow->packet_type;
+ base_flow->dl_src = flow->dl_src;
+ base_flow->dl_dst = flow->dl_dst;
+ } else {
+ commit_set_ether_addr_action(flow, base_flow, odp_actions, wc,
+ use_masked);
+ }
+ } else {
+ if (base_flow->packet_type == htonl(PT_ETH)) {
+ odp_put_pop_eth_action(odp_actions);
+ base_flow->packet_type = flow->packet_type;
+ }
+ }
+}
+
+static void
commit_vlan_action(const struct flow* flow, struct flow *base,
struct ofpbuf *odp_actions, struct flow_wildcards *wc)
{
@@ -6165,7 +6279,7 @@ commit_odp_actions(const struct flow *flow, struct flow *base,
enum slow_path_reason slow1, slow2;
bool mpls_done = false;
- commit_set_ether_addr_action(flow, base, odp_actions, wc, use_masked);
+ commit_ether_action(flow, base, odp_actions, wc, use_masked);
/* Make packet a non-MPLS packet before committing L3/4 actions,
* which would otherwise do nothing. */
if (eth_type_mpls(base->dl_type) && !eth_type_mpls(flow->dl_type)) {
diff --git a/lib/odp-util.h b/lib/odp-util.h
index fafbb10..76dc8ab 100644
--- a/lib/odp-util.h
+++ b/lib/odp-util.h
@@ -233,9 +233,9 @@ uint32_t odp_flow_key_hash(const struct nlattr *, size_t);
/* Estimated space needed for metadata. */
enum { ODP_KEY_METADATA_SIZE = 9 * 8 };
-void odp_key_from_pkt_metadata(struct ofpbuf *, const struct pkt_metadata *);
-void odp_key_to_pkt_metadata(const struct nlattr *key, size_t key_len,
- struct pkt_metadata *md);
+void odp_key_from_dp_packet(struct ofpbuf *, const struct dp_packet *);
+void odp_key_to_dp_packet(const struct nlattr *key, size_t key_len,
+ struct dp_packet *md);
/* How well a kernel-provided flow key (a sequence of OVS_KEY_ATTR_*
* attributes) matches OVS userspace expectations.
diff --git a/lib/packets.h b/lib/packets.h
index 7dbf6dd..d53e0b7 100644
--- a/lib/packets.h
+++ b/lib/packets.h
@@ -132,7 +132,6 @@ pkt_metadata_init(struct pkt_metadata *md, odp_port_t port)
memset(md, 0, offsetof(struct pkt_metadata, in_port));
md->tunnel.ip_dst = 0;
md->tunnel.ipv6_dst = in6addr_any;
-
md->in_port.odp_port = port;
}
diff --git a/ofproto/ofproto-dpif-sflow.c b/ofproto/ofproto-dpif-sflow.c
index 43483d7..d9fddb1 100644
--- a/ofproto/ofproto-dpif-sflow.c
+++ b/ofproto/ofproto-dpif-sflow.c
@@ -1049,6 +1049,7 @@ sflow_read_set_action(const struct nlattr *attr,
case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4:
case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6:
case OVS_KEY_ATTR_UNSPEC:
+ case OVS_KEY_ATTR_PACKET_TYPE:
case __OVS_KEY_ATTR_MAX:
default:
break;
diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c
index 2191673..efd5d8a 100644
--- a/ofproto/ofproto-dpif-upcall.c
+++ b/ofproto/ofproto-dpif-upcall.c
@@ -1414,8 +1414,8 @@ handle_upcalls(struct udpif *udpif, struct upcall *upcalls,
op->dop.type = DPIF_OP_EXECUTE;
op->dop.u.execute.packet = CONST_CAST(struct dp_packet *, packet);
op->dop.u.execute.flow = upcall->flow;
- odp_key_to_pkt_metadata(upcall->key, upcall->key_len,
- &op->dop.u.execute.packet->md);
+ odp_key_to_dp_packet(upcall->key, upcall->key_len,
+ op->dop.u.execute.packet);
op->dop.u.execute.actions = upcall->odp_actions.data;
op->dop.u.execute.actions_len = upcall->odp_actions.size;
op->dop.u.execute.needs_help = (upcall->xout.slow & SLOW_ACTION) != 0;
diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
index bc3a310..993c2aa 100644
--- a/ofproto/ofproto-dpif-xlate.c
+++ b/ofproto/ofproto-dpif-xlate.c
@@ -165,6 +165,7 @@ struct xport {
bool may_enable; /* May be enabled in bonds. */
bool is_tunnel; /* Is a tunnel port. */
+ bool is_layer3; /* Is a layer 3 port. */
struct cfm *cfm; /* CFM handle or null. */
struct bfd *bfd; /* BFD handle or null. */
@@ -904,6 +905,7 @@ xlate_xport_set(struct xport *xport, odp_port_t odp_port,
xport->state = state;
xport->stp_port_no = stp_port_no;
xport->is_tunnel = is_tunnel;
+ xport->is_layer3 = netdev_vport_is_layer3(netdev);
xport->may_enable = may_enable;
xport->odp_port = odp_port;
@@ -2689,7 +2691,7 @@ xlate_normal(struct xlate_ctx *ctx)
/* Learn source MAC. */
bool is_grat_arp = is_gratuitous_arp(flow, wc);
- if (ctx->xin->allow_side_effects) {
+ if (ctx->xin->allow_side_effects && !in_port->is_layer3) {
update_learning_table(ctx, in_xbundle, flow->dl_src, vlan,
is_grat_arp);
}
@@ -3211,6 +3213,12 @@ build_tunnel_send(struct xlate_ctx *ctx, const struct xport *xport,
}
tnl_push_data.tnl_port = odp_to_u32(tunnel_odp_port);
tnl_push_data.out_port = odp_to_u32(out_dev->odp_port);
+
+ /* After tunnel header has been added, packet_type of flow and base_flow
+ * need to be set to PT_ETH. */
+ ctx->xin->flow.packet_type = htonl(PT_ETH);
+ ctx->base_flow.packet_type = htonl(PT_ETH);
+
odp_put_tnl_push_action(ctx->odp_actions, &tnl_push_data);
return 0;
}
@@ -3312,6 +3320,18 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
}
}
+ if (flow->packet_type == htonl(PT_ETH) && xport->is_layer3 ) {
+ /* Ethernet packet to L3 outport -> pop ethernet header. */
+ flow->packet_type = PACKET_TYPE_BE(OFPHTN_ETHERTYPE,
+ ntohs(flow->dl_type));
+ }
+ else if (flow->packet_type != htonl(PT_ETH) && !xport->is_layer3) {
+ /* L2 outport and non-ethernet packet_type -> add dummy eth header. */
+ flow->packet_type = htonl(PT_ETH);
+ flow->dl_dst = eth_addr_zero;
+ flow->dl_src = eth_addr_zero;
+ }
+
if (xport->peer) {
const struct xport *peer = xport->peer;
struct flow old_flow = ctx->xin->flow;
@@ -4210,6 +4230,12 @@ execute_controller_action(struct xlate_ctx *ctx, int len,
dp_packet_delete(packet);
return;
}
+
+ if (packet->packet_type != htonl(PT_ETH)) {
+ dp_packet_delete(packet);
+ return;
+ }
+
/* A packet sent by an action in a table-miss rule is considered an
* explicit table miss. OpenFlow before 1.3 doesn't have that concept so
* it will get translated back to OFPR_ACTION for those versions. */
@@ -6107,6 +6133,13 @@ xlate_wc_finish(struct xlate_ctx *ctx)
* use non-header fields as part of the cache. */
flow_wildcards_clear_non_packet_fields(ctx->wc);
+ /* Wildcard ethernet addresses if the original packet type was not
+ * Ethernet. */
+ if (ctx->xin->upcall_flow->packet_type != htonl(PT_ETH)) {
+ ctx->wc->masks.dl_dst = eth_addr_zero;
+ ctx->wc->masks.dl_src = eth_addr_zero;
+ }
+
/* ICMPv4 and ICMPv6 have 8-bit "type" and "code" fields. struct flow
* uses the low 8 bits of the 16-bit tp_src and tp_dst members to
* represent these fields. The datapath interface, on the other hand,
@@ -6332,6 +6365,21 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
}
ctx.wc->masks.tunnel.metadata.tab = flow->tunnel.metadata.tab;
+ /* Get the proximate input port of the packet. (If xin->frozen_state,
+ * flow->in_port is the ultimate input port of the packet.) */
+ struct xport *in_port = get_ofp_port(xbridge,
+ ctx.base_flow.in_port.ofp_port);
+
+ if (flow->packet_type != htonl(PT_ETH) && in_port && in_port->is_layer3 &&
+ ctx.table_id == 0) {
+ /* Add dummy Ethernet header to non-L2 packet if it's coming from a
+ * L3 port. So all packets will be L2 packets for lookup.
+ * The dl_type has already been set from the packet_type. */
+ flow->packet_type = htonl(PT_ETH);
+ flow->dl_src = eth_addr_zero;
+ flow->dl_dst = eth_addr_zero;
+ }
+
if (!xin->ofpacts && !ctx.rule) {
ctx.rule = rule_dpif_lookup_from_table(
ctx.xbridge->ofproto, ctx.xin->tables_version, flow, ctx.wc,
@@ -6351,11 +6399,6 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
xlate_report_table(&ctx, ctx.rule, ctx.table_id);
}
- /* Get the proximate input port of the packet. (If xin->frozen_state,
- * flow->in_port is the ultimate input port of the packet.) */
- struct xport *in_port = get_ofp_port(xbridge,
- ctx.base_flow.in_port.ofp_port);
-
/* Tunnel stats only for not-thawed packets. */
if (!xin->frozen_state && in_port && in_port->is_tunnel) {
if (ctx.xin->resubmit_stats) {
diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index dc5f004..14a9c1b 100644
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -139,7 +139,6 @@ struct ofport_dpif {
struct lldp *lldp; /* lldp, if any. */
bool may_enable; /* May be enabled in bonds. */
bool is_tunnel; /* This port is a tunnel. */
- bool is_layer3; /* This is a layer 3 port. */
long long int carrier_seq; /* Carrier status changes. */
struct ofport_dpif *peer; /* Peer if patch port. */
@@ -1829,7 +1828,6 @@ port_construct(struct ofport *port_)
port->qdscp = NULL;
port->n_qdscp = 0;
port->carrier_seq = netdev_get_carrier_resets(netdev);
- port->is_layer3 = netdev_vport_is_layer3(netdev);
if (netdev_vport_is_patch(netdev)) {
/* By bailing out here, we don't submit the port to the sFlow module
@@ -2856,7 +2854,7 @@ bundle_update(struct ofbundle *bundle)
bundle->floodable = true;
LIST_FOR_EACH (port, bundle_node, &bundle->ports) {
if (port->up.pp.config & OFPUTIL_PC_NO_FLOOD
- || port->is_layer3
+ || netdev_vport_is_layer3(port->up.netdev)
|| (bundle->ofproto->stp && !stp_forward_in_state(port->stp_state))
|| (bundle->ofproto->rstp && !rstp_forward_in_state(port->rstp_state))) {
bundle->floodable = false;
@@ -2905,7 +2903,7 @@ bundle_add_port(struct ofbundle *bundle, ofp_port_t ofp_port,
port->bundle = bundle;
ovs_list_push_back(&bundle->ports, &port->bundle_node);
if (port->up.pp.config & OFPUTIL_PC_NO_FLOOD
- || port->is_layer3
+ || netdev_vport_is_layer3(port->up.netdev)
|| (bundle->ofproto->stp && !stp_forward_in_state(port->stp_state))
|| (bundle->ofproto->rstp && !rstp_forward_in_state(port->rstp_state))) {
bundle->floodable = false;
diff --git a/ofproto/tunnel.c b/ofproto/tunnel.c
index e285d54..3d02005 100644
--- a/ofproto/tunnel.c
+++ b/ofproto/tunnel.c
@@ -381,6 +381,9 @@ tnl_wc_init(struct flow *flow, struct flow_wildcards *wc)
&& IP_ECN_is_ce(flow->tunnel.ip_tos)) {
wc->masks.nw_tos |= IP_ECN_MASK;
}
+ /* Match on packet_type for tunneled packets.*/
+ wc->masks.packet_type = OVS_BE32_MAX;
+
}
}
diff --git a/tests/tunnel-push-pop-ipv6.at b/tests/tunnel-push-pop-ipv6.at
index 16dc571..12c16de 100644
--- a/tests/tunnel-push-pop-ipv6.at
+++ b/tests/tunnel-push-pop-ipv6.at
@@ -88,28 +88,28 @@ AT_CHECK([tail -1 stdout], [0],
dnl Check VXLAN tunnel push
AT_CHECK([ovs-ofctl add-flow int-br action=2])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(4789),header(size=70,type=4,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x86dd),ipv6(src=2001:cafe::88,dst=2001:cafe::92,label=0,proto=17,tclass=0x0,hlimit=64),udp(src=0,dst=4789,csum=0xffff),vxlan(flags=0x8000000,vni=0x7b)),out_port(100))
])
dnl Check VXLAN tunnel push set tunnel id by flow and checksum
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_tunnel:124,4"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(4789),header(size=70,type=4,eth(dst=f8:bc:12:44:34:b7,src=aa:55:aa:55:00:00,dl_type=0x86dd),ipv6(src=2001:cafe::88,dst=2001:cafe::93,label=0,proto=17,tclass=0x0,hlimit=64),udp(src=0,dst=4789,csum=0xffff),vxlan(flags=0x8000000,vni=0x7c)),out_port(100))
])
dnl Check GRE tunnel push
AT_CHECK([ovs-ofctl add-flow int-br action=3])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(3),header(size=62,type=3,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x86dd),ipv6(src=2001:cafe::88,dst=2001:cafe::92,label=0,proto=47,tclass=0x0,hlimit=64),gre((flags=0x2000,proto=0x6558),key=0x1c8)),out_port(100))
])
dnl Check Geneve tunnel push
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_field:2001:cafe::92->tun_ipv6_dst,5"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(6081),header(size=70,type=5,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x86dd),ipv6(src=2001:cafe::88,dst=2001:cafe::92,label=0,proto=17,tclass=0x0,hlimit=64),udp(src=0,dst=6081,csum=0xffff),geneve(vni=0x7b)),out_port(100))
])
@@ -117,7 +117,7 @@ AT_CHECK([tail -1 stdout], [0],
dnl Check Geneve tunnel push with options
AT_CHECK([ovs-ofctl add-tlv-map int-br "{class=0xffff,type=0x80,len=4}->tun_metadata0"])
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_field:2001:cafe::92->tun_ipv6_dst,set_field:0xa->tun_metadata0,5"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(6081),header(size=78,type=5,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x86dd),ipv6(src=2001:cafe::88,dst=2001:cafe::92,label=0,proto=17,tclass=0x0,hlimit=64),udp(src=0,dst=6081,csum=0xffff),geneve(crit,vni=0x7b,options({class=0xffff,type=0x80,len=4,0xa}))),out_port(100))
])
diff --git a/tests/tunnel-push-pop.at b/tests/tunnel-push-pop.at
index 4eeac41..ddec1d0 100644
--- a/tests/tunnel-push-pop.at
+++ b/tests/tunnel-push-pop.at
@@ -105,35 +105,35 @@ AT_CHECK([tail -1 stdout], [0],
dnl Check VXLAN tunnel push
AT_CHECK([ovs-ofctl add-flow int-br action=2])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(4789),header(size=50,type=4,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.92,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=4789,csum=0x0),vxlan(flags=0x8000000,vni=0x7b)),out_port(100))
])
dnl Check VXLAN tunnel push set tunnel id by flow and checksum
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_tunnel:124,4"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(4789),header(size=50,type=4,eth(dst=f8:bc:12:44:34:b7,src=aa:55:aa:55:00:00,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.93,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=4789,csum=0xffff),vxlan(flags=0x8000000,vni=0x7c)),out_port(100))
])
dnl Check GRE tunnel push
AT_CHECK([ovs-ofctl add-flow int-br action=3])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(3),header(size=42,type=3,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.92,proto=47,tos=0,ttl=64,frag=0x4000),gre((flags=0x2000,proto=0x6558),key=0x1c8)),out_port(100))
])
dnl Check Geneve tunnel push
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_field:1.1.2.92->tun_dst,5"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(6081),header(size=50,type=5,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.92,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=6081,csum=0x0),geneve(vni=0x7b)),out_port(100))
])
dnl Check Geneve tunnel push with pkt-mark
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_tunnel:234,6"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: set(skb_mark(0x4d2)),tnl_push(tnl_port(6081),header(size=50,type=5,eth(dst=f8:bc:12:44:34:b7,src=aa:55:aa:55:00:00,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.93,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=6081,csum=0x0),geneve(vni=0xea)),out_port(100))
])
@@ -141,7 +141,7 @@ AT_CHECK([tail -1 stdout], [0],
dnl Check Geneve tunnel push with options
AT_CHECK([ovs-ofctl add-tlv-map int-br "{class=0xffff,type=0x80,len=4}->tun_metadata0"])
AT_CHECK([ovs-ofctl add-flow int-br "actions=set_field:1.1.2.92->tun_dst,set_field:0xa->tun_metadata0,5"])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(2),eth(src=f8:bc:12:44:34:b6,dst=aa:55:aa:55:00:01),eth_type(0x0800),ipv4(src=1.1.3.88,dst=1.1.3.112,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
AT_CHECK([tail -1 stdout], [0],
[Datapath actions: tnl_push(tnl_port(6081),header(size=58,type=5,eth(dst=f8:bc:12:44:34:b6,src=aa:55:aa:55:00:00,dl_type=0x0800),ipv4(src=1.1.2.88,dst=1.1.2.92,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=6081,csum=0x0),geneve(crit,vni=0x7b,options({class=0xffff,type=0x80,len=4,0xa}))),out_port(100))
])
diff --git a/tests/tunnel.at b/tests/tunnel.at
index b9e9e21..a887d30 100644
--- a/tests/tunnel.at
+++ b/tests/tunnel.at
@@ -82,28 +82,28 @@ AT_CHECK([ovs-appctl dpif/show | tail -n +3], [0], [dnl
dnl Tunnel CE and encapsulated packet CE
AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'tunnel(src=1.1.1.1,dst=2.2.2.2,tos=0x3,ttl=64,flags()),in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=6,tos=3,ttl=64,frag=no),tcp(src=8,dst=9)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=3,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=3,nw_frag=no
Datapath actions: 2
])
dnl Tunnel CE and encapsulated packet ECT(1)
AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'tunnel(src=1.1.1.1,dst=2.2.2.2,tos=0x3,ttl=64,flags()),in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=6,tos=1,ttl=64,frag=no),tcp(src=8,dst=9)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=1,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=1,nw_frag=no
Datapath actions: set(ipv4(tos=0x3/0x3)),2
])
dnl Tunnel CE and encapsulated packet ECT(2)
AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'tunnel(src=1.1.1.1,dst=2.2.2.2,tos=0x3,ttl=64,flags()),in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=6,tos=2,ttl=64,frag=no),tcp(src=8,dst=9)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=2,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=2,nw_frag=no
Datapath actions: set(ipv4(tos=0x3/0x3)),2
])
dnl Tunnel CE and encapsulated packet Non-ECT
AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'tunnel(src=1.1.1.1,dst=2.2.2.2,tos=0x3,ttl=64,flags()),in_port(1),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(src=192.168.0.1,dst=192.168.0.2,proto=6,tos=0,ttl=64,frag=no),tcp(src=8,dst=9)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=0,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=2.2.2.2,tun_tos=3,tun_flags=-df-csum-key,in_port=1,nw_ecn=0,nw_frag=no
Datapath actions: drop
])
OVS_VSWITCHD_STOP(["/dropping tunnel packet marked ECN CE but is not ECN capable/d"])
@@ -488,16 +488,16 @@ AT_CHECK([tail -1 stdout], [0],
])
dnl Option match
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=0,len=4,0xb}),flags(df|key)),in_port(6081),skb_mark(0),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=0,len=4,0xb}),flags(df|key)),in_port(6081),skb_mark(0),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata0=0xb/0xf,in_port=1,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata0=0xb/0xf,in_port=1,nw_frag=no
Datapath actions: 2
])
dnl Skip unknown option
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=0,len=4,0xb}{class=0xffff,type=2,len=4,0xc}),flags(df|key)),in_port(6081),skb_mark(0),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=0,len=4,0xb}{class=0xffff,type=2,len=4,0xc}),flags(df|key)),in_port(6081),skb_mark(0),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata0=0xb/0xf,in_port=1,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata0=0xb/0xf,in_port=1,nw_frag=no
Datapath actions: 2
])
@@ -529,9 +529,9 @@ AT_CHECK([ovs-ofctl del-tlv-map br0 "{class=0xffff,type=3,len=4}->tun_metadata3"
AT_CHECK([ovs-ofctl add-tlv-map br0 "{class=0xffff,type=3,len=8}->tun_metadata3"])
AT_CHECK([ovs-ofctl add-flow br0 tun_metadata3=0x1234567890abcdef,actions=2])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=3,len=8,0x1234567890abcdef}),flags(df|key)),in_port(6081),skb_mark(0),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=3,len=8,0x1234567890abcdef}),flags(df|key)),in_port(6081),skb_mark(0),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata3=0x1234567890abcdef,in_port=1,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata3=0x1234567890abcdef,in_port=1,nw_frag=no
Datapath actions: 2
])
@@ -638,15 +638,15 @@ AT_CHECK([ovs-ofctl dump-flows br0 | ofctl_strip | sort],
NXST_FLOW reply:
])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=0,len=4,0x12345678}),flags(df|key)),in_port(6081),skb_mark(0),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=0,len=4,0x12345678}),flags(df|key)),in_port(6081),skb_mark(0),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata0,tun_metadata1=NP,tun_metadata2=NP,in_port=1,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata0,tun_metadata1=NP,tun_metadata2=NP,in_port=1,nw_frag=no
Datapath actions: 2
])
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=1,len=0}),flags(df|key)),in_port(6081),skb_mark(0),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(0),tunnel(tun_id=0x0,src=1.1.1.1,dst=1.1.1.2,ttl=64,geneve({class=0xffff,type=1,len=0}),flags(df|key)),in_port(6081),skb_mark(0),eth(src=50:54:00:00:00:05,dst=50:54:00:00:00:07),eth_type(0x0800),ipv4(frag=no)'], [0], [stdout])
AT_CHECK([tail -2 stdout], [0],
- [Megaflow: recirc_id=0,ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata1,tun_metadata2=NP,in_port=1,nw_ecn=0,nw_frag=no
+ [Megaflow: recirc_id=0,packet_type=(0,0x0),ip,tun_id=0,tun_src=1.1.1.1,tun_dst=1.1.1.2,tun_tos=0,tun_flags=+df-csum+key,tun_metadata1,tun_metadata2=NP,in_port=1,nw_ecn=0,nw_frag=no
Datapath actions: set(tunnel(tun_id=0x0,dst=1.1.1.1,ttl=64,tp_dst=6081,geneve({class=0xffff,type=0x1,len=0}),flags(df|key))),6081
])
--
1.9.1
More information about the dev
mailing list