[ovs-dev] [PATCH v7] Extend OVS IPFIX exporter to export tunnel headers

Wenyu Zhang wenyuz at vmware.com
Thu Jul 31 11:12:30 UTC 2014


Extend IPFIX exporter to export tunnel headers when both input and output
of the port.
Add three other_config options in IPFIX table: enable-input-sampling,
enable-output-sampling and enable-tunnel-sampling, to control whether
sampling tunnel info, on which direction (input or output).
Insert sampling action before output action and the output tunnel port
is sent to datapath in the sampling action.
Make datapath collect output tunnel info and send it back to userpace
in upcall message with a new additional optional attribute.
Add a tunnel ports map to make the tunnel port lookup faster in sampling
upcalls in IPFIX exporter. Make the IPFIX exporter generate IPFIX template
sets with enterprise elements for the tunnel info, save the tunnel info
in IPFIX cache entries, and send IPFIX DATA with tunnel info.
Add flowDirection element in IPFIX templates.

Signed-off-by: Wenyu Zhang <wenyuz at vmware.com>
Acked-by: Romain Lenglet <rlenglet at vmware.com>
---
v2: Address Romain's comments
v3: Address Pravin's comments, make datapath sent all tunnel info,
    not only tunnel key to userspace.
v4: Address Pravin's comments, introduce a common function to get output
    tunnel info for all vports, remove duplicated codes.
    Rebase.
v5: Address Pravin's comments on v4, correct sparse errors, make a common
    function to setup tunnel info data for both input and output case. 
v6: Address Pravin's comments on v5, correct coding style issues in kernel.
    Rebase.
v7: Address Pravin's minor comments on v6.
    Rebase.
---
 datapath/actions.c                    |   18 ++
 datapath/datapath.c                   |   48 +++-
 datapath/datapath.h                   |    2 +
 datapath/flow.h                       |   59 +++--
 datapath/flow_netlink.c               |   57 ++++-
 datapath/flow_netlink.h               |    2 +
 datapath/vport-geneve.c               |   33 ++-
 datapath/vport-gre.c                  |   32 ++-
 datapath/vport-lisp.c                 |   37 ++-
 datapath/vport-vxlan.c                |   35 ++-
 datapath/vport.c                      |   43 ++++
 datapath/vport.h                      |   11 +
 include/linux/openvswitch.h           |   13 +
 lib/dpif-linux.c                      |    2 +
 lib/dpif.h                            |    1 +
 lib/odp-util.c                        |  183 +++++++++-----
 lib/odp-util.h                        |    2 +
 lib/packets.h                         |    9 +-
 ofproto/automake.mk                   |    3 +
 ofproto/ipfix-enterprise-entities.def |   19 ++
 ofproto/ofproto-dpif-ipfix.c          |  444 ++++++++++++++++++++++++++++++---
 ofproto/ofproto-dpif-ipfix.h          |   14 +-
 ofproto/ofproto-dpif-upcall.c         |   20 +-
 ofproto/ofproto-dpif-xlate.c          |   51 +++-
 ofproto/ofproto-dpif.c                |   20 ++
 ofproto/ofproto.h                     |    3 +
 ofproto/tunnel.c                      |    4 +
 tests/odp.at                          |    9 +-
 utilities/ovs-vsctl.8.in              |    8 +-
 vswitchd/bridge.c                     |    9 +
 vswitchd/vswitch.ovsschema            |    5 +-
 vswitchd/vswitch.xml                  |   27 ++
 32 files changed, 1040 insertions(+), 183 deletions(-)
 create mode 100644 ofproto/ipfix-enterprise-entities.def

diff --git a/datapath/actions.c b/datapath/actions.c
index 792c3a9..8dcd1b6 100644
--- a/datapath/actions.c
+++ b/datapath/actions.c
@@ -502,10 +502,12 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb,
 	struct dp_upcall_info upcall;
 	const struct nlattr *a;
 	int rem;
+	struct ovs_tunnel_info info;
 
 	upcall.cmd = OVS_PACKET_CMD_ACTION;
 	upcall.userdata = NULL;
 	upcall.portid = 0;
+	upcall.out_tun_info = NULL;
 
 	for (a = nla_data(attr), rem = nla_len(attr); rem > 0;
 		 a = nla_next(a, &rem)) {
@@ -517,7 +519,23 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb,
 		case OVS_USERSPACE_ATTR_PID:
 			upcall.portid = nla_get_u32(a);
 			break;
+
+		case OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT: {
+			/* Get out tunnel info. */
+			int err;
+			struct vport *vport;
+			vport = ovs_vport_rcu(dp, nla_get_u32(a));
+			if (vport && vport->ops &&
+			    vport->ops->get_out_tun_info) {
+				err = vport->ops->get_out_tun_info(vport, skb,
+								   &info);
+				if (err == 0)
+					upcall.out_tun_info = &info;
+			}
+			break;
 		}
+
+		} /* End of switch. */
 	}
 
 	return ovs_dp_upcall(dp, skb, &upcall);
diff --git a/datapath/datapath.c b/datapath/datapath.c
index 1185f60..2d920bf 100644
--- a/datapath/datapath.c
+++ b/datapath/datapath.c
@@ -273,6 +273,7 @@ void ovs_dp_process_packet_with_key(struct sk_buff *skb,
 		upcall.cmd = OVS_PACKET_CMD_MISS;
 		upcall.userdata = NULL;
 		upcall.portid = ovs_vport_find_upcall_portid(p, skb);
+		upcall.out_tun_info = NULL;
 		ovs_dp_upcall(dp, skb, &upcall);
 		consume_skb(skb);
 		stats_counter = &stats->n_missed;
@@ -386,6 +387,23 @@ static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 	return err;
 }
 
+static size_t tun_key_attr_size(void)
+{
+	/* Whenever adding new OVS_TUNNEL_KEY_ FIELDS, we should consider
+	 * updating this function.  */
+	return    nla_total_size(8)    /* OVS_TUNNEL_KEY_ATTR_ID */
+		+ nla_total_size(4)    /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
+		+ nla_total_size(4)    /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
+		+ nla_total_size(1)    /* OVS_TUNNEL_KEY_ATTR_TOS */
+		+ nla_total_size(1)    /* OVS_TUNNEL_KEY_ATTR_TTL */
+		+ nla_total_size(0)    /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
+		+ nla_total_size(0)    /* OVS_TUNNEL_KEY_ATTR_CSUM */
+		+ nla_total_size(2)    /* OVS_TUNNEL_KEY_ATTR_TP_SRC */
+		+ nla_total_size(2)    /* OVS_TUNNEL_KEY_ATTR_TP_DST */
+		+ nla_total_size(0)    /* OVS_TUNNEL_KEY_ATTR_OAM */
+		+ nla_total_size(256); /* OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS */
+}
+
 static size_t key_attr_size(void)
 {
 	/* Whenever adding new OVS_KEY_ FIELDS, we should consider
@@ -394,15 +412,7 @@ static size_t key_attr_size(void)
 
 	return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
 		+ nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
-		  + nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
-		  + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
-		  + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
-		  + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
-		  + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
-		  + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
-		  + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
-		  + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_OAM */
-		  + nla_total_size(256) /* OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS */
+		  + tun_key_attr_size()
 		+ nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
 		+ nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
 		+ nla_total_size(4)   /* OVS_KEY_ATTR_DP_HASH */
@@ -417,7 +427,7 @@ static size_t key_attr_size(void)
 		+ nla_total_size(28); /* OVS_KEY_ATTR_ND */
 }
 
-static size_t upcall_msg_size(const struct nlattr *userdata,
+static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
 			      unsigned int hdrlen)
 {
 	size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
@@ -425,8 +435,12 @@ static size_t upcall_msg_size(const struct nlattr *userdata,
 		+ nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 
 	/* OVS_PACKET_ATTR_USERDATA */
-	if (userdata)
-		size += NLA_ALIGN(userdata->nla_len);
+	if (upcall_info->userdata)
+		size += NLA_ALIGN(upcall_info->userdata->nla_len);
+
+	/* OVS_PACKET_ATTR_OUT_TUNNEL_KEY */
+	if (upcall_info->out_tun_info)
+		size += nla_total_size(tun_key_attr_size());
 
 	return size;
 }
@@ -486,7 +500,7 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 	else
 		hlen = skb->len;
 
-	len = upcall_msg_size(upcall_info->userdata, hlen);
+	len = upcall_msg_size(upcall_info, hlen);
 	user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
 	if (!user_skb) {
 		err = -ENOMEM;
@@ -507,6 +521,14 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 			  nla_len(upcall_info->userdata),
 			  nla_data(upcall_info->userdata));
 
+	if (upcall_info->out_tun_info) {
+		nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_OUT_TUNNEL_KEY);
+		err = ovs_nla_put_out_tunnel_key(user_skb,
+						 upcall_info->out_tun_info);
+		BUG_ON(err);
+		nla_nest_end(user_skb, nla);
+	}
+
 	/* Only reserve room for attribute header, packet data is added
 	 * in skb_zerocopy() */
 	if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
diff --git a/datapath/datapath.h b/datapath/datapath.h
index 92ef19f..a563761 100644
--- a/datapath/datapath.h
+++ b/datapath/datapath.h
@@ -118,11 +118,13 @@ struct ovs_skb_cb {
  * @portid: Netlink PID to which packet should be sent.  If @portid is 0 then no
  * packet is sent and the packet is accounted in the datapath's @n_lost
  * counter.
+ * @out_tun_info: If nonnull, becomes %OVS_PACKET_ATTR_OUT_TUNNEL_KEY.
  */
 struct dp_upcall_info {
 	const struct nlattr *userdata;
 	u32 portid;
 	u8 cmd;
+	const struct ovs_tunnel_info *out_tun_info;
 };
 
 /**
diff --git a/datapath/flow.h b/datapath/flow.h
index f6afa48..b94e1d4 100644
--- a/datapath/flow.h
+++ b/datapath/flow.h
@@ -39,8 +39,8 @@ struct sk_buff;
 
 /* Used to memset ovs_key_ipv4_tunnel padding. */
 #define OVS_TUNNEL_KEY_SIZE					\
-        (offsetof(struct ovs_key_ipv4_tunnel, ipv4_ttl) + 	\
-         FIELD_SIZEOF(struct ovs_key_ipv4_tunnel, ipv4_ttl))
+	(offsetof(struct ovs_key_ipv4_tunnel, tp_dst) +		\
+	 FIELD_SIZEOF(struct ovs_key_ipv4_tunnel, tp_dst))
 
 struct ovs_key_ipv4_tunnel {
 	__be64 tun_id;
@@ -49,6 +49,8 @@ struct ovs_key_ipv4_tunnel {
 	__be16 tun_flags;
 	u8   ipv4_tos;
 	u8   ipv4_ttl;
+	__be16 tp_src;
+	__be16 tp_dst;
 } __packed __aligned(4); /* Minimize padding. */
 
 struct ovs_tunnel_info {
@@ -66,27 +68,56 @@ struct ovs_tunnel_info {
 					FIELD_SIZEOF(struct sw_flow_key, tun_opts) - \
 					   opt_len)
 
-static inline void ovs_flow_tun_info_init(struct ovs_tunnel_info *tun_info,
-					 const struct iphdr *iph, __be64 tun_id,
-					 __be16 tun_flags,
-					 struct geneve_opt *opts,
-					 u8 opts_len)
+static inline void __ovs_flow_tun_info_init(struct ovs_tunnel_info *tun_info,
+					    __be32 saddr, __be32 daddr,
+					    u8 tos, u8 ttl,
+					    __be16 tp_src,
+					    __be16 tp_dst,
+					    __be64 tun_id,
+					    __be16 tun_flags,
+					    struct geneve_opt *opts,
+					    u8 opts_len)
 {
 	tun_info->tunnel.tun_id = tun_id;
-	tun_info->tunnel.ipv4_src = iph->saddr;
-	tun_info->tunnel.ipv4_dst = iph->daddr;
-	tun_info->tunnel.ipv4_tos = iph->tos;
-	tun_info->tunnel.ipv4_ttl = iph->ttl;
+	tun_info->tunnel.ipv4_src = saddr;
+	tun_info->tunnel.ipv4_dst = daddr;
+	tun_info->tunnel.ipv4_tos = tos;
+	tun_info->tunnel.ipv4_ttl = ttl;
 	tun_info->tunnel.tun_flags = tun_flags;
 
-	/* clear struct padding. */
-	memset((unsigned char *) &tun_info->tunnel + OVS_TUNNEL_KEY_SIZE, 0,
-	       sizeof(tun_info->tunnel) - OVS_TUNNEL_KEY_SIZE);
+	/* For the tunnel types on the top of IPsec, the tp_src and tp_dst of
+	 * the upper tunnel are used.
+	 * E.g: GRE over IPSEC, the tp_src and tp_port are zero.
+	 */
+	tun_info->tunnel.tp_src = tp_src;
+	tun_info->tunnel.tp_dst = tp_dst;
+
+	/* Clear struct padding. */
+	if (sizeof(tun_info->tunnel) != OVS_TUNNEL_KEY_SIZE)
+		memset((unsigned char *) &tun_info->tunnel +
+					 OVS_TUNNEL_KEY_SIZE,
+			0, sizeof(tun_info->tunnel) - OVS_TUNNEL_KEY_SIZE);
 
 	tun_info->options = opts;
 	tun_info->options_len = opts_len;
 }
 
+static inline void ovs_flow_tun_info_init(struct ovs_tunnel_info *tun_info,
+					  const struct iphdr *iph,
+					  __be16 tp_src,
+					  __be16 tp_dst,
+					  __be64 tun_id,
+					  __be16 tun_flags,
+					  struct geneve_opt *opts,
+					  u8 opts_len)
+{
+	__ovs_flow_tun_info_init(tun_info, iph->saddr, iph->daddr,
+				 iph->tos, iph->ttl,
+				 tp_src, tp_dst,
+				 tun_id, tun_flags,
+				 opts, opts_len);
+}
+
 struct sw_flow_key {
 	u8 tun_opts[255];
 	u8 tun_opts_len;
diff --git a/datapath/flow_netlink.c b/datapath/flow_netlink.c
index e1eadbb..475da7f 100644
--- a/datapath/flow_netlink.c
+++ b/datapath/flow_netlink.c
@@ -370,6 +370,8 @@ static int ipv4_tun_from_nlattr(const struct nlattr *attr,
 			[OVS_TUNNEL_KEY_ATTR_TTL] = 1,
 			[OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT] = 0,
 			[OVS_TUNNEL_KEY_ATTR_CSUM] = 0,
+			[OVS_TUNNEL_KEY_ATTR_TP_SRC] = sizeof(u16),
+			[OVS_TUNNEL_KEY_ATTR_TP_DST] = sizeof(u16),
 			[OVS_TUNNEL_KEY_ATTR_OAM] = 0,
 			[OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS] = -1,
 		};
@@ -417,6 +419,13 @@ static int ipv4_tun_from_nlattr(const struct nlattr *attr,
 		case OVS_TUNNEL_KEY_ATTR_CSUM:
 			tun_flags |= TUNNEL_CSUM;
 			break;
+		case OVS_TUNNEL_KEY_ATTR_TP_SRC:
+			SW_FLOW_KEY_PUT(match, tun_key.tp_src,
+					nla_get_be16(a), is_mask);
+			break;
+		case OVS_TUNNEL_KEY_ATTR_TP_DST:
+			SW_FLOW_KEY_PUT(match, tun_key.tp_dst,
+					nla_get_be16(a), is_mask);
 		case OVS_TUNNEL_KEY_ATTR_OAM:
 			tun_flags |= TUNNEL_OAM;
 			break;
@@ -498,17 +507,11 @@ static int ipv4_tun_from_nlattr(const struct nlattr *attr,
 	return 0;
 }
 
-static int ipv4_tun_to_nlattr(struct sk_buff *skb,
-			      const struct ovs_key_ipv4_tunnel *output,
-			      const struct geneve_opt *tun_opts,
-			      int swkey_tun_opts_len)
+static int __ipv4_tun_to_nlattr(struct sk_buff *skb,
+				const struct ovs_key_ipv4_tunnel *output,
+				const struct geneve_opt *tun_opts,
+				int swkey_tun_opts_len)
 {
-	struct nlattr *nla;
-
-	nla = nla_nest_start(skb, OVS_KEY_ATTR_TUNNEL);
-	if (!nla)
-		return -EMSGSIZE;
-
 	if (output->tun_flags & TUNNEL_KEY &&
 	    nla_put_be64(skb, OVS_TUNNEL_KEY_ATTR_ID, output->tun_id))
 		return -EMSGSIZE;
@@ -529,6 +532,12 @@ static int ipv4_tun_to_nlattr(struct sk_buff *skb,
 	if ((output->tun_flags & TUNNEL_CSUM) &&
 		nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_CSUM))
 		return -EMSGSIZE;
+	if (output->tp_src &&
+		nla_put_be16(skb, OVS_TUNNEL_KEY_ATTR_TP_SRC, output->tp_src))
+		return -EMSGSIZE;
+	if (output->tp_dst &&
+		nla_put_be16(skb, OVS_TUNNEL_KEY_ATTR_TP_DST, output->tp_dst))
+		return -EMSGSIZE;
 	if ((output->tun_flags & TUNNEL_OAM) &&
 		nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_OAM))
 		return -EMSGSIZE;
@@ -537,10 +546,37 @@ static int ipv4_tun_to_nlattr(struct sk_buff *skb,
 		    swkey_tun_opts_len, tun_opts))
 		return -EMSGSIZE;
 
+	return 0;
+}
+
+
+static int ipv4_tun_to_nlattr(struct sk_buff *skb,
+			      const struct ovs_key_ipv4_tunnel *output,
+			      const struct geneve_opt *tun_opts,
+			      int swkey_tun_opts_len)
+{
+	struct nlattr *nla;
+	int err;
+
+	nla = nla_nest_start(skb, OVS_KEY_ATTR_TUNNEL);
+	if (!nla)
+		return -EMSGSIZE;
+
+	err = __ipv4_tun_to_nlattr(skb, output, tun_opts, swkey_tun_opts_len);
+	if (err)
+		return err;
+
 	nla_nest_end(skb, nla);
 	return 0;
 }
 
+int ovs_nla_put_out_tunnel_key(struct sk_buff *skb,
+			       const struct ovs_tunnel_info *out_tun_info)
+{
+	return __ipv4_tun_to_nlattr(skb, &out_tun_info->tunnel,
+				    out_tun_info->options,
+				    out_tun_info->options_len);
+}
 
 static int metadata_from_nlattrs(struct sw_flow_match *match,  u64 *attrs,
 				 const struct nlattr **a, bool is_mask)
@@ -1632,6 +1668,7 @@ static int validate_userspace(const struct nlattr *attr)
 	static const struct nla_policy userspace_policy[OVS_USERSPACE_ATTR_MAX + 1] = {
 		[OVS_USERSPACE_ATTR_PID] = {.type = NLA_U32 },
 		[OVS_USERSPACE_ATTR_USERDATA] = {.type = NLA_UNSPEC },
+		[OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT] = {.type = NLA_U32 },
 	};
 	struct nlattr *a[OVS_USERSPACE_ATTR_MAX + 1];
 	int error;
diff --git a/datapath/flow_netlink.h b/datapath/flow_netlink.h
index 0c20e86..4d322fd 100644
--- a/datapath/flow_netlink.h
+++ b/datapath/flow_netlink.h
@@ -47,6 +47,8 @@ int ovs_nla_get_flow_metadata(struct sw_flow *flow,
 int ovs_nla_get_match(struct sw_flow_match *match,
 		      const struct nlattr *,
 		      const struct nlattr *);
+int ovs_nla_put_out_tunnel_key(struct sk_buff *,
+			       const struct ovs_tunnel_info *);
 
 int ovs_nla_copy_actions(const struct nlattr *attr,
 			 const struct sw_flow_key *key,
diff --git a/datapath/vport-geneve.c b/datapath/vport-geneve.c
index 99841d4..35f7da8 100644
--- a/datapath/vport-geneve.c
+++ b/datapath/vport-geneve.c
@@ -195,7 +195,9 @@ static int geneve_rcv(struct sock *sk, struct sk_buff *skb)
 		(geneveh->critical ? TUNNEL_CRIT_OPT : 0);
 
 	key = vni_to_tunnel_id(geneveh->vni);
-	ovs_flow_tun_info_init(&tun_info, ip_hdr(skb), key, flags,
+	ovs_flow_tun_info_init(&tun_info, ip_hdr(skb),
+				udp_hdr(skb)->source, udp_hdr(skb)->dest,
+				key, flags,
 				geneveh->options, opts_len);
 
 	ovs_vport_receive(vport_from_priv(geneve_port), skb, &tun_info);
@@ -439,11 +441,28 @@ static const char *geneve_get_name(const struct vport *vport)
 	return geneve_port->name;
 }
 
+static int geneve_get_out_tun_info(struct vport *vport, struct sk_buff *skb,
+				   struct ovs_tunnel_info *out_tun_info)
+{
+	struct geneve_port *geneve_port = geneve_vport(vport);
+
+	/*
+	 * Get tp_src and tp_dst, refert to geneve_build_header().
+	 */
+	return ovs_vport_get_out_tun_info(out_tun_info, OVS_CB(skb)->tun_info,
+					  ovs_dp_get_net(vport->dp),
+					  IPPROTO_UDP, skb->mark,
+					  vxlan_src_port(1, USHRT_MAX, skb),
+					  inet_sport(geneve_port->sock->sk));
+
+}
+
 const struct vport_ops ovs_geneve_vport_ops = {
-	.type		= OVS_VPORT_TYPE_GENEVE,
-	.create		= geneve_tnl_create,
-	.destroy	= geneve_tnl_destroy,
-	.get_name	= geneve_get_name,
-	.get_options	= geneve_get_options,
-	.send		= geneve_send,
+	.type			= OVS_VPORT_TYPE_GENEVE,
+	.create			= geneve_tnl_create,
+	.destroy		= geneve_tnl_destroy,
+	.get_name		= geneve_get_name,
+	.get_options		= geneve_get_options,
+	.send			= geneve_send,
+	.get_out_tun_info	= geneve_get_out_tun_info,
 };
diff --git a/datapath/vport-gre.c b/datapath/vport-gre.c
index d2a2602..37bb43e 100644
--- a/datapath/vport-gre.c
+++ b/datapath/vport-gre.c
@@ -110,7 +110,7 @@ static int gre_rcv(struct sk_buff *skb,
 		return PACKET_REJECT;
 
 	key = key_to_tunnel_id(tpi->key, tpi->seq);
-	ovs_flow_tun_info_init(&tun_info, ip_hdr(skb), key,
+	ovs_flow_tun_info_init(&tun_info, ip_hdr(skb), 0, 0, key,
 			       filter_tnl_flags(tpi->flags), NULL, 0);
 
 	ovs_vport_receive(vport, skb, &tun_info);
@@ -292,12 +292,21 @@ static int gre_send(struct vport *vport, struct sk_buff *skb)
 	return __send(vport, skb, hlen, 0, 0);
 }
 
+static int gre_get_out_tun_info(struct vport *vport, struct sk_buff *skb,
+				struct ovs_tunnel_info *out_tun_info)
+{
+	return ovs_vport_get_out_tun_info(out_tun_info, OVS_CB(skb)->tun_info,
+					  ovs_dp_get_net(vport->dp),
+					  IPPROTO_GRE, skb->mark, 0, 0);
+}
+
 const struct vport_ops ovs_gre_vport_ops = {
-	.type		= OVS_VPORT_TYPE_GRE,
-	.create		= gre_create,
-	.destroy	= gre_tnl_destroy,
-	.get_name	= gre_get_name,
-	.send		= gre_send,
+	.type			= OVS_VPORT_TYPE_GRE,
+	.create			= gre_create,
+	.destroy		= gre_tnl_destroy,
+	.get_name		= gre_get_name,
+	.send			= gre_send,
+	.get_out_tun_info	= gre_get_out_tun_info,
 };
 
 /* GRE64 vport. */
@@ -369,10 +378,11 @@ static int gre64_send(struct vport *vport, struct sk_buff *skb)
 }
 
 const struct vport_ops ovs_gre64_vport_ops = {
-	.type		= OVS_VPORT_TYPE_GRE64,
-	.create		= gre64_create,
-	.destroy	= gre64_tnl_destroy,
-	.get_name	= gre_get_name,
-	.send		= gre64_send,
+	.type			= OVS_VPORT_TYPE_GRE64,
+	.create			= gre64_create,
+	.destroy		= gre64_tnl_destroy,
+	.get_name		= gre_get_name,
+	.send			= gre64_send,
+	.get_out_tun_info	= gre_get_out_tun_info,
 };
 #endif
diff --git a/datapath/vport-lisp.c b/datapath/vport-lisp.c
index 81ecf92..ed4007b 100644
--- a/datapath/vport-lisp.c
+++ b/datapath/vport-lisp.c
@@ -245,7 +245,9 @@ static int lisp_rcv(struct sock *sk, struct sk_buff *skb)
 
 	/* Save outer tunnel values */
 	iph = ip_hdr(skb);
-	ovs_flow_tun_info_init(&tun_info, iph, key, TUNNEL_KEY, NULL, 0);
+	ovs_flow_tun_info_init(&tun_info, iph,
+			       udp_hdr(skb)->source, udp_hdr(skb)->dest,
+			       key, TUNNEL_KEY, NULL, 0);
 
 	/* Drop non-IP inner packets */
 	inner_iph = (struct iphdr *)(lisph + 1);
@@ -514,11 +516,32 @@ static const char *lisp_get_name(const struct vport *vport)
 	return lisp_port->name;
 }
 
+static int lisp_get_out_tun_info(struct vport *vport, struct sk_buff *skb,
+				 struct ovs_tunnel_info *out_tun_info)
+{
+	struct net *net = ovs_dp_get_net(vport->dp);
+	struct lisp_port *lisp_port = lisp_vport(vport);
+
+	if (skb->protocol != htons(ETH_P_IP) &&
+	    skb->protocol != htons(ETH_P_IPV6)) {
+		return -EINVAL;
+	}
+
+	/*
+	 * Get tp_src and tp_dst, refert to lisp_build_header().
+	 */
+	return ovs_vport_get_out_tun_info(out_tun_info, OVS_CB(skb)->tun_info,
+					  net, IPPROTO_UDP, skb->mark,
+					  htons(get_src_port(net, skb)),
+					  lisp_port->dst_port);
+}
+
 const struct vport_ops ovs_lisp_vport_ops = {
-	.type		= OVS_VPORT_TYPE_LISP,
-	.create		= lisp_tnl_create,
-	.destroy	= lisp_tnl_destroy,
-	.get_name	= lisp_get_name,
-	.get_options	= lisp_get_options,
-	.send		= lisp_send,
+	.type			= OVS_VPORT_TYPE_LISP,
+	.create			= lisp_tnl_create,
+	.destroy		= lisp_tnl_destroy,
+	.get_name		= lisp_get_name,
+	.get_options		= lisp_get_options,
+	.send			= lisp_send,
+	.get_out_tun_info	= lisp_get_out_tun_info,
 };
diff --git a/datapath/vport-vxlan.c b/datapath/vport-vxlan.c
index d93a844..8880104 100644
--- a/datapath/vport-vxlan.c
+++ b/datapath/vport-vxlan.c
@@ -68,7 +68,9 @@ static void vxlan_rcv(struct vxlan_sock *vs, struct sk_buff *skb, __be32 vx_vni)
 	/* Save outer tunnel values */
 	iph = ip_hdr(skb);
 	key = cpu_to_be64(ntohl(vx_vni) >> 8);
-	ovs_flow_tun_info_init(&tun_info, iph, key, TUNNEL_KEY, NULL, 0);
+	ovs_flow_tun_info_init(&tun_info, iph,
+			       udp_hdr(skb)->source, udp_hdr(skb)->dest,
+			       key, TUNNEL_KEY, NULL, 0);
 
 	ovs_vport_receive(vport, skb, &tun_info);
 }
@@ -187,6 +189,24 @@ error:
 	return err;
 }
 
+static int vxlan_get_out_tun_info(struct vport *vport, struct sk_buff *skb,
+				  struct ovs_tunnel_info *out_tun_info)
+{
+	struct net *net = ovs_dp_get_net(vport->dp);
+	struct vxlan_port *vxlan_port = vxlan_vport(vport);
+	__be16 dst_port = inet_sport(vxlan_port->vs->sock->sk);
+	__be16 src_port;
+	int port_min;
+	int port_max;
+
+	inet_get_local_port_range(net, &port_min, &port_max);
+	src_port = vxlan_src_port(port_min, port_max, skb);
+
+	return ovs_vport_get_out_tun_info(out_tun_info, OVS_CB(skb)->tun_info,
+					  net, IPPROTO_UDP, skb->mark,
+					  src_port, dst_port);
+}
+
 static const char *vxlan_get_name(const struct vport *vport)
 {
 	struct vxlan_port *vxlan_port = vxlan_vport(vport);
@@ -194,10 +214,11 @@ static const char *vxlan_get_name(const struct vport *vport)
 }
 
 const struct vport_ops ovs_vxlan_vport_ops = {
-	.type		= OVS_VPORT_TYPE_VXLAN,
-	.create		= vxlan_tnl_create,
-	.destroy	= vxlan_tnl_destroy,
-	.get_name	= vxlan_get_name,
-	.get_options	= vxlan_get_options,
-	.send		= vxlan_tnl_send,
+	.type			= OVS_VPORT_TYPE_VXLAN,
+	.create			= vxlan_tnl_create,
+	.destroy		= vxlan_tnl_destroy,
+	.get_name		= vxlan_get_name,
+	.get_options		= vxlan_get_options,
+	.send			= vxlan_tnl_send,
+	.get_out_tun_info	= vxlan_get_out_tun_info,
 };
diff --git a/datapath/vport.c b/datapath/vport.c
index 5d250aae..8f65ed0 100644
--- a/datapath/vport.c
+++ b/datapath/vport.c
@@ -568,3 +568,46 @@ void ovs_vport_deferred_free(struct vport *vport)
 
 	call_rcu(&vport->rcu, free_vport_rcu);
 }
+
+int ovs_vport_get_out_tun_info(struct ovs_tunnel_info *out_tun_info,
+			       const struct ovs_tunnel_info *tun_info,
+			       struct net *net,
+			       u8 ipproto,
+			       u32 skb_mark,
+			       __be16 tp_src,
+			       __be16 tp_dst)
+{
+	struct rtable *rt;
+	__be32 saddr;
+
+	if (unlikely(!tun_info))
+		return -EINVAL;
+
+	saddr = tun_info->tunnel.ipv4_src;
+	/* Route lookup to get srouce IP address: saddr.
+	 * The process may need to be changed if the corresponding process
+	 * in vports ops changed.
+	 */
+	rt = find_route(net,
+			&saddr,
+			tun_info->tunnel.ipv4_dst,
+			ipproto,
+			tun_info->tunnel.ipv4_tos,
+			skb_mark);
+	if (IS_ERR(rt))
+		return PTR_ERR(rt);
+
+	ip_rt_put(rt);
+
+	/* Generate out_tun_info based on tun_info, saddr, tp_src and tp_dst*/
+	__ovs_flow_tun_info_init(out_tun_info, saddr, tun_info->tunnel.ipv4_dst,
+				 tun_info->tunnel.ipv4_tos,
+				 tun_info->tunnel.ipv4_ttl,
+				 tp_src, tp_dst,
+				 tun_info->tunnel.tun_id,
+				 tun_info->tunnel.tun_flags,
+				 tun_info->options,
+				 tun_info->options_len);
+
+	return 0;
+}
diff --git a/datapath/vport.h b/datapath/vport.h
index bdd9a89..7887549 100644
--- a/datapath/vport.h
+++ b/datapath/vport.h
@@ -57,6 +57,13 @@ u32 ovs_vport_find_upcall_portid(const struct vport *, struct sk_buff *);
 
 int ovs_vport_send(struct vport *, struct sk_buff *);
 
+int ovs_vport_get_out_tun_info(struct ovs_tunnel_info *out_tun_info,
+			       const struct ovs_tunnel_info *tun_info,
+			       struct net *net,
+			       u8 ipproto,
+			       u32 skb_mark,
+			       __be16 tp_src,
+			       __be16 tp_dst);
 /* The following definitions are for implementers of vport devices: */
 
 struct vport_err_stats {
@@ -150,6 +157,7 @@ struct vport_parms {
  * @get_name: Get the device's name.
  * @send: Send a packet on the device.  Returns the length of the packet sent,
  * zero for dropped packets or negative for error.
+ * @get_out_tun_info: Get the output tunnel 5-tuple and other info for a packet.
  */
 struct vport_ops {
 	enum ovs_vport_type type;
@@ -165,6 +173,9 @@ struct vport_ops {
 	const char *(*get_name)(const struct vport *);
 
 	int (*send)(struct vport *, struct sk_buff *);
+
+	int (*get_out_tun_info)(struct vport *, struct sk_buff *,
+				struct ovs_tunnel_info *);
 };
 
 enum vport_err_type {
diff --git a/include/linux/openvswitch.h b/include/linux/openvswitch.h
index 271a14e..49fa188 100644
--- a/include/linux/openvswitch.h
+++ b/include/linux/openvswitch.h
@@ -180,6 +180,11 @@ enum ovs_packet_cmd {
  * notification if the %OVS_ACTION_ATTR_USERSPACE action specified an
  * %OVS_USERSPACE_ATTR_USERDATA attribute, with the same length and content
  * specified there.
+ * @OVS_PACKET_ATTR_OUT_TUNNEL_KEY: Present for an %OVS_PACKET_CMD_ACTION
+ * notification if the %OVS_ACTION_ATTR_USERSPACE action specified an
+ * %OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT attribute, which is sent only if the
+ * output port is actually a tunnel port. Contains the output tunnel key
+ * extracted from the packet as nested %OVS_TUNNEL_KEY_ATTR_* attributes.
  *
  * These attributes follow the &struct ovs_header within the Generic Netlink
  * payload for %OVS_PACKET_* commands.
@@ -190,6 +195,8 @@ enum ovs_packet_attr {
 	OVS_PACKET_ATTR_KEY,         /* Nested OVS_KEY_ATTR_* attributes. */
 	OVS_PACKET_ATTR_ACTIONS,     /* Nested OVS_ACTION_ATTR_* attributes. */
 	OVS_PACKET_ATTR_USERDATA,    /* OVS_ACTION_ATTR_USERSPACE arg. */
+	OVS_PACKET_ATTR_OUT_TUNNEL_KEY,  /* Nested OVS_TUNNEL_KEY_ATTR_*
+					    attributes. */
 	__OVS_PACKET_ATTR_MAX
 };
 
@@ -340,6 +347,8 @@ enum ovs_tunnel_key_attr {
 	OVS_TUNNEL_KEY_ATTR_TTL,		/* u8 Tunnel IP TTL. */
 	OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT,	/* No argument, set DF. */
 	OVS_TUNNEL_KEY_ATTR_CSUM,		/* No argument. CSUM packet. */
+	OVS_TUNNEL_KEY_ATTR_TP_SRC,		/* u16 src Transport Port. */
+	OVS_TUNNEL_KEY_ATTR_TP_DST,		/* u16 dst Transport Port. */
 	OVS_TUNNEL_KEY_ATTR_OAM,		/* No argument, OAM frame. */
 	OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS,	/* Array of Geneve options */
 	__OVS_TUNNEL_KEY_ATTR_MAX
@@ -506,11 +515,15 @@ enum ovs_sample_attr {
  * message should be sent.  Required.
  * @OVS_USERSPACE_ATTR_USERDATA: If present, its variable-length argument is
  * copied to the %OVS_PACKET_CMD_ACTION message as %OVS_PACKET_ATTR_USERDATA.
+ * @OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT: If present, u32 output port id to get
+ * tunnel info.
  */
 enum ovs_userspace_attr {
 	OVS_USERSPACE_ATTR_UNSPEC,
 	OVS_USERSPACE_ATTR_PID,	      /* u32 Netlink PID to receive upcalls. */
 	OVS_USERSPACE_ATTR_USERDATA,  /* Optional user-specified cookie. */
+	OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT,  /* Optional, u32 output port id
+					      * to get tunnel info. */
 	__OVS_USERSPACE_ATTR_MAX
 };
 
diff --git a/lib/dpif-linux.c b/lib/dpif-linux.c
index b98413d..fdfe36b 100644
--- a/lib/dpif-linux.c
+++ b/lib/dpif-linux.c
@@ -1709,6 +1709,7 @@ parse_odp_packet(struct ofpbuf *buf, struct dpif_upcall *upcall,
 
         /* OVS_PACKET_CMD_ACTION only. */
         [OVS_PACKET_ATTR_USERDATA] = { .type = NL_A_UNSPEC, .optional = true },
+        [OVS_PACKET_ATTR_OUT_TUNNEL_KEY] = { .type = NL_A_NESTED, .optional = true },
     };
 
     struct ovs_header *ovs_header;
@@ -1743,6 +1744,7 @@ parse_odp_packet(struct ofpbuf *buf, struct dpif_upcall *upcall,
                              nl_attr_get(a[OVS_PACKET_ATTR_KEY]));
     upcall->key_len = nl_attr_get_size(a[OVS_PACKET_ATTR_KEY]);
     upcall->userdata = a[OVS_PACKET_ATTR_USERDATA];
+    upcall->out_tun_key = a[OVS_PACKET_ATTR_OUT_TUNNEL_KEY];
 
     /* Allow overwriting the netlink attribute header without reallocating. */
     ofpbuf_use_stub(&upcall->packet,
diff --git a/lib/dpif.h b/lib/dpif.h
index 8d8e43a..479a971 100644
--- a/lib/dpif.h
+++ b/lib/dpif.h
@@ -669,6 +669,7 @@ struct dpif_upcall {
 
     /* DPIF_UC_ACTION only. */
     struct nlattr *userdata;    /* Argument to OVS_ACTION_ATTR_USERSPACE. */
+    struct nlattr *out_tun_key;    /* Argument to OVS_ACTION_ATTR_USERSPACE. */
 };
 
 typedef void exec_upcall_cb(struct dpif *, struct dpif_upcall *,
diff --git a/lib/odp-util.c b/lib/odp-util.c
index 162d85a..49d3b0f 100644
--- a/lib/odp-util.c
+++ b/lib/odp-util.c
@@ -261,9 +261,12 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
         [OVS_USERSPACE_ATTR_PID] = { .type = NL_A_U32 },
         [OVS_USERSPACE_ATTR_USERDATA] = { .type = NL_A_UNSPEC,
                                           .optional = true },
+        [OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT] = { .type = NL_A_U32,
+                                                 .optional = true },
     };
     struct nlattr *a[ARRAY_SIZE(ovs_userspace_policy)];
     const struct nlattr *userdata_attr;
+    const struct nlattr *tunnel_out_port_attr;
 
     if (!nl_parse_nested(attr, ovs_userspace_policy, a, ARRAY_SIZE(a))) {
         ds_put_cstr(ds, "userspace(error)");
@@ -314,7 +317,8 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
                               cookie.flow_sample.obs_point_id);
             } else if (userdata_len >= sizeof cookie.ipfix
                        && cookie.type == USER_ACTION_COOKIE_IPFIX) {
-                ds_put_format(ds, ",ipfix");
+                ds_put_format(ds, ",ipfix(output_port=%"PRIu32")",
+                              cookie.ipfix.output_odp_port);
             } else {
                 userdata_unspec = true;
             }
@@ -330,6 +334,12 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
         }
     }
 
+    tunnel_out_port_attr = a[OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT];
+    if (tunnel_out_port_attr) {
+        ds_put_format(ds, ",tunnel_out_port=%"PRIu32,
+                      nl_attr_get_u32(tunnel_out_port_attr));
+    }
+
     ds_put_char(ds, ')');
 }
 
@@ -506,50 +516,36 @@ format_odp_actions(struct ds *ds, const struct nlattr *actions,
     }
 }
 
+/* Separate out parse_odp_userspace_action() function. */
 static int
-parse_odp_action(const char *s, const struct simap *port_names,
-                 struct ofpbuf *actions)
+parse_odp_userspace_action(const char *s, struct ofpbuf *actions)
 {
-    {
-        uint32_t port;
-        int n;
+    uint32_t pid;
+    union user_action_cookie cookie;
+    struct ofpbuf buf;
+    uint32_t tunnel_out_port;
+    int n = -1;
+    void *user_data = NULL;
+    size_t user_data_size = 0;
 
-        if (ovs_scan(s, "%"SCNi32"%n", &port, &n)) {
-            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, port);
-            return n;
-        }
-    }
-
-    if (port_names) {
-        int len = strcspn(s, delimiters);
-        struct simap_node *node;
-
-        node = simap_find_len(port_names, s, len);
-        if (node) {
-            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, node->data);
-            return len;
-        }
+    if (!ovs_scan(s, "userspace(pid=%"SCNi32"%n", &pid, &n)){
+        return -EINVAL;
     }
 
     {
-        uint32_t pid;
         uint32_t output;
         uint32_t probability;
         uint32_t collector_set_id;
         uint32_t obs_domain_id;
         uint32_t obs_point_id;
         int vid, pcp;
-        int n = -1;
-
-        if (ovs_scan(s, "userspace(pid=%"SCNi32")%n", &pid, &n)) {
-            odp_put_userspace_action(pid, NULL, 0, actions);
-            return n;
-        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",sFlow(vid=%i,"
-                            "pcp=%i,output=%"SCNi32"))%n",
-                            &pid, &vid, &pcp, &output, &n)) {
-            union user_action_cookie cookie;
+        int n1 = -1;
+        if (ovs_scan(&s[n], ",sFlow(vid=%i,"
+                     "pcp=%i,output=%"SCNi32")%n",
+                     &vid, &pcp, &output, &n1)) {
             uint16_t tci;
 
+            n += n1;
             tci = vid | (pcp << VLAN_PCP_SHIFT);
             if (tci) {
                 tci |= VLAN_CFI;
@@ -558,14 +554,13 @@ parse_odp_action(const char *s, const struct simap *port_names,
             cookie.type = USER_ACTION_COOKIE_SFLOW;
             cookie.sflow.vlan_tci = htons(tci);
             cookie.sflow.output = output;
-            odp_put_userspace_action(pid, &cookie, sizeof cookie.sflow,
-                                     actions);
-            return n;
-        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",slow_path%n",
-                            &pid, &n)) {
-            union user_action_cookie cookie;
+            user_data = &cookie;
+            user_data_size = sizeof cookie.sflow;
+        } else if (ovs_scan(&s[n], ",slow_path%n",
+                            &n1)) {
             int res;
 
+            n += n1;
             cookie.type = USER_ACTION_COOKIE_SLOW_PATH;
             cookie.slow_path.unused = 0;
             cookie.slow_path.reason = 0;
@@ -576,53 +571,91 @@ parse_odp_action(const char *s, const struct simap *port_names,
                 return res;
             }
             n += res;
-            if (s[n] != ')') {
-                return -EINVAL;
-            }
-            n++;
 
-            odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path,
-                                     actions);
-            return n;
-        } else if (ovs_scan(s, "userspace(pid=%"SCNi32","
-                            "flow_sample(probability=%"SCNi32","
+            user_data = &cookie;
+            user_data_size = sizeof cookie.slow_path;
+        } else if (ovs_scan(&s[n], ",flow_sample(probability=%"SCNi32","
                             "collector_set_id=%"SCNi32","
                             "obs_domain_id=%"SCNi32","
-                            "obs_point_id=%"SCNi32"))%n",
-                            &pid, &probability, &collector_set_id,
-                            &obs_domain_id, &obs_point_id, &n)) {
-            union user_action_cookie cookie;
+                            "obs_point_id=%"SCNi32")%n",
+                            &probability, &collector_set_id,
+                            &obs_domain_id, &obs_point_id, &n1)) {
+            n += n1;
 
             cookie.type = USER_ACTION_COOKIE_FLOW_SAMPLE;
             cookie.flow_sample.probability = probability;
             cookie.flow_sample.collector_set_id = collector_set_id;
             cookie.flow_sample.obs_domain_id = obs_domain_id;
             cookie.flow_sample.obs_point_id = obs_point_id;
-            odp_put_userspace_action(pid, &cookie, sizeof cookie.flow_sample,
-                                     actions);
-            return n;
-        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",ipfix)%n", &pid, &n)) {
-            union user_action_cookie cookie;
-
+            user_data = &cookie;
+            user_data_size = sizeof cookie.flow_sample;
+        } else if (ovs_scan(&s[n], ",ipfix(output_port=%"SCNi32")%n",
+                            &output, &n1) ) {
+            n += n1;
             cookie.type = USER_ACTION_COOKIE_IPFIX;
-            odp_put_userspace_action(pid, &cookie, sizeof cookie.ipfix,
-                                     actions);
-            return n;
-        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",userdata(%n",
-                            &pid, &n)) {
-            struct ofpbuf buf;
+            cookie.ipfix.output_odp_port = output;
+            user_data = &cookie;
+            user_data_size = sizeof cookie.ipfix;
+        } else if (ovs_scan(&s[n], ",userdata(%n",
+                            &n1)) {
             char *end;
 
+            n += n1;
             ofpbuf_init(&buf, 16);
             end = ofpbuf_put_hex(&buf, &s[n], NULL);
-            if (end[0] == ')' && end[1] == ')') {
-                odp_put_userspace_action(pid, ofpbuf_data(&buf), ofpbuf_size(&buf), actions);
-                ofpbuf_uninit(&buf);
-                return (end + 2) - s;
+            if (end[0] != ')') {
+                return -EINVAL;
             }
+            user_data = ofpbuf_data(&buf);
+            user_data_size = ofpbuf_size(&buf);
+            n = (end + 1) - s;
         }
     }
 
+    {
+        int n1 = -1;
+        if (ovs_scan(&s[n], ",tunnel_out_port=%"SCNi32")%n",
+                     &tunnel_out_port, &n1)) {
+            odp_put_userspace_action(pid, user_data, user_data_size, &tunnel_out_port, actions);
+            return n + n1;
+        } else if (s[n] == ')') {
+            odp_put_userspace_action(pid, user_data, user_data_size, NULL, actions);
+            return n + 1;
+        }
+    }
+
+    return -EINVAL;
+}
+
+static int
+parse_odp_action(const char *s, const struct simap *port_names,
+                 struct ofpbuf *actions)
+{
+    {
+        uint32_t port;
+        int n;
+
+        if (ovs_scan(s, "%"SCNi32"%n", &port, &n)) {
+            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, port);
+            return n;
+        }
+    }
+
+    if (port_names) {
+        int len = strcspn(s, delimiters);
+        struct simap_node *node;
+
+        node = simap_find_len(port_names, s, len);
+        if (node) {
+            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, node->data);
+            return len;
+        }
+    }
+
+    if (!strncmp(s, "userspace(", 10)) {
+        return parse_odp_userspace_action(s, actions);
+    }
+
     if (!strncmp(s, "set(", 4)) {
         size_t start_ofs;
         int retval;
@@ -832,6 +865,8 @@ tunnel_key_attr_len(int type)
     case OVS_TUNNEL_KEY_ATTR_TTL: return 1;
     case OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT: return 0;
     case OVS_TUNNEL_KEY_ATTR_CSUM: return 0;
+    case OVS_TUNNEL_KEY_ATTR_TP_SRC: return 2;
+    case OVS_TUNNEL_KEY_ATTR_TP_DST: return 2;
     case OVS_TUNNEL_KEY_ATTR_OAM: return 0;
     case OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS: return -2;
     case __OVS_TUNNEL_KEY_ATTR_MAX:
@@ -914,6 +949,12 @@ odp_tun_key_from_attr(const struct nlattr *attr, struct flow_tnl *tun)
         case OVS_TUNNEL_KEY_ATTR_CSUM:
             tun->flags |= FLOW_TNL_F_CSUM;
             break;
+        case OVS_TUNNEL_KEY_ATTR_TP_SRC:
+            tun->tp_src = nl_attr_get_be16(a);
+            break;
+        case OVS_TUNNEL_KEY_ATTR_TP_DST:
+            tun->tp_dst = nl_attr_get_be16(a);
+            break;
         case OVS_TUNNEL_KEY_ATTR_OAM:
             tun->flags |= FLOW_TNL_F_OAM;
             break;
@@ -970,6 +1011,12 @@ tun_key_to_attr(struct ofpbuf *a, const struct flow_tnl *tun_key)
     if (tun_key->flags & FLOW_TNL_F_CSUM) {
         nl_msg_put_flag(a, OVS_TUNNEL_KEY_ATTR_CSUM);
     }
+    if (tun_key->tp_src) {
+        nl_msg_put_be16(a, OVS_TUNNEL_KEY_ATTR_TP_SRC, tun_key->tp_src);
+    }
+    if (tun_key->tp_dst) {
+        nl_msg_put_be16(a, OVS_TUNNEL_KEY_ATTR_TP_DST, tun_key->tp_dst);
+    }
     if (tun_key->flags & FLOW_TNL_F_OAM) {
         nl_msg_put_flag(a, OVS_TUNNEL_KEY_ATTR_OAM);
     }
@@ -3503,6 +3550,7 @@ odp_key_fitness_to_string(enum odp_key_fitness fitness)
 size_t
 odp_put_userspace_action(uint32_t pid,
                          const void *userdata, size_t userdata_size,
+                         const uint32_t *tunnel_out_pid,
                          struct ofpbuf *odp_actions)
 {
     size_t userdata_ofs;
@@ -3529,6 +3577,9 @@ odp_put_userspace_action(uint32_t pid,
     } else {
         userdata_ofs = 0;
     }
+    if (tunnel_out_pid) {
+        nl_msg_put_u32(odp_actions, OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT, *tunnel_out_pid);
+    }
     nl_msg_end_nested(odp_actions, offset);
 
     return userdata_ofs;
diff --git a/lib/odp-util.h b/lib/odp-util.h
index 4ce693c..11d76fd 100644
--- a/lib/odp-util.h
+++ b/lib/odp-util.h
@@ -230,12 +230,14 @@ union user_action_cookie {
 
     struct {
         uint16_t type;          /* USER_ACTION_COOKIE_IPFIX. */
+        uint32_t output_odp_port; /* The output odp port on datapath to get tunnel info. */
     } ipfix;
 };
 BUILD_ASSERT_DECL(sizeof(union user_action_cookie) == 16);
 
 size_t odp_put_userspace_action(uint32_t pid,
                                 const void *userdata, size_t userdata_size,
+                                const uint32_t *tunnel_out_pid,
                                 struct ofpbuf *odp_actions);
 void odp_put_tunnel_action(const struct flow_tnl *tunnel,
                            struct ofpbuf *odp_actions);
diff --git a/lib/packets.h b/lib/packets.h
index c04e3bb..e54b287 100644
--- a/lib/packets.h
+++ b/lib/packets.h
@@ -33,12 +33,17 @@ struct ds;
 
 /* Tunnel information used in flow key and metadata. */
 struct flow_tnl {
-    ovs_be64 tun_id;
+    union {
+        ovs_be64 tun_id;
+        uint8_t tun_id_octarray[8];
+    };
     ovs_be32 ip_src;
     ovs_be32 ip_dst;
     uint16_t flags;
     uint8_t ip_tos;
     uint8_t ip_ttl;
+    ovs_be16 tp_src;
+    ovs_be16 tp_dst;
 };
 
 /* Unfortunately, a "struct flow" sometimes has to handle OpenFlow port
@@ -63,7 +68,7 @@ struct pkt_metadata {
 };
 
 #define PKT_METADATA_INITIALIZER(PORT) \
-    (struct pkt_metadata){ 0, 0, { 0, 0, 0, 0, 0, 0}, 0, 0, {(PORT)} }
+    (struct pkt_metadata){ 0, 0, { {0}, 0, 0, 0, 0, 0, 0, 0}, 0, 0, {(PORT)} }
 
 bool dpid_from_string(const char *s, uint64_t *dpidp);
 
diff --git a/ofproto/automake.mk b/ofproto/automake.mk
index 22c50d1..d622b64 100644
--- a/ofproto/automake.mk
+++ b/ofproto/automake.mk
@@ -73,3 +73,6 @@ dist_noinst_SCRIPTS = ofproto/ipfix-gen-entities
 ofproto/ipfix-entities.def: ofproto/ipfix.xml ofproto/ipfix-gen-entities
 	$(run_python) $(srcdir)/ofproto/ipfix-gen-entities $< > $@.tmp
 	mv $@.tmp $@
+
+# IPFIX enterprise entity definition macros.
+EXTRA_DIST += ofproto/ipfix-enterprise-entities.def
diff --git a/ofproto/ipfix-enterprise-entities.def b/ofproto/ipfix-enterprise-entities.def
new file mode 100644
index 0000000..013cee2
--- /dev/null
+++ b/ofproto/ipfix-enterprise-entities.def
@@ -0,0 +1,19 @@
+/* IPFIX enterprise entities. */
+#ifndef IPFIX_ENTERPRISE_ENTITY
+#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)
+#endif
+
+#ifndef IPFIX_ENTERPRISE_VMWARE
+#define IPFIX_ENTERPRISE_VMWARE 6876
+#endif
+
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_TYPE, 891, 1, tunnelType, IPFIX_ENTERPRISE_VMWARE)
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_KEY, 892, 0, tunnelKey, IPFIX_ENTERPRISE_VMWARE)
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_SOURCE_IPV4_ADDRESS, 893, 4, tunnelSourceIPv4Address, IPFIX_ENTERPRISE_VMWARE)
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_DESTINATION_IPV4_ADDRESS, 894, 4, tunnelDestinationIPv4Address, IPFIX_ENTERPRISE_VMWARE)
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_PROTOCOL_IDENTIFIER, 895, 1, tunnelProtocolIdentifier, IPFIX_ENTERPRISE_VMWARE)
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_SOURCE_TRANSPORT_PORT, 896, 2, tunnelSourceTransportPort, IPFIX_ENTERPRISE_VMWARE)
+IPFIX_ENTERPRISE_ENTITY(TUNNEL_DESTINATION_TRANSPORT_PORT, 897, 2, tunnelDestinationTransportPort, IPFIX_ENTERPRISE_VMWARE)
+
+#undef IPFIX_ENTERPRISE_VMWARE
+#undef IPFIX_ENTERPRISE_ENTITY
diff --git a/ofproto/ofproto-dpif-ipfix.c b/ofproto/ofproto-dpif-ipfix.c
index 1584c25..e742fba 100644
--- a/ofproto/ofproto-dpif-ipfix.c
+++ b/ofproto/ofproto-dpif-ipfix.c
@@ -25,6 +25,7 @@
 #include "list.h"
 #include "ofpbuf.h"
 #include "ofproto.h"
+#include "ofproto-dpif.h"
 #include "packets.h"
 #include "poll-loop.h"
 #include "sset.h"
@@ -41,6 +42,39 @@ static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
 /* Cf. IETF RFC 5101 Section 10.3.4. */
 #define IPFIX_DEFAULT_COLLECTOR_PORT 4739
 
+/* The standard layer2SegmentId (ID 351) element is included in vDS to send
+ * the VxLAN tunnel's VNI. It is 64-bit long, the most significant byte is
+ * used to indicate the type of tunnel (0x01 = VxLAN, 0x02 = GRE) and the three
+ * lowest significant bytes hold the value of the layer 2 overlay network
+ * segment identifier: a 24-bit VxLAN tunnel's VNI or a 24-bit GRE tunnel's
+ * TNI. This is not compatible with GRE-64, as implementd in OVS, as its
+ * tunnel IDs are 64-bit.
+ *
+ * Two new enterprise information elements are defined which are simlar to
+ * laryerSegmentId but support 64-bit IDs:
+ *     tunnelType (ID 891) and tunnelKey (ID 892).
+ *
+ * The enum dpif_ipfix_tunnel_type is to declare the types supported in the
+ * tunnelType element.
+ */
+enum dpif_ipfix_tunnel_type {
+    DPIF_IPFIX_TUNNEL_UNKNOWN = 0x00,
+    DPIF_IPFIX_TUNNEL_VXLAN = 0x01,
+    DPIF_IPFIX_TUNNEL_GRE = 0x02,
+    DPIF_IPFIX_TUNNEL_LISP = 0x03,
+    DPIF_IPFIX_TUNNEL_IPSEC_GRE = 0x05,
+    DPIF_IPFIX_TUNNEL_GENEVE = 0x07,
+    NUM_DPIF_IPFIX_TUNNEL
+};
+
+struct dpif_ipfix_port {
+    struct hmap_node hmap_node; /* In struct dpif_ipfix's "tunnel_ports" hmap. */
+    struct ofport *ofport;      /* To retrieve port stats. */
+    odp_port_t odp_port;
+    enum dpif_ipfix_tunnel_type tunnel_type;
+    uint8_t tunnel_key_length;
+};
+
 struct dpif_ipfix_exporter {
     struct collectors *collectors;
     uint32_t seq_number;
@@ -70,6 +104,9 @@ struct dpif_ipfix_flow_exporter_map_node {
 struct dpif_ipfix {
     struct dpif_ipfix_bridge_exporter bridge_exporter;
     struct hmap flow_exporter_map;  /* dpif_ipfix_flow_exporter_map_node. */
+    struct hmap tunnel_ports;       /* Contains "struct dpif_ipfix_port"s.
+                                     * It makes tunnel port lookups faster in
+                                     * sampling upcalls. */
     struct ovs_refcount ref_cnt;
 };
 
@@ -121,6 +158,11 @@ enum ipfix_proto_l4 {
     IPFIX_PROTO_L4_ICMP,
     NUM_IPFIX_PROTO_L4
 };
+enum ipfix_proto_tunnel {
+    IPFIX_PROTO_TUNNEL_UNKNOWN = 0,
+    IPFIX_PROTO_TUNNEL_KNOWN,  /* Support gre, lisp and vxlan. */
+    NUM_IPFIX_PROTO_TUNNEL
+};
 
 /* Any Template ID > 255 is usable for Template Records. */
 #define IPFIX_TEMPLATE_ID_MIN 256
@@ -134,33 +176,60 @@ struct ipfix_template_record_header {
 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_record_header) == 4);
 
 enum ipfix_entity_id {
+/* standard IPFIX elements */
 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ID_##ENUM = ID,
 #include "ofproto/ipfix-entities.def"
+/* non-standard IPFIX elements */
+#define IPFIX_SET_ENTERPRISE(v) (((v) | 0x8000))
+#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)  IPFIX_ENTITY_ID_##ENUM = IPFIX_SET_ENTERPRISE(ID),
+#include "ofproto/ipfix-enterprise-entities.def"
 };
 
 enum ipfix_entity_size {
+/* standard IPFIX elements */
 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 #include "ofproto/ipfix-entities.def"
+/* non-standard IPFIX elements */
+#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
+#include "ofproto/ipfix-enterprise-entities.def"
+};
+
+enum ipfix_entity_enterprise {
+/* standard IPFIX elements */
+#define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ENTERPRISE_##ENUM = 0,
+#include "ofproto/ipfix-entities.def"
+/* non-standard IPFIX elements */
+#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)  IPFIX_ENTITY_ENTERPRISE_##ENUM = ENTERPRISE,
+#include "ofproto/ipfix-enterprise-entities.def"
 };
 
 OVS_PACKED(
 struct ipfix_template_field_specifier {
     ovs_be16 element_id;  /* IPFIX_ENTITY_ID_*. */
-    ovs_be16 field_length;  /* Length of the field's value, in bytes. */
-    /* No Enterprise ID, since only standard element IDs are specified. */
+    ovs_be16 field_length;  /* Length of the field's value, in bytes.
+                             * For Variable-Length element, it should be 65535.
+                             */
+    ovs_be32 enterprise;  /* Enterprise number */
 });
-BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 4);
+BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 8);
+
+/* Cf. IETF RFC 5102 Section 5.11.6. */
+enum ipfix_flow_direction {
+    INGRESS_FLOW = 0x00,
+    EGRESS_FLOW = 0x01
+};
 
 /* Part of data record flow key for common metadata and Ethernet entities. */
 OVS_PACKED(
 struct ipfix_data_record_flow_key_common {
     ovs_be32 observation_point_id;  /* OBSERVATION_POINT_ID */
+    uint8_t flow_direction;  /* FLOW_DIRECTION */
     uint8_t source_mac_address[6];  /* SOURCE_MAC_ADDRESS */
     uint8_t destination_mac_address[6];  /* DESTINATION_MAC_ADDRESS */
     ovs_be16 ethernet_type;  /* ETHERNET_TYPE */
     uint8_t ethernet_header_length;  /* ETHERNET_HEADER_LENGTH */
 });
-BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 19);
+BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 20);
 
 /* Part of data record flow key for VLAN entities. */
 OVS_PACKED(
@@ -217,6 +286,33 @@ struct ipfix_data_record_flow_key_icmp {
 });
 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_icmp) == 2);
 
+/* For the tunnel type that is on the top of IPSec, the protocol identifier
+ * of the upper tunnel type is used.
+ */
+static uint8_t tunnel_protocol[NUM_DPIF_IPFIX_TUNNEL] = {
+    0,              /* reserved */
+    IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_VXLAN */
+    IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_GRE */
+    IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_LISP*/
+    0          ,    /* reserved */
+    IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_IPSEC_GRE */
+    0          ,    /* reserved */
+    IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_GENEVE*/
+};
+
+OVS_PACKED(
+struct ipfix_data_record_flow_key_tunnel {
+    ovs_be32 tunnel_source_ipv4_address;  /* TUNNEL_SOURCE_IPV4_ADDRESS */
+    ovs_be32 tunnel_destination_ipv4_address;  /* TUNNEL_DESTINATION_IPV4_ADDRESS */
+    uint8_t tunnel_protocol_identifier;  /* TUNNEL_PROTOCOL_IDENTIFIER */
+    ovs_be16 tunnel_source_transport_port;  /* TUNNEL_SOURCE_TRANSPORT_PORT */
+    ovs_be16 tunnel_destination_transport_port;  /* TUNNEL_DESTINATION_TRANSPORT_PORT */
+    uint8_t tunnel_type;  /* TUNNEL_TYPE */
+    uint8_t tunnel_key_length;  /* length of TUNNEL_KEY */
+    uint8_t tunnel_key[];  /* data of  TUNNEL_KEY */
+});
+BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_tunnel) == 15);
+
 /* Cf. IETF RFC 5102 Section 5.11.3. */
 enum ipfix_flow_end_reason {
     IDLE_TIMEOUT = 0x01,
@@ -247,6 +343,14 @@ struct ipfix_data_record_aggregated_ip {
 });
 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
 
+/*
+ * support tunnel key for:
+ * VxLAN: 24-bit VIN,
+ * GRE: 32- or 64-bit key,
+ * LISP: 24-bit instance ID
+ */
+#define MAX_TUNNEL_KEY_LEN 8
+
 #define MAX_FLOW_KEY_LEN                                        \
     (sizeof(struct ipfix_data_record_flow_key_common)           \
      + sizeof(struct ipfix_data_record_flow_key_vlan)           \
@@ -254,7 +358,9 @@ BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
      + MAX(sizeof(struct ipfix_data_record_flow_key_ipv4),      \
            sizeof(struct ipfix_data_record_flow_key_ipv6))      \
      + MAX(sizeof(struct ipfix_data_record_flow_key_icmp),      \
-           sizeof(struct ipfix_data_record_flow_key_transport)))
+           sizeof(struct ipfix_data_record_flow_key_transport)) \
+     + sizeof(struct ipfix_data_record_flow_key_tunnel)         \
+     + MAX_TUNNEL_KEY_LEN)
 
 #define MAX_DATA_RECORD_LEN                                 \
     (MAX_FLOW_KEY_LEN                                       \
@@ -315,6 +421,9 @@ ofproto_ipfix_bridge_exporter_options_equal(
             && a->sampling_rate == b->sampling_rate
             && a->cache_active_timeout == b->cache_active_timeout
             && a->cache_max_flows == b->cache_max_flows
+            && a->enable_tunnel_sampling == b->enable_tunnel_sampling
+            && a->enable_input_sampling == b->enable_input_sampling
+            && a->enable_output_sampling == b->enable_output_sampling
             && sset_equals(&a->targets, &b->targets));
 }
 
@@ -422,6 +531,114 @@ dpif_ipfix_exporter_set_options(struct dpif_ipfix_exporter *exporter,
     return true;
 }
 
+static struct dpif_ipfix_port *
+dpif_ipfix_find_port(const struct dpif_ipfix *di,
+                     odp_port_t odp_port) OVS_REQUIRES(mutex)
+{
+    struct dpif_ipfix_port *dip;
+
+    HMAP_FOR_EACH_IN_BUCKET (dip, hmap_node, hash_odp_port(odp_port),
+                             &di->tunnel_ports) {
+        if (dip->odp_port == odp_port) {
+            return dip;
+        }
+    }
+    return NULL;
+}
+
+static void
+dpif_ipfix_del_port(struct dpif_ipfix *di,
+                      struct dpif_ipfix_port *dip)
+    OVS_REQUIRES(mutex)
+{
+    hmap_remove(&di->tunnel_ports, &dip->hmap_node);
+    free(dip);
+}
+
+void
+dpif_ipfix_add_tunnel_port(struct dpif_ipfix *di, struct ofport *ofport,
+                           odp_port_t odp_port) OVS_EXCLUDED(mutex)
+{
+    struct dpif_ipfix_port *dip;
+    const char *type;
+
+    ovs_mutex_lock(&mutex);
+    dip = dpif_ipfix_find_port(di, odp_port);
+    if (dip) {
+        dpif_ipfix_del_port(di, dip);
+    }
+
+    type = netdev_get_type(ofport->netdev);
+    if (type == NULL) {
+        goto out;
+    }
+
+    /* Add to table of tunnel ports. */
+    dip = xmalloc(sizeof *dip);
+    if (dip == NULL) {
+        goto out;
+    }
+    dip->ofport = ofport;
+    dip->odp_port = odp_port;
+    if (strncmp(type, "gre", sizeof "gre") == 0) {
+        /* 32-bit key gre */
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
+        dip->tunnel_key_length = 4;
+    } else if (strncmp(type, "gre64", sizeof "gre64") == 0) {
+        /* 64-bit key gre */
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
+        dip->tunnel_key_length = 8;
+    } else if (strncmp(type, "ipsec_gre", sizeof "ipsec_gre") == 0) {
+        /* 32-bit key ipsec_gre */
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
+        dip->tunnel_key_length = 4;
+    } else if (strncmp(type, "ipsec_gre64", sizeof "ipsec_gre64") == 0) {
+        /* 64-bit key ipsec_gre */
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
+        dip->tunnel_key_length = 8;
+    } else if (strncmp(type, "vxlan", sizeof "vxlan") == 0) {
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_VXLAN;
+        dip->tunnel_key_length = 3;
+    } else if (strncmp(type, "lisp", sizeof "lisp") == 0) {
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_LISP;
+        dip->tunnel_key_length = 3;
+    } else if (strncmp(type, "geneve", sizeof "geneve") == 0) {
+        dip->tunnel_type = DPIF_IPFIX_TUNNEL_GENEVE;
+        dip->tunnel_key_length = 3;
+    } else {
+        free(dip);
+        goto out;
+    }
+    hmap_insert(&di->tunnel_ports, &dip->hmap_node, hash_odp_port(odp_port));
+
+out:
+    ovs_mutex_unlock(&mutex);
+}
+
+void
+dpif_ipfix_del_tunnel_port(struct dpif_ipfix *di, odp_port_t odp_port)
+    OVS_EXCLUDED(mutex)
+{
+    struct dpif_ipfix_port *dip;
+    ovs_mutex_lock(&mutex);
+    dip = dpif_ipfix_find_port(di, odp_port);
+    if (dip) {
+        dpif_ipfix_del_port(di, dip);
+    }
+    ovs_mutex_unlock(&mutex);
+}
+
+bool
+dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *di, odp_port_t odp_port)
+    OVS_EXCLUDED(mutex)
+{
+    struct dpif_ipfix_port *dip;
+    ovs_mutex_lock(&mutex);
+    dip = dpif_ipfix_find_port(di, odp_port);
+    ovs_mutex_unlock(&mutex);
+    return (dip != NULL);
+}
+
 static void
 dpif_ipfix_bridge_exporter_init(struct dpif_ipfix_bridge_exporter *exporter)
 {
@@ -652,6 +869,7 @@ dpif_ipfix_create(void)
     di = xzalloc(sizeof *di);
     dpif_ipfix_bridge_exporter_init(&di->bridge_exporter);
     hmap_init(&di->flow_exporter_map);
+    hmap_init(&di->tunnel_ports);
     ovs_refcount_init(&di->ref_cnt);
     return di;
 }
@@ -677,10 +895,50 @@ dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *di)
     return ret;
 }
 
+bool
+dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *di)
+    OVS_EXCLUDED(mutex)
+{
+    bool ret = true;
+    ovs_mutex_lock(&mutex);
+    if (di->bridge_exporter.options) {
+        ret = di->bridge_exporter.options->enable_input_sampling;
+    }
+    ovs_mutex_unlock(&mutex);
+    return ret;
+}
+
+bool
+dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *di)
+    OVS_EXCLUDED(mutex)
+{
+    bool ret = true;
+    ovs_mutex_lock(&mutex);
+    if (di->bridge_exporter.options) {
+        ret = di->bridge_exporter.options->enable_output_sampling;
+    }
+    ovs_mutex_unlock(&mutex);
+    return ret;
+}
+
+bool
+dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *di)
+    OVS_EXCLUDED(mutex)
+{
+    bool ret = false;
+    ovs_mutex_lock(&mutex);
+    if (di->bridge_exporter.options) {
+        ret = di->bridge_exporter.options->enable_tunnel_sampling;
+    }
+    ovs_mutex_unlock(&mutex);
+    return ret;
+}
+
 static void
 dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
 {
     struct dpif_ipfix_flow_exporter_map_node *exp_node, *exp_next;
+    struct dpif_ipfix_port *dip, *next;
 
     dpif_ipfix_bridge_exporter_clear(&di->bridge_exporter);
 
@@ -689,6 +947,10 @@ dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
         dpif_ipfix_flow_exporter_destroy(&exp_node->exporter);
         free(exp_node);
     }
+
+    HMAP_FOR_EACH_SAFE (dip, next, hmap_node, &di->tunnel_ports) {
+        dpif_ipfix_del_port(di, dip);
+    }
 }
 
 void
@@ -699,6 +961,7 @@ dpif_ipfix_unref(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
         dpif_ipfix_clear(di);
         dpif_ipfix_bridge_exporter_destroy(&di->bridge_exporter);
         hmap_destroy(&di->flow_exporter_map);
+        hmap_destroy(&di->tunnel_ports);
         free(di);
         ovs_mutex_unlock(&mutex);
     }
@@ -733,42 +996,64 @@ ipfix_send_msg(const struct collectors *collectors, struct ofpbuf *msg)
 
 static uint16_t
 ipfix_get_template_id(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
-                      enum ipfix_proto_l4 l4)
+                      enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel)
 {
     uint16_t template_id;
     template_id = l2;
     template_id = template_id * NUM_IPFIX_PROTO_L3 + l3;
     template_id = template_id * NUM_IPFIX_PROTO_L4 + l4;
+    template_id = template_id * NUM_IPFIX_PROTO_TUNNEL + tunnel;
     return IPFIX_TEMPLATE_ID_MIN + template_id;
 }
 
 static void
 ipfix_define_template_entity(enum ipfix_entity_id id,
-                             enum ipfix_entity_size size, struct ofpbuf *msg)
+                             enum ipfix_entity_size size,
+                             enum ipfix_entity_enterprise enterprise,
+                             struct ofpbuf *msg)
 {
     struct ipfix_template_field_specifier *field;
+    size_t field_size;
 
-    field = ofpbuf_put_zeros(msg, sizeof *field);
+    if (enterprise) {
+        field_size = sizeof *field;
+    } else {
+        /* No enterprise number */
+        field_size = sizeof *field - sizeof(ovs_be32);
+    }
+    field = ofpbuf_put_zeros(msg, field_size);
     field->element_id = htons(id);
-    field->field_length = htons(size);
+    if (size) {
+        field->field_length = htons(size);
+    } else {
+        /* RFC 5101, Section 7. Variable-Length Information Element */
+        field->field_length = 0xffff;
+    }
+    if (enterprise) {
+        field->enterprise = htonl(enterprise);
+    }
+
 }
 
 static uint16_t
 ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
-                             enum ipfix_proto_l4 l4, struct ofpbuf *msg)
+                             enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel,
+                             struct ofpbuf *msg)
 {
     uint16_t count = 0;
 
 #define DEF(ID) \
     { \
         ipfix_define_template_entity(IPFIX_ENTITY_ID_##ID, \
-                                     IPFIX_ENTITY_SIZE_##ID, msg); \
+                                     IPFIX_ENTITY_SIZE_##ID, \
+                                     IPFIX_ENTITY_ENTERPRISE_##ID, msg); \
         count++; \
     }
 
     /* 1. Flow key. */
 
     DEF(OBSERVATION_POINT_ID);
+    DEF(FLOW_DIRECTION);
 
     /* Common Ethernet entities. */
     DEF(SOURCE_MAC_ADDRESS);
@@ -814,6 +1099,16 @@ ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
         }
     }
 
+    if (tunnel != IPFIX_PROTO_TUNNEL_UNKNOWN){
+        DEF(TUNNEL_SOURCE_IPV4_ADDRESS);
+        DEF(TUNNEL_DESTINATION_IPV4_ADDRESS);
+        DEF(TUNNEL_PROTOCOL_IDENTIFIER);
+        DEF(TUNNEL_SOURCE_TRANSPORT_PORT);
+        DEF(TUNNEL_DESTINATION_TRANSPORT_PORT);
+        DEF(TUNNEL_TYPE);
+        DEF(TUNNEL_KEY);
+    }
+
     /* 2. Flow aggregated data. */
 
     DEF(FLOW_START_DELTA_MICROSECONDS);
@@ -829,14 +1124,15 @@ ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
         DEF(MAXIMUM_IP_TOTAL_LENGTH);
     }
 
+
 #undef DEF
 
     return count;
 }
 
 static void
-ipfix_send_template_msg(struct dpif_ipfix_exporter *exporter,
-                        uint32_t export_time_sec, uint32_t obs_domain_id)
+ipfix_send_template_msgs(struct dpif_ipfix_exporter *exporter,
+                         uint32_t export_time_sec, uint32_t obs_domain_id)
 {
     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
     struct ofpbuf msg;
@@ -847,6 +1143,7 @@ ipfix_send_template_msg(struct dpif_ipfix_exporter *exporter,
     enum ipfix_proto_l2 l2;
     enum ipfix_proto_l3 l3;
     enum ipfix_proto_l4 l4;
+    enum ipfix_proto_tunnel tunnel;
 
     ofpbuf_use_stub(&msg, msg_stub, sizeof msg_stub);
 
@@ -867,14 +1164,43 @@ ipfix_send_template_msg(struct dpif_ipfix_exporter *exporter,
                     l4 != IPFIX_PROTO_L4_UNKNOWN) {
                     continue;
                 }
-                tmpl_hdr_offset = ofpbuf_size(&msg);
-                tmpl_hdr = ofpbuf_put_zeros(&msg, sizeof *tmpl_hdr);
-                tmpl_hdr->template_id = htons(
-                    ipfix_get_template_id(l2, l3, l4));
-                field_count = ipfix_define_template_fields(l2, l3, l4, &msg);
-                tmpl_hdr = (struct ipfix_template_record_header*)
-                    ((uint8_t*)ofpbuf_data(&msg) + tmpl_hdr_offset);
-                tmpl_hdr->field_count = htons(field_count);
+                for (tunnel = 0; tunnel < NUM_IPFIX_PROTO_TUNNEL; tunnel++) {
+                    /* When the size of the tmplate packet reaches
+                     * MAX_MESSAGE_LEN(1024), send it out.
+                     * And then reinit the msg to construct a new packet for
+                     * the following templates.
+                     */
+                    if (ofpbuf_size(&msg) >= MAX_MESSAGE_LEN) {
+                        /* Send template message. */
+                        set_hdr = (struct ipfix_set_header*)
+                                  ((uint8_t*)ofpbuf_data(&msg) + set_hdr_offset);
+                        set_hdr->length = htons(ofpbuf_size(&msg) - set_hdr_offset);
+
+                        ipfix_send_msg(exporter->collectors, &msg);
+                        ofpbuf_uninit(&msg);
+
+                        /* Reinit the msg buf. */
+                        ofpbuf_use_stub(&msg, msg_stub, sizeof msg_stub);
+
+                        ipfix_init_header(export_time_sec, exporter->seq_number,
+                                          obs_domain_id, &msg);
+                        set_hdr_offset = ofpbuf_size(&msg);
+
+                        /* Add a Template Set. */
+                        set_hdr = ofpbuf_put_zeros(&msg, sizeof *set_hdr);
+                        set_hdr->set_id = htons(IPFIX_SET_ID_TEMPLATE);
+                    }
+
+                    tmpl_hdr_offset = ofpbuf_size(&msg);
+                    tmpl_hdr = ofpbuf_put_zeros(&msg, sizeof *tmpl_hdr);
+                    tmpl_hdr->template_id = htons(
+                        ipfix_get_template_id(l2, l3, l4, tunnel));
+                    field_count =
+                        ipfix_define_template_fields(l2, l3, l4, tunnel, &msg);
+                    tmpl_hdr = (struct ipfix_template_record_header*)
+                        ((uint8_t*)ofpbuf_data(&msg) + tmpl_hdr_offset);
+                    tmpl_hdr->field_count = htons(field_count);
+                }
             }
         }
     }
@@ -1021,13 +1347,16 @@ static void
 ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
                        struct ofpbuf *packet, const struct flow *flow,
                        uint64_t packet_delta_count, uint32_t obs_domain_id,
-                       uint32_t obs_point_id)
+                       uint32_t obs_point_id, odp_port_t output_odp_port,
+                       const struct dpif_ipfix_port *tunnel_port,
+                       const struct flow_tnl *tunnel_key)
 {
     struct ipfix_flow_key *flow_key;
     struct ofpbuf msg;
     enum ipfix_proto_l2 l2;
     enum ipfix_proto_l3 l3;
     enum ipfix_proto_l4 l4;
+    enum ipfix_proto_tunnel tunnel = IPFIX_PROTO_TUNNEL_UNKNOWN;
     uint8_t ethernet_header_length;
     uint16_t ethernet_total_length;
 
@@ -1075,8 +1404,12 @@ ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
         l4 = IPFIX_PROTO_L4_UNKNOWN;
     }
 
+    if (tunnel_port && tunnel_key) {
+       tunnel = IPFIX_PROTO_TUNNEL_KNOWN;
+    }
+
     flow_key->obs_domain_id = obs_domain_id;
-    flow_key->template_id = ipfix_get_template_id(l2, l3, l4);
+    flow_key->template_id = ipfix_get_template_id(l2, l3, l4, tunnel);
 
     /* The fields defined in the ipfix_data_record_* structs and sent
      * below must match exactly the templates defined in
@@ -1092,6 +1425,8 @@ ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
 
         data_common = ofpbuf_put_zeros(&msg, sizeof *data_common);
         data_common->observation_point_id = htonl(obs_point_id);
+        data_common->flow_direction =
+            (output_odp_port == ODPP_NONE) ? INGRESS_FLOW : EGRESS_FLOW;
         memcpy(data_common->source_mac_address, flow->dl_src,
                sizeof flow->dl_src);
         memcpy(data_common->destination_mac_address, flow->dl_dst,
@@ -1154,6 +1489,34 @@ ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
         data_icmp->icmp_code = ntohs(flow->tp_dst) & 0xff;
     }
 
+    if (tunnel == IPFIX_PROTO_TUNNEL_KNOWN) {
+        struct ipfix_data_record_flow_key_tunnel * data_tunnel;
+
+        data_tunnel = ofpbuf_put_zeros(&msg, sizeof(*data_tunnel) +
+                                             tunnel_port->tunnel_key_length);
+        data_tunnel->tunnel_source_ipv4_address = tunnel_key->ip_src;
+        data_tunnel->tunnel_destination_ipv4_address = tunnel_key->ip_dst;
+        /* The tunnel_protocol_identifier is from tunnel_proto array, which
+         * contians protocol_identifiers of each tunnel type.
+         * For the tunnel type on the top of IPSec, which uses the protocol
+         * identifier of the upper tunnel type is used, the tcp_src and tcp_dst
+         * are decided based on the protocol identifires.
+         * E.g:
+         * The protocol identifier of DPIF_IPFIX_TUNNEL_IPSEC_GRE is IPPROTO_GRE,
+         * and both tp_src and tp_dst are zero.
+         */
+        data_tunnel->tunnel_protocol_identifier =
+            tunnel_protocol[tunnel_port->tunnel_type];
+        data_tunnel->tunnel_source_transport_port = tunnel_key->tp_src;
+        data_tunnel->tunnel_destination_transport_port = tunnel_key->tp_dst;
+        data_tunnel->tunnel_type = tunnel_port->tunnel_type;
+        data_tunnel->tunnel_key_length = tunnel_port->tunnel_key_length;
+        /* tun_id is in network order, and tunnel key is in low bits*/
+        memcpy(data_tunnel->tunnel_key,
+               &tunnel_key->tun_id_octarray[8 - tunnel_port->tunnel_key_length],
+               tunnel_port->tunnel_key_length);
+    }
+
     flow_key->flow_key_msg_part_size = ofpbuf_size(&msg);
 
     {
@@ -1286,31 +1649,51 @@ static void
 dpif_ipfix_sample(struct dpif_ipfix_exporter *exporter,
                   struct ofpbuf *packet, const struct flow *flow,
                   uint64_t packet_delta_count, uint32_t obs_domain_id,
-                  uint32_t obs_point_id)
+                  uint32_t obs_point_id, odp_port_t output_odp_port,
+                  const struct dpif_ipfix_port *tunnel_port,
+                  const struct flow_tnl *tunnel_key)
 {
     struct ipfix_flow_cache_entry *entry;
 
     /* Create a flow cache entry from the sample. */
     entry = xmalloc(sizeof *entry);
     ipfix_cache_entry_init(entry, packet, flow, packet_delta_count,
-                           obs_domain_id, obs_point_id);
+                           obs_domain_id, obs_point_id,
+                           output_odp_port, tunnel_port, tunnel_key);
     ipfix_cache_update(exporter, entry);
 }
 
 void
 dpif_ipfix_bridge_sample(struct dpif_ipfix *di, struct ofpbuf *packet,
-                         const struct flow *flow) OVS_EXCLUDED(mutex)
+                         const struct flow *flow,
+                         odp_port_t input_odp_port, odp_port_t output_odp_port,
+                         const struct flow_tnl *output_tunnel_key) OVS_EXCLUDED(mutex)
 {
     uint64_t packet_delta_count;
+    const struct flow_tnl *tunnel_key = NULL;
+    struct dpif_ipfix_port * tunnel_port = NULL;
 
     ovs_mutex_lock(&mutex);
     /* Use the sampling probability as an approximation of the number
      * of matched packets. */
     packet_delta_count = UINT32_MAX / di->bridge_exporter.probability;
+    if (di->bridge_exporter.options->enable_tunnel_sampling) {
+        if (output_odp_port == ODPP_NONE && flow->tunnel.ip_dst) {
+            /* Input tunnel. */
+            tunnel_key = &flow->tunnel;
+            tunnel_port = dpif_ipfix_find_port(di, input_odp_port);
+        }
+        if (output_odp_port != ODPP_NONE && output_tunnel_key) {
+            /* Output tunnel, output_tunnel_key must valid. */
+            tunnel_key = output_tunnel_key;
+            tunnel_port = dpif_ipfix_find_port(di, output_odp_port);
+        }
+    }
     dpif_ipfix_sample(&di->bridge_exporter.exporter, packet, flow,
                       packet_delta_count,
                       di->bridge_exporter.options->obs_domain_id,
-                      di->bridge_exporter.options->obs_point_id);
+                      di->bridge_exporter.options->obs_point_id,
+                      output_odp_port, tunnel_port, tunnel_key);
     ovs_mutex_unlock(&mutex);
 }
 
@@ -1329,7 +1712,8 @@ dpif_ipfix_flow_sample(struct dpif_ipfix *di, struct ofpbuf *packet,
     node = dpif_ipfix_find_flow_exporter_map_node(di, collector_set_id);
     if (node) {
         dpif_ipfix_sample(&node->exporter.exporter, packet, flow,
-                          packet_delta_count, obs_domain_id, obs_point_id);
+                          packet_delta_count, obs_domain_id, obs_point_id,
+                          ODPP_NONE, NULL, NULL);
     }
     ovs_mutex_unlock(&mutex);
 }
@@ -1374,8 +1758,8 @@ dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *exporter,
         if (!template_msg_sent
             && (exporter->last_template_set_time + IPFIX_TEMPLATE_INTERVAL)
                 <= export_time_sec) {
-            ipfix_send_template_msg(exporter, export_time_sec,
-                                    entry->flow_key.obs_domain_id);
+            ipfix_send_template_msgs(exporter, export_time_sec,
+                                     entry->flow_key.obs_domain_id);
             exporter->last_template_set_time = export_time_sec;
             template_msg_sent = true;
         }
diff --git a/ofproto/ofproto-dpif-ipfix.h b/ofproto/ofproto-dpif-ipfix.h
index 6ebf8b0..0222054 100644
--- a/ofproto/ofproto-dpif-ipfix.h
+++ b/ofproto/ofproto-dpif-ipfix.h
@@ -19,24 +19,36 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
+#include "lib/odp-util.h"
 
 struct flow;
 struct ofpbuf;
 struct ofproto_ipfix_bridge_exporter_options;
 struct ofproto_ipfix_flow_exporter_options;
+struct flow_tnl;
+struct ofport;
 
 struct dpif_ipfix *dpif_ipfix_create(void);
 struct dpif_ipfix *dpif_ipfix_ref(const struct dpif_ipfix *);
 void dpif_ipfix_unref(struct dpif_ipfix *);
 
+void dpif_ipfix_add_tunnel_port(struct dpif_ipfix *, struct ofport *, odp_port_t);
+void dpif_ipfix_del_tunnel_port(struct dpif_ipfix *, odp_port_t);
+
 uint32_t dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *);
+bool dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *);
+bool dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *);
+bool dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *);
+bool dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *, odp_port_t);
 void dpif_ipfix_set_options(
     struct dpif_ipfix *,
     const struct ofproto_ipfix_bridge_exporter_options *,
     const struct ofproto_ipfix_flow_exporter_options *, size_t);
 
 void dpif_ipfix_bridge_sample(struct dpif_ipfix *, struct ofpbuf *,
-                              const struct flow *);
+                              const struct flow *,
+                              uint32_t, uint32_t, const struct flow_tnl *);
 void dpif_ipfix_flow_sample(struct dpif_ipfix *, struct ofpbuf *,
                             const struct flow *, uint32_t, uint16_t, uint32_t,
                             uint32_t);
diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c
index 654fbd3..ebd5dd9 100644
--- a/ofproto/ofproto-dpif-upcall.c
+++ b/ofproto/ofproto-dpif-upcall.c
@@ -720,7 +720,7 @@ compose_slow_path(struct udpif *udpif, struct xlate_out *xout,
         ? ODPP_NONE
         : odp_in_port;
     pid = dpif_port_get_pid(udpif->dpif, port, flow_hash_5tuple(flow, 0));
-    odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path, buf);
+    odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path, NULL, buf);
 }
 
 static void
@@ -899,7 +899,23 @@ convert_upcall(struct udpif *udpif, struct upcall *upcall)
         break;
     case IPFIX_UPCALL:
         if (ipfix) {
-            dpif_ipfix_bridge_sample(ipfix, packet, &flow);
+            union user_action_cookie cookie;
+            struct flow_tnl output_tunnel_key;
+
+            memset(&cookie, 0, sizeof cookie);
+            memcpy(&cookie, nl_attr_get(dupcall->userdata),
+                   sizeof cookie.ipfix);
+
+            if (dupcall->out_tun_key){
+                memset(&output_tunnel_key, 0, sizeof output_tunnel_key);
+                odp_tun_key_from_attr(dupcall->out_tun_key,
+                                      &output_tunnel_key);
+            }
+            dpif_ipfix_bridge_sample(ipfix, packet, &flow,
+                                     odp_in_port,
+                                     cookie.ipfix.output_odp_port,
+                                     dupcall->out_tun_key ?
+                                         &output_tunnel_key : NULL);
         }
         break;
     case FLOW_SAMPLE_UPCALL:
diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
index 4aedb59..ded98fe 100644
--- a/ofproto/ofproto-dpif-xlate.c
+++ b/ofproto/ofproto-dpif-xlate.c
@@ -2153,7 +2153,8 @@ compose_sample_action(const struct xbridge *xbridge,
                       const struct flow *flow,
                       const uint32_t probability,
                       const union user_action_cookie *cookie,
-                      const size_t cookie_size)
+                      const size_t cookie_size,
+                      const uint32_t *tunnel_out_port)
 {
     size_t sample_offset, actions_offset;
     odp_port_t odp_port;
@@ -2170,7 +2171,7 @@ compose_sample_action(const struct xbridge *xbridge,
     pid = dpif_port_get_pid(xbridge->dpif, odp_port,
                             flow_hash_5tuple(flow, 0));
     cookie_offset = odp_put_userspace_action(pid, cookie, cookie_size,
-                                             odp_actions);
+                                             tunnel_out_port, odp_actions);
 
     nl_msg_end_nested(odp_actions, actions_offset);
     nl_msg_end_nested(odp_actions, sample_offset);
@@ -2228,7 +2229,7 @@ compose_sflow_action(const struct xbridge *xbridge,
                          odp_port == ODPP_NONE ? 0 : 1, &cookie);
 
     return compose_sample_action(xbridge, odp_actions, flow,  probability,
-                                 &cookie, sizeof cookie.sflow);
+                                 &cookie, sizeof cookie.sflow, NULL);
 }
 
 static void
@@ -2244,29 +2245,53 @@ compose_flow_sample_cookie(uint16_t probability, uint32_t collector_set_id,
 }
 
 static void
-compose_ipfix_cookie(union user_action_cookie *cookie)
+compose_ipfix_cookie(union user_action_cookie *cookie,
+                     odp_port_t output_odp_port)
 {
     cookie->type = USER_ACTION_COOKIE_IPFIX;
+    cookie->ipfix.output_odp_port = output_odp_port;
 }
 
 /* Compose SAMPLE action for IPFIX bridge sampling. */
 static void
 compose_ipfix_action(const struct xbridge *xbridge,
                      struct ofpbuf *odp_actions,
-                     const struct flow *flow)
+                     const struct flow *flow,
+                     odp_port_t output_odp_port)
 {
     uint32_t probability;
     union user_action_cookie cookie;
+    uint32_t *tunnel_out_port = NULL;
 
     if (!xbridge->ipfix || flow->in_port.ofp_port == OFPP_NONE) {
         return;
     }
 
+    /* For input case, output_odp_port is ODPP_NONE, which is an invalid port number. */
+    if (output_odp_port == ODPP_NONE &&
+        !dpif_ipfix_get_bridge_exporter_input_sampling(xbridge->ipfix)) {
+        return;
+    }
+
+    /* For output case, output_pid is valid*/
+    if (output_odp_port != ODPP_NONE) {
+        if (!dpif_ipfix_get_bridge_exporter_output_sampling(xbridge->ipfix)) {
+            return;
+        }
+        /* If enable-tunnel-sampling is enabled, need put an additional option
+         * attribute: OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT
+         */
+        if (dpif_ipfix_get_bridge_exporter_tunnel_sampling(xbridge->ipfix) &&
+            dpif_ipfix_get_tunnel_port(xbridge->ipfix, output_odp_port) ) {
+           tunnel_out_port = &output_odp_port;
+        }
+    }
+
     probability = dpif_ipfix_get_bridge_exporter_probability(xbridge->ipfix);
-    compose_ipfix_cookie(&cookie);
+    compose_ipfix_cookie(&cookie, output_odp_port);
 
     compose_sample_action(xbridge, odp_actions, flow,  probability,
-                          &cookie, sizeof cookie.ipfix);
+                          &cookie, sizeof cookie.ipfix, tunnel_out_port);
 }
 
 /* SAMPLE action for sFlow must be first action in any given list of
@@ -2288,7 +2313,14 @@ static void
 add_ipfix_action(struct xlate_ctx *ctx)
 {
     compose_ipfix_action(ctx->xbridge, &ctx->xout->odp_actions,
-                         &ctx->xin->flow);
+                         &ctx->xin->flow, ODPP_NONE);
+}
+
+static void
+add_ipfix_output_action(struct xlate_ctx *ctx, odp_port_t port)
+{
+    compose_ipfix_action(ctx->xbridge, &ctx->xout->odp_actions,
+                         &ctx->xin->flow, port);
 }
 
 /* Fix SAMPLE action according to data collected while composing ODP actions.
@@ -2519,6 +2551,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
             nl_msg_put_u32(&ctx->xout->odp_actions, OVS_ACTION_ATTR_RECIRC,
                            xr->recirc_id);
         } else {
+            add_ipfix_output_action(ctx, out_port);
             nl_msg_put_odp_port(&ctx->xout->odp_actions, OVS_ACTION_ATTR_OUTPUT,
                                 out_port);
         }
@@ -3360,7 +3393,7 @@ xlate_sample_action(struct xlate_ctx *ctx,
   compose_flow_sample_cookie(os->probability, os->collector_set_id,
                              os->obs_domain_id, os->obs_point_id, &cookie);
   compose_sample_action(ctx->xbridge, &ctx->xout->odp_actions, &ctx->xin->flow,
-                        probability, &cookie, sizeof cookie.flow_sample);
+                        probability, &cookie, sizeof cookie.flow_sample, NULL);
 }
 
 static bool
diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index 980b04f..12fb668 100644
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -1314,6 +1314,7 @@ destruct(struct ofproto *ofproto_)
 
     netflow_unref(ofproto->netflow);
     dpif_sflow_unref(ofproto->sflow);
+    dpif_ipfix_unref(ofproto->ipfix);
     hmap_destroy(&ofproto->bundles);
     mac_learning_unref(ofproto->ml);
     mcast_snooping_unref(ofproto->ms);
@@ -1602,6 +1603,9 @@ port_construct(struct ofport *port_)
     if (netdev_get_tunnel_config(netdev)) {
         tnl_port_add(port, port->up.netdev, port->odp_port);
         port->is_tunnel = true;
+        if (ofproto->ipfix) {
+           dpif_ipfix_add_tunnel_port(ofproto->ipfix, port_, port->odp_port);
+        }
     } else {
         /* Sanity-check that a mapping doesn't already exist.  This
          * shouldn't happen for non-tunnel ports. */
@@ -1663,6 +1667,10 @@ port_destruct(struct ofport *port_)
         ovs_rwlock_unlock(&ofproto->backer->odp_to_ofport_lock);
     }
 
+    if (port->is_tunnel && ofproto->ipfix) {
+       dpif_ipfix_del_tunnel_port(ofproto->ipfix, port->odp_port);
+    }
+
     tnl_port_del(port);
     sset_find_and_delete(&ofproto->ports, devname);
     sset_find_and_delete(&ofproto->ghost_ports, devname);
@@ -1764,9 +1772,11 @@ set_ipfix(
     struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
     struct dpif_ipfix *di = ofproto->ipfix;
     bool has_options = bridge_exporter_options || flow_exporters_options;
+    bool new_di = false;
 
     if (has_options && !di) {
         di = ofproto->ipfix = dpif_ipfix_create();
+        new_di = true;
     }
 
     if (di) {
@@ -1776,6 +1786,16 @@ set_ipfix(
             di, bridge_exporter_options, flow_exporters_options,
             n_flow_exporters_options);
 
+        /* Add tunnel ports only when a new ipfix created */
+        if (new_di == true) {
+            struct ofport_dpif *ofport;
+            HMAP_FOR_EACH (ofport, up.hmap_node, &ofproto->up.ports) {
+                if (ofport->is_tunnel == true) {
+                    dpif_ipfix_add_tunnel_port(di, &ofport->up, ofport->odp_port);
+                }
+            }
+        }
+
         if (!has_options) {
             dpif_ipfix_unref(di);
             ofproto->ipfix = NULL;
diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h
index c71662e..87e8bc8 100644
--- a/ofproto/ofproto.h
+++ b/ofproto/ofproto.h
@@ -71,6 +71,9 @@ struct ofproto_ipfix_bridge_exporter_options {
     uint32_t obs_point_id;  /* Bridge-wide Observation Point ID. */
     uint32_t cache_active_timeout;
     uint32_t cache_max_flows;
+    bool enable_tunnel_sampling;
+    bool enable_input_sampling;
+    bool enable_output_sampling;
 };
 
 struct ofproto_ipfix_flow_exporter_options {
diff --git a/ofproto/tunnel.c b/ofproto/tunnel.c
index 2b5aa50..46b0719 100644
--- a/ofproto/tunnel.c
+++ b/ofproto/tunnel.c
@@ -341,6 +341,10 @@ tnl_xlate_init(const struct flow *base_flow, struct flow *flow,
                                   FLOW_TNL_F_KEY);
         wc->masks.tunnel.ip_tos = UINT8_MAX;
         wc->masks.tunnel.ip_ttl = UINT8_MAX;
+        /* The tp_src and tp_dst members in flow_tnl are set to be always
+         * wildcarded, not to unwildcard them here. */
+        wc->masks.tunnel.tp_src = 0;
+        wc->masks.tunnel.tp_dst = 0;
 
         memset(&wc->masks.pkt_mark, 0xff, sizeof wc->masks.pkt_mark);
 
diff --git a/tests/odp.at b/tests/odp.at
index e725f70..5c00764 100644
--- a/tests/odp.at
+++ b/tests/odp.at
@@ -230,12 +230,19 @@ AT_SETUP([OVS datapath actions parsing and formatting - valid forms])
 AT_DATA([actions.txt], [dnl
 1,2,3
 userspace(pid=555666777)
+userspace(pid=555666777,tunnel_out_port=10)
 userspace(pid=6633,sFlow(vid=9,pcp=7,output=10))
+userspace(pid=6633,sFlow(vid=9,pcp=7,output=10),tunnel_out_port=10)
 userspace(pid=9765,slow_path())
+userspace(pid=9765,slow_path(),tunnel_out_port=10)
 userspace(pid=9765,slow_path(cfm))
+userspace(pid=9765,slow_path(cfm),tunnel_out_port=10)
 userspace(pid=1234567,userdata(0102030405060708090a0b0c0d0e0f))
+userspace(pid=1234567,userdata(0102030405060708090a0b0c0d0e0f),tunnel_out_port=10)
 userspace(pid=6633,flow_sample(probability=123,collector_set_id=1234,obs_domain_id=2345,obs_point_id=3456))
-userspace(pid=6633,ipfix)
+userspace(pid=6633,flow_sample(probability=123,collector_set_id=1234,obs_domain_id=2345,obs_point_id=3456),tunnel_out_port=10)
+userspace(pid=6633,ipfix(output_port=10))
+userspace(pid=6633,ipfix(output_port=10),tunnel_out_port=10)
 set(in_port(2))
 set(eth(src=00:01:02:03:04:05,dst=10:11:12:13:14:15))
 set(eth_type(0x1234))
diff --git a/utilities/ovs-vsctl.8.in b/utilities/ovs-vsctl.8.in
index d397721..a68b0bc 100644
--- a/utilities/ovs-vsctl.8.in
+++ b/utilities/ovs-vsctl.8.in
@@ -940,11 +940,15 @@ Deconfigure sFlow from \fBbr0\fR, which also destroys the sFlow record
 Configure bridge \fBbr0\fR to send one IPFIX flow record per packet
 sample to UDP port 4739 on host 192.168.0.34, with Observation Domain
 ID 123 and Observation Point ID 456, a flow cache active timeout of 1
-minute (60 seconds), and a maximum flow cache size of 13 flows:
+minute (60 seconds), maximum flow cache size of 13 flows, and flows
+sampled on output port with tunnel info(sampling on input and output
+port is enabled by default if not disabled) :
 .IP
 .B "ovs\-vsctl \-\- set Bridge br0 ipfix=@i \(rs"
 .IP
-.B "\-\- \-\-id=@i create IPFIX targets=\(rs\(dq192.168.0.34:4739\(rs\(dq obs_domain_id=123 obs_point_id=456 cache_active_timeout=60 cache_max_flows=13"
+.B "\-\- \-\-id=@i create IPFIX targets=\(rs\(dq192.168.0.34:4739\(rs\(dq obs_domain_id=123 obs_point_id=456 cache_active_timeout=60 cache_max_flows=13 \(rs"
+.IP
+.B "other_config:enable-input-sampling=false other_config:enable-tunnel-sampling=true"
 .PP
 Deconfigure the IPFIX settings from \fBbr0\fR, which also destroys the
 IPFIX record (since it is now unreferenced):
diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
index 6dcc2b8..ed12326 100644
--- a/vswitchd/bridge.c
+++ b/vswitchd/bridge.c
@@ -1160,6 +1160,15 @@ bridge_configure_ipfix(struct bridge *br)
         if (be_cfg->cache_max_flows) {
             be_opts.cache_max_flows = *be_cfg->cache_max_flows;
         }
+
+        be_opts.enable_tunnel_sampling = smap_get_bool(&be_cfg->other_config,
+                                             "enable-tunnel-sampling", true);
+
+        be_opts.enable_input_sampling = !smap_get_bool(&be_cfg->other_config,
+                                              "enable-input-sampling", false);
+
+        be_opts.enable_output_sampling = !smap_get_bool(&be_cfg->other_config,
+                                              "enable-output-sampling", false);
     }
 
     if (n_fe_opts > 0) {
diff --git a/vswitchd/vswitch.ovsschema b/vswitchd/vswitch.ovsschema
index bc9ea73..bf86f20 100644
--- a/vswitchd/vswitch.ovsschema
+++ b/vswitchd/vswitch.ovsschema
@@ -1,6 +1,6 @@
 {"name": "Open_vSwitch",
  "version": "7.8.0",
- "cksum": "2676751133 20740",
+ "cksum": "4147598271 20869",
  "tables": {
    "Open_vSwitch": {
      "columns": {
@@ -449,6 +449,9 @@
                           "minInteger": 0,
                           "maxInteger": 4294967295},
                   "min": 0, "max": 1}},
+       "other_config": {
+         "type": {"key": "string", "value": "string",
+                  "min": 0, "max": "unlimited"}},
        "external_ids": {
          "type": {"key": "string", "value": "string",
                   "min": 0, "max": "unlimited"}}}},
diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index d47fc1a..78ccda8 100644
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -3941,6 +3941,33 @@
       disabled.
     </column>
 
+    <column name="other_config" key="enable-tunnel-sampling"
+            type='{"type": "boolean"}'>
+      For per-bridge packet sampling, i.e. when this row is referenced
+      from a <ref table="Bridge"/>, enable sampling and reporting tunnel
+      header 7-tuples in IPFIX flow records. Disabled by default.
+      Ignored for per-flow sampling, i.e. when this row is referenced
+      from a <ref table="Flow_Sample_Collector_Set"/>.
+    </column>
+
+    <column name="other_config" key="enable-input-sampling"
+            type='{"type": "boolean"}'>
+      For per-bridge packet sampling, i.e. when this row is referenced
+      from a <ref table="Bridge"/>, enable sampling and reporting flows
+      at bridge port input in IPFIX flow records. Enabled by default.
+      Ignored for per-flow sampling, i.e. when this row is referenced
+      from a <ref table="Flow_Sample_Collector_Set"/>.
+    </column>
+
+    <column name="other_config" key="enable-output-sampling"
+            type='{"type": "boolean"}'>
+      For per-bridge packet sampling, i.e. when this row is referenced
+      from a <ref table="Bridge"/>, enable sampling and reporting flows
+      at bridge port output in IPFIX flow records. Enabled by default.
+      Ignored for per-flow sampling, i.e. when this row is referenced
+      from a <ref table="Flow_Sample_Collector_Set"/>.
+    </column>
+
     <group title="Common Columns">
       The overall purpose of these columns is described under <code>Common
       Columns</code> at the beginning of this document.
-- 
1.7.9.5




More information about the dev mailing list