[ovs-dev] [PATCH v2] Extend OVS IPFIX exporter to export tunnel headers

Pravin Shelar pshelar at nicira.com
Mon Jul 14 21:47:43 UTC 2014


On Sun, Jun 29, 2014 at 8:41 PM, Wenyu Zhang <wenyuz at vmware.com> wrote:
> Extend IPFIX exporter to export tunnel headers when both input and output
> of the port.
> Add three other_config options in IPFIX table: enable-input-sampling,
> enable-output-sampling and enable-tunnel-sampling, to control whether
> sampling tunnel info, on which direction (input or output).
> Insert sampling action before output action and the output tunnel port
> is sent to datapath in the sampling action.
> Make datapath collect output tunnel info and send it back to userpace
> in upcall message with a new additional optional attribute.
> Add a tunnel ports map to make the tunnel port lookup faster in sampling
> upcalls in IPFIX exporter. Make the IPFIX exporter generate IPFIX template
> sets with enterprise elements for the tunnel info, save the tunnel info
> in IPFIX cache entries, and send IPFIX DATA with tunnel info.
> Add flowDirection element in IPFIX templates.
>
> Signed-off-by: Wenyu Zhang <wenyuz at vmware.com>
> Acked-by: Romain Lenglet <rlenglet at vmware.com>
> ---
>  datapath/actions.c                    |   16 +-
>  datapath/datapath.c                   |   30 ++-
>  datapath/datapath.h                   |    2 +
>  datapath/flow.h                       |   25 +-
>  datapath/flow_netlink.c               |   56 ++++-
>  datapath/flow_netlink.h               |    2 +
>  datapath/vport-geneve.c               |   59 ++++-
>  datapath/vport-gre.c                  |   66 ++++-
>  datapath/vport-lisp.c                 |   66 ++++-
>  datapath/vport-vxlan.c                |   70 +++++-
>  datapath/vport.h                      |    4 +
>  include/linux/openvswitch.h           |   21 +-
>  lib/dpif-linux.c                      |    2 +
>  lib/dpif.h                            |    1 +
>  lib/odp-util.c                        |  183 +++++++++-----
>  lib/odp-util.h                        |    2 +
>  lib/packets.h                         |    9 +-
>  ofproto/automake.mk                   |    4 +
>  ofproto/ipfix-enterprise-entities.def |   19 ++
>  ofproto/ofproto-dpif-ipfix.c          |  448 ++++++++++++++++++++++++++++++---
>  ofproto/ofproto-dpif-ipfix.h          |   14 +-
>  ofproto/ofproto-dpif-upcall.c         |   20 +-
>  ofproto/ofproto-dpif-xlate.c          |   51 +++-
>  ofproto/ofproto-dpif.c                |   20 ++
>  ofproto/ofproto.h                     |    3 +
>  ofproto/tunnel.c                      |    4 +
>  tests/odp.at                          |    9 +-
>  utilities/ovs-vsctl.8.in              |    8 +-
>  vswitchd/bridge.c                     |    9 +
>  vswitchd/vswitch.ovsschema            |    5 +-
>  vswitchd/vswitch.xml                  |   27 ++
>  31 files changed, 1090 insertions(+), 165 deletions(-)
>  create mode 100644 ofproto/ipfix-enterprise-entities.def
>
> diff --git a/datapath/actions.c b/datapath/actions.c
> index cb26ad5..07f5749 100644
> --- a/datapath/actions.c
> +++ b/datapath/actions.c
> @@ -508,7 +508,9 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb,
>  {
>         struct dp_upcall_info upcall;
>         const struct nlattr *a;
> -       int rem;
> +       int rem, err;
> +       struct ovs_key_ipv4_tunnel output_tunnel_key;
> +       struct vport *vport;
>
>         BUG_ON(!OVS_CB(skb)->pkt_key);
>
> @@ -516,6 +518,7 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb,
>         upcall.key = OVS_CB(skb)->pkt_key;
>         upcall.userdata = NULL;
>         upcall.portid = 0;
> +       upcall.out_tun_key = NULL;
>
>         for (a = nla_data(attr), rem = nla_len(attr); rem > 0;
>                  a = nla_next(a, &rem)) {
> @@ -527,6 +530,17 @@ static int output_userspace(struct datapath *dp, struct sk_buff *skb,
>                 case OVS_USERSPACE_ATTR_PID:
>                         upcall.portid = nla_get_u32(a);
>                         break;
> +
> +               case OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT:
> +                       /* Get out tunnel key*/
> +                       vport = ovs_vport_ovsl_rcu(dp, nla_get_u32(a));
> +                       if (vport->ops->get_out_tun_key){
> +                               err = vport->ops->get_out_tun_key(vport, skb, &output_tunnel_key);
> +                               if (err == 0) {
> +                                       upcall.out_tun_key = &output_tunnel_key;
> +                               }
> +                        }
> +                       break;
>                 }
>         }
>
> diff --git a/datapath/datapath.c b/datapath/datapath.c
> index 4ec908e..bfdec44 100644
> --- a/datapath/datapath.c
> +++ b/datapath/datapath.c
> @@ -263,6 +263,7 @@ void ovs_dp_process_packet_with_key(struct sk_buff *skb,
>                 upcall.key = pkt_key;
>                 upcall.userdata = NULL;
>                 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
> +               upcall.out_tun_key = NULL;
>                 ovs_dp_upcall(dp, skb, &upcall);
>                 consume_skb(skb);
>                 stats_counter = &stats->n_missed;
> @@ -393,6 +394,8 @@ static size_t key_attr_size(void)
>                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
>                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
>                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
> +                 + nla_total_size(2)   /* OVS_TUNNEL_KEY_ATTR_TP_SRC */
> +                 + nla_total_size(2)   /* OVS_TUNNEL_KEY_ATTR_TP_DST */
>                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_OAM */
>                   + nla_total_size(256) /* OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS */
>                 + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
> @@ -409,7 +412,21 @@ static size_t key_attr_size(void)
>                 + nla_total_size(28); /* OVS_KEY_ATTR_ND */
>  }
>
> +static size_t out_tun_key_attr_size(void)
> +{
> +       return   nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
> +              + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
> +              + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
> +              + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
> +              + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
> +              + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
> +              + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
> +              + nla_total_size(2)   /* OVS_TUNNEL_KEY_ATTR_TP_SRC */
> +              + nla_total_size(2);  /* OVS_TUNNEL_KEY_ATTR_TP_DST */
> +}
> +
We can send all tunnel parameter than choosing some of them. userspace
can discard it if not needed.

>  static size_t upcall_msg_size(const struct nlattr *userdata,
> +                              bool out_tun_key,
>                               unsigned int hdrlen)
>  {
>         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
> @@ -420,6 +437,10 @@ static size_t upcall_msg_size(const struct nlattr *userdata,
>         if (userdata)
>                 size += NLA_ALIGN(userdata->nla_len);
>
> +       /* OVS_PACKET_ATTR_OUT_TUNNEL_KEY */
> +        if (out_tun_key)
> +               size += nla_total_size(out_tun_key_attr_size());
> +
>         return size;
>  }
>
> @@ -477,7 +498,8 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
>         else
>                 hlen = skb->len;
>
> -       len = upcall_msg_size(upcall_info->userdata, hlen);
> +       len = upcall_msg_size(upcall_info->userdata, upcall_info->out_tun_key != NULL,
> +                             hlen);
>         user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
>         if (!user_skb) {
>                 err = -ENOMEM;
> @@ -499,6 +521,12 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
>                           nla_len(upcall_info->userdata),
>                           nla_data(upcall_info->userdata));
>
> +       if (upcall_info->out_tun_key) {
> +               nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_OUT_TUNNEL_KEY);
> +               ovs_nla_put_out_tunnel_key(user_skb, upcall_info->out_tun_key);
> +               nla_nest_end(user_skb, nla);
> +        }
> +
>         /* Only reserve room for attribute header, packet data is added
>          * in skb_zerocopy() */
>         if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
> diff --git a/datapath/datapath.h b/datapath/datapath.h
> index fcd8e86..b6bb9a6 100644
> --- a/datapath/datapath.h
> +++ b/datapath/datapath.h
> @@ -119,12 +119,14 @@ struct ovs_skb_cb {
>   * @portid: Netlink PID to which packet should be sent.  If @portid is 0 then no
>   * packet is sent and the packet is accounted in the datapath's @n_lost
>   * counter.
> + * @out_tun_key: Becomes %OVS_PACKET_ATTR_OUT_TUNNEL_KEY.
>   */
>  struct dp_upcall_info {
>         u8 cmd;
>         const struct sw_flow_key *key;
>         const struct nlattr *userdata;
>         u32 portid;
> +       const struct ovs_key_ipv4_tunnel * out_tun_key;
>  };
>
>  /**
> diff --git a/datapath/flow.h b/datapath/flow.h
> index f6afa48..a48474b 100644
> --- a/datapath/flow.h
> +++ b/datapath/flow.h
> @@ -39,8 +39,8 @@ struct sk_buff;
>
>  /* Used to memset ovs_key_ipv4_tunnel padding. */
>  #define OVS_TUNNEL_KEY_SIZE                                    \
> -        (offsetof(struct ovs_key_ipv4_tunnel, ipv4_ttl) +      \
> -         FIELD_SIZEOF(struct ovs_key_ipv4_tunnel, ipv4_ttl))
> +        (offsetof(struct ovs_key_ipv4_tunnel, tp_dst) +        \
> +         FIELD_SIZEOF(struct ovs_key_ipv4_tunnel, tp_dst))
>
>  struct ovs_key_ipv4_tunnel {
>         __be64 tun_id;
> @@ -49,6 +49,8 @@ struct ovs_key_ipv4_tunnel {
>         __be16 tun_flags;
>         u8   ipv4_tos;
>         u8   ipv4_ttl;
> +       __be16 tp_src;
> +       __be16 tp_dst;
>  } __packed __aligned(4); /* Minimize padding. */
>
>  struct ovs_tunnel_info {
> @@ -67,7 +69,8 @@ struct ovs_tunnel_info {
>                                            opt_len)
>
>  static inline void ovs_flow_tun_info_init(struct ovs_tunnel_info *tun_info,
> -                                        const struct iphdr *iph, __be64 tun_id,
> +                                        const struct iphdr *iph,  const void *tph,
> +                                         __be64 tun_id,
>                                          __be16 tun_flags,
>                                          struct geneve_opt *opts,
>                                          u8 opts_len)
> @@ -79,6 +82,22 @@ static inline void ovs_flow_tun_info_init(struct ovs_tunnel_info *tun_info,
>         tun_info->tunnel.ipv4_ttl = iph->ttl;
>         tun_info->tunnel.tun_flags = tun_flags;
>
> +        /* For the tunnel types on the top of IPsec,the tp_src and tp_dst of
> +         * the upper tunnel are used.
> +         * E.g: GRE over IPSEC, the tp_src and tp_port are zero.
> +         *      STT over IPSEC, the L4 ports of STT tunnel packet are used.
> +         */
> +       if (tph &&
> +           (iph->protocol == IPPROTO_UDP ||
> +            iph->protocol == IPPROTO_TCP)) {
> +               const struct udphdr * udp_hdr = tph;
> +               tun_info->tunnel.tp_src = udp_hdr->source;
> +               tun_info->tunnel.tp_dst = udp_hdr->dest;
> +       } else {
> +               tun_info->tunnel.tp_src = 0;
> +               tun_info->tunnel.tp_dst = 0;
> +        }
> +
If we pass port number to this function, we can avoid all these checks.

>         /* clear struct padding. */
>         memset((unsigned char *) &tun_info->tunnel + OVS_TUNNEL_KEY_SIZE, 0,
>                sizeof(tun_info->tunnel) - OVS_TUNNEL_KEY_SIZE);
> diff --git a/datapath/flow_netlink.c b/datapath/flow_netlink.c
> index 5a978f0..0e38451 100644
> --- a/datapath/flow_netlink.c
> +++ b/datapath/flow_netlink.c
> @@ -361,6 +361,8 @@ static int ipv4_tun_from_nlattr(const struct nlattr *attr,
>                         [OVS_TUNNEL_KEY_ATTR_TTL] = 1,
>                         [OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT] = 0,
>                         [OVS_TUNNEL_KEY_ATTR_CSUM] = 0,
> +                       [OVS_TUNNEL_KEY_ATTR_TP_SRC] = sizeof(u16),
> +                       [OVS_TUNNEL_KEY_ATTR_TP_DST] = sizeof(u16),
>                         [OVS_TUNNEL_KEY_ATTR_OAM] = 0,
>                         [OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS] = -1,
>                 };
> @@ -408,6 +410,13 @@ static int ipv4_tun_from_nlattr(const struct nlattr *attr,
>                 case OVS_TUNNEL_KEY_ATTR_CSUM:
>                         tun_flags |= TUNNEL_CSUM;
>                         break;
> +               case OVS_TUNNEL_KEY_ATTR_TP_SRC:
> +                       SW_FLOW_KEY_PUT(match, tun_key.tp_src,
> +                                       nla_get_be16(a), is_mask);
> +                       break;
> +               case OVS_TUNNEL_KEY_ATTR_TP_DST:
> +                       SW_FLOW_KEY_PUT(match, tun_key.tp_dst,
> +                                       nla_get_be16(a), is_mask);
>                 case OVS_TUNNEL_KEY_ATTR_OAM:
>                         tun_flags |= TUNNEL_OAM;
>                         break;
> @@ -488,17 +497,11 @@ static int ipv4_tun_from_nlattr(const struct nlattr *attr,
>         return 0;
>  }
>
> -static int ipv4_tun_to_nlattr(struct sk_buff *skb,
> -                             const struct ovs_key_ipv4_tunnel *output,
> -                             const struct geneve_opt *tun_opts,
> -                             int swkey_tun_opts_len)
> +static int ipv4_tun_to_nlattr__(struct sk_buff *skb,
> +                               const struct ovs_key_ipv4_tunnel *output,
> +                               const struct geneve_opt *tun_opts,
> +                               int swkey_tun_opts_len)
>  {
> -       struct nlattr *nla;
> -
> -       nla = nla_nest_start(skb, OVS_KEY_ATTR_TUNNEL);
> -       if (!nla)
> -               return -EMSGSIZE;
> -
>         if (output->tun_flags & TUNNEL_KEY &&
>             nla_put_be64(skb, OVS_TUNNEL_KEY_ATTR_ID, output->tun_id))
>                 return -EMSGSIZE;
> @@ -519,6 +522,12 @@ static int ipv4_tun_to_nlattr(struct sk_buff *skb,
>         if ((output->tun_flags & TUNNEL_CSUM) &&
>                 nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_CSUM))
>                 return -EMSGSIZE;
> +       if (output->tp_src &&
> +               nla_put_be16(skb, OVS_TUNNEL_KEY_ATTR_TP_SRC, output->tp_src))
> +               return -EMSGSIZE;
> +       if (output->tp_dst &&
> +               nla_put_be16(skb, OVS_TUNNEL_KEY_ATTR_TP_DST, output->tp_dst))
> +               return -EMSGSIZE;
>         if ((output->tun_flags & TUNNEL_OAM) &&
>                 nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_OAM))
>                 return -EMSGSIZE;
> @@ -526,10 +535,36 @@ static int ipv4_tun_to_nlattr(struct sk_buff *skb,
>             nla_put(skb, OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS,
>                     swkey_tun_opts_len, tun_opts));
>
> +       return 0;
> +}
> +
> +
> +static int ipv4_tun_to_nlattr(struct sk_buff *skb,
> +                             const struct ovs_key_ipv4_tunnel *output,
> +                             const struct geneve_opt *tun_opts,
> +                             int swkey_tun_opts_len)
> +{
> +       struct nlattr *nla;
> +       int err;
> +
> +       nla = nla_nest_start(skb, OVS_KEY_ATTR_TUNNEL);
> +       if (!nla)
> +               return -EMSGSIZE;
> +
> +       err = ipv4_tun_to_nlattr__(skb, output, tun_opts, swkey_tun_opts_len);
> +       if (err) {
> +               return err;
> +       }
> +
>         nla_nest_end(skb, nla);
>         return 0;
>  }
>
> +int ovs_nla_put_out_tunnel_key(struct sk_buff *skb,
> +                              const struct ovs_key_ipv4_tunnel *output)
> +{
> +       return ipv4_tun_to_nlattr__(skb, output, NULL, 0);
> +}
>
>  static int metadata_from_nlattrs(struct sw_flow_match *match,  u64 *attrs,
>                                  const struct nlattr **a, bool is_mask)
> @@ -1578,6 +1613,7 @@ static int validate_userspace(const struct nlattr *attr)
>         static const struct nla_policy userspace_policy[OVS_USERSPACE_ATTR_MAX + 1] = {
>                 [OVS_USERSPACE_ATTR_PID] = {.type = NLA_U32 },
>                 [OVS_USERSPACE_ATTR_USERDATA] = {.type = NLA_UNSPEC },
> +               [OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT] = {.type = NLA_U32 },
>         };
>         struct nlattr *a[OVS_USERSPACE_ATTR_MAX + 1];
>         int error;
> diff --git a/datapath/flow_netlink.h b/datapath/flow_netlink.h
> index 0c20e86..4070ddf 100644
> --- a/datapath/flow_netlink.h
> +++ b/datapath/flow_netlink.h
> @@ -47,6 +47,8 @@ int ovs_nla_get_flow_metadata(struct sw_flow *flow,
>  int ovs_nla_get_match(struct sw_flow_match *match,
>                       const struct nlattr *,
>                       const struct nlattr *);
> +int ovs_nla_put_out_tunnel_key(struct sk_buff *,
> +                              const struct ovs_key_ipv4_tunnel *);
>
>  int ovs_nla_copy_actions(const struct nlattr *attr,
>                          const struct sw_flow_key *key,
> diff --git a/datapath/vport-geneve.c b/datapath/vport-geneve.c
> index 33047f2..7c712ea 100644
> --- a/datapath/vport-geneve.c
> +++ b/datapath/vport-geneve.c
> @@ -195,7 +195,8 @@ static int geneve_rcv(struct sock *sk, struct sk_buff *skb)
>                 (geneveh->critical ? TUNNEL_CRIT_OPT : 0);
>
>         key = vni_to_tunnel_id(geneveh->vni);
> -       ovs_flow_tun_info_init(&tun_info, ip_hdr(skb), key, flags,
> +       ovs_flow_tun_info_init(&tun_info, ip_hdr(skb),
> +                               skb_transport_header(skb), key, flags,
>                                 geneveh->options, opts_len);
>
>         ovs_vport_receive(vport_from_priv(geneve_port), skb, &tun_info);
> @@ -434,11 +435,55 @@ static const char *geneve_get_name(const struct vport *vport)
>         return geneve_port->name;
>  }
>
> +static int geneve_get_out_tun_key(struct vport *vport, struct sk_buff *skb,
> +                                 struct ovs_key_ipv4_tunnel * out_tun_key)
> +{
> +       struct geneve_port *geneve_port = geneve_vport(vport);
> +       struct ovs_key_ipv4_tunnel *tun_key;
> +       struct rtable *rt;
> +       __be32 saddr;
> +       int err = 0;
> +
> +
> +       if (unlikely(!OVS_CB(skb)->tun_info))
> +               return -EINVAL;
> +
> +       tun_key = &OVS_CB(skb)->tun_info->tunnel;
> +
> +       /* Route lookup */
> +       saddr = tun_key->ipv4_src;
> +       rt = find_route(ovs_dp_get_net(vport->dp),
> +                       &saddr, tun_key->ipv4_dst,
> +                       IPPROTO_UDP, tun_key->ipv4_tos,
> +                       skb->mark);
> +       if (IS_ERR(rt)) {
> +               err = PTR_ERR(rt);
> +               goto error;
> +       }
> +       ip_rt_put(rt);
> +
> +       /* Now the tun_key should contain the output tun key*/
> +       memset(out_tun_key, 0, sizeof(*out_tun_key));
> +       out_tun_key->tun_flags |= TUNNEL_KEY;
> +       out_tun_key->tun_id = tun_key->tun_id;
> +       out_tun_key->ipv4_src = saddr;
> +       out_tun_key->ipv4_dst = tun_key->ipv4_dst;
> +       out_tun_key->ipv4_tos = tun_key->ipv4_tos;
> +       out_tun_key->ipv4_ttl = tun_key->ipv4_ttl;
> +       /* Get tp_src and tp_dst, refert to geneve_build_header() */
> +       out_tun_key->tp_src = vxlan_src_port(1, USHRT_MAX, skb);
> +       out_tun_key->tp_dst = inet_sport(geneve_port->sock->sk);
> +
You can just copy tun_key to out_tun_key and avoid memset.

> +error:
> +       return err;
> +}
> +
All definitions of get_out_tun_key() are pretty much same, we can
avoid code duplication by adding one api that take required parameter
and set out_tun_key. This new api can be called from all
get_out_tun_key() definitions.

>  const struct vport_ops ovs_geneve_vport_ops = {
> -       .type           = OVS_VPORT_TYPE_GENEVE,
> -       .create         = geneve_tnl_create,
> -       .destroy        = geneve_tnl_destroy,
> -       .get_name       = geneve_get_name,
> -       .get_options    = geneve_get_options,
> -       .send           = geneve_send,
> +       .type                   = OVS_VPORT_TYPE_GENEVE,
> +       .create                 = geneve_tnl_create,
> +       .destroy                = geneve_tnl_destroy,
> +       .get_name               = geneve_get_name,
> +       .get_options            = geneve_get_options,
> +       .send                   = geneve_send,
> +       .get_out_tun_key        = geneve_get_out_tun_key,
>  };
> diff --git a/datapath/vport-gre.c b/datapath/vport-gre.c
> index d2a2602..8a807a0 100644
> --- a/datapath/vport-gre.c
> +++ b/datapath/vport-gre.c
> @@ -110,7 +110,7 @@ static int gre_rcv(struct sk_buff *skb,
>                 return PACKET_REJECT;
>
>         key = key_to_tunnel_id(tpi->key, tpi->seq);
> -       ovs_flow_tun_info_init(&tun_info, ip_hdr(skb), key,
> +       ovs_flow_tun_info_init(&tun_info, ip_hdr(skb), NULL, key,
>                                filter_tnl_flags(tpi->flags), NULL, 0);
>
>         ovs_vport_receive(vport, skb, &tun_info);
> @@ -292,12 +292,55 @@ static int gre_send(struct vport *vport, struct sk_buff *skb)
>         return __send(vport, skb, hlen, 0, 0);
>  }
>
> +static int gre_get_out_tun_key(struct vport *vport, struct sk_buff *skb,
> +                               struct ovs_key_ipv4_tunnel * out_tun_key)
> +{
> +       struct ovs_key_ipv4_tunnel *tun_key;
> +       struct rtable *rt;
> +       __be32 saddr;
> +       int err = 0;
> +
> +       if (unlikely(!OVS_CB(skb)->tun_info))
> +               return -EINVAL;
> +
> +       tun_key = &OVS_CB(skb)->tun_info->tunnel;
> +
> +       /* Route lookup */
> +       saddr = tun_key->ipv4_src;
> +       rt = find_route(ovs_dp_get_net(vport->dp),
> +                       &saddr,
> +                       tun_key->ipv4_dst,
> +                       IPPROTO_GRE,
> +                       tun_key->ipv4_tos,
> +                       skb->mark);
> +       if (IS_ERR(rt)) {
> +               err = PTR_ERR(rt);
> +               goto error;
> +       }
> +
> +       ip_rt_put(rt);
> +
> +       /* Now the tun_key should contain the output tun key*/
> +       memset(out_tun_key, 0, sizeof(*out_tun_key));
> +       out_tun_key->tun_flags |= TUNNEL_KEY;
> +       out_tun_key->tun_id = tun_key->tun_id;
> +       out_tun_key->ipv4_src = saddr;
> +       out_tun_key->ipv4_dst = tun_key->ipv4_dst;
> +       out_tun_key->ipv4_tos = tun_key->ipv4_tos;
> +       out_tun_key->ipv4_ttl = tun_key->ipv4_ttl;
> +       out_tun_key->tp_src = 0;
> +       out_tun_key->tp_dst = 0;
> +error:
> +       return err;
> +}
> +
>  const struct vport_ops ovs_gre_vport_ops = {
> -       .type           = OVS_VPORT_TYPE_GRE,
> -       .create         = gre_create,
> -       .destroy        = gre_tnl_destroy,
> -       .get_name       = gre_get_name,
> -       .send           = gre_send,
> +       .type                   = OVS_VPORT_TYPE_GRE,
> +       .create                 = gre_create,
> +       .destroy                = gre_tnl_destroy,
> +       .get_name               = gre_get_name,
> +       .send                   = gre_send,
> +       .get_out_tun_key        = gre_get_out_tun_key,
>  };
>
>  /* GRE64 vport. */
> @@ -369,10 +412,11 @@ static int gre64_send(struct vport *vport, struct sk_buff *skb)
>  }
>
>  const struct vport_ops ovs_gre64_vport_ops = {
> -       .type           = OVS_VPORT_TYPE_GRE64,
> -       .create         = gre64_create,
> -       .destroy        = gre64_tnl_destroy,
> -       .get_name       = gre_get_name,
> -       .send           = gre64_send,
> +       .type                   = OVS_VPORT_TYPE_GRE64,
> +       .create                 = gre64_create,
> +       .destroy                = gre64_tnl_destroy,
> +       .get_name               = gre_get_name,
> +       .send                   = gre64_send,
> +       .get_out_tun_key        = gre_get_out_tun_key,
>  };
>  #endif
> diff --git a/datapath/vport-lisp.c b/datapath/vport-lisp.c
> index c41e09e..9cbbe47 100644
> --- a/datapath/vport-lisp.c
> +++ b/datapath/vport-lisp.c
> @@ -245,7 +245,8 @@ static int lisp_rcv(struct sock *sk, struct sk_buff *skb)
>
>         /* Save outer tunnel values */
>         iph = ip_hdr(skb);
> -       ovs_flow_tun_info_init(&tun_info, iph, key, TUNNEL_KEY, NULL, 0);
> +       ovs_flow_tun_info_init(&tun_info, iph, skb_transport_header(skb),
> +                              key, TUNNEL_KEY, NULL, 0);
>
>         /* Drop non-IP inner packets */
>         inner_iph = (struct iphdr *)(lisph + 1);
> @@ -509,11 +510,62 @@ static const char *lisp_get_name(const struct vport *vport)
>         return lisp_port->name;
>  }
>
> +static int lisp_get_out_tun_key(struct vport *vport, struct sk_buff *skb,
> +                                struct ovs_key_ipv4_tunnel * out_tun_key)
> +{
> +       struct ovs_key_ipv4_tunnel *tun_key;
> +       struct rtable *rt;
> +       struct net *net = ovs_dp_get_net(vport->dp);
> +       struct lisp_port *lisp_port = lisp_vport(vport);
> +       __be32 saddr;
> +       int err = 0;
> +
> +
> +       if (unlikely(!OVS_CB(skb)->tun_info))
> +               return -EINVAL;
> +
> +       tun_key = &OVS_CB(skb)->tun_info->tunnel;
> +
> +       if (skb->protocol != htons(ETH_P_IP) &&
> +           skb->protocol != htons(ETH_P_IPV6)) {
> +               return 1;
> +       }
> +
> +       /* Route lookup */
> +       saddr = tun_key->ipv4_src;
> +       rt = find_route(ovs_dp_get_net(vport->dp),
> +                       &saddr,
> +                       tun_key->ipv4_dst,
> +                       IPPROTO_UDP,
> +                       tun_key->ipv4_tos,
> +                       skb->mark);
> +       if (IS_ERR(rt)) {
> +               err = PTR_ERR(rt);
> +               goto error;
> +       }
> +       ip_rt_put(rt);
> +
> +       /* Now the tun_key should contain the output tun key*/
> +       memset(out_tun_key, 0, sizeof(*out_tun_key));
> +       out_tun_key->tun_flags |= TUNNEL_KEY;
> +       out_tun_key->tun_id = tun_key->tun_id;
> +       out_tun_key->ipv4_src = saddr;
> +       out_tun_key->ipv4_dst = tun_key->ipv4_dst;
> +       out_tun_key->ipv4_tos = tun_key->ipv4_tos;
> +       out_tun_key->ipv4_ttl = tun_key->ipv4_ttl;
> +       /* Get tp_src and tp_dst, refer to lisp_build_header()*/
> +       out_tun_key->tp_src = htons(get_src_port(net, skb));
> +       out_tun_key->tp_dst = lisp_port->dst_port;
> +error:
> +       return err;
> +}
> +
>  const struct vport_ops ovs_lisp_vport_ops = {
> -       .type           = OVS_VPORT_TYPE_LISP,
> -       .create         = lisp_tnl_create,
> -       .destroy        = lisp_tnl_destroy,
> -       .get_name       = lisp_get_name,
> -       .get_options    = lisp_get_options,
> -       .send           = lisp_send,
> +       .type                   = OVS_VPORT_TYPE_LISP,
> +       .create                 = lisp_tnl_create,
> +       .destroy                = lisp_tnl_destroy,
> +       .get_name               = lisp_get_name,
> +       .get_options            = lisp_get_options,
> +       .send                   = lisp_send,
> +       .get_out_tun_key        = lisp_get_out_tun_key,
>  };
> diff --git a/datapath/vport-vxlan.c b/datapath/vport-vxlan.c
> index d93a844..16f47b9 100644
> --- a/datapath/vport-vxlan.c
> +++ b/datapath/vport-vxlan.c
> @@ -68,7 +68,8 @@ static void vxlan_rcv(struct vxlan_sock *vs, struct sk_buff *skb, __be32 vx_vni)
>         /* Save outer tunnel values */
>         iph = ip_hdr(skb);
>         key = cpu_to_be64(ntohl(vx_vni) >> 8);
> -       ovs_flow_tun_info_init(&tun_info, iph, key, TUNNEL_KEY, NULL, 0);
> +       ovs_flow_tun_info_init(&tun_info, iph, skb_transport_header(skb),
> +                              key, TUNNEL_KEY, NULL, 0);
>
>         ovs_vport_receive(vport, skb, &tun_info);
>  }
> @@ -187,6 +188,60 @@ error:
>         return err;
>  }
>
> +static int vxlan_get_out_tun_key(struct vport *vport, struct sk_buff *skb,
> +                                 struct ovs_key_ipv4_tunnel * out_tun_key)
> +{
> +       struct ovs_key_ipv4_tunnel *tun_key;
> +       struct net *net = ovs_dp_get_net(vport->dp);
> +       struct vxlan_port *vxlan_port = vxlan_vport(vport);
> +       __be16 dst_port = inet_sport(vxlan_port->vs->sock->sk);
> +       struct rtable *rt;
> +       __be16 src_port;
> +       __be32 saddr;
> +       int port_min;
> +       int port_max;
> +       int err = 0;
> +
> +       if (unlikely(!OVS_CB(skb)->tun_info)) {
> +               err = -EINVAL;
> +               goto error;
> +       }
> +
> +       tun_key = &OVS_CB(skb)->tun_info->tunnel;
> +
> +       /* Route lookup */
> +       saddr = tun_key->ipv4_src;
> +       rt = find_route(ovs_dp_get_net(vport->dp),
> +                       &saddr,
> +                       tun_key->ipv4_dst,
> +                       IPPROTO_UDP,
> +                       tun_key->ipv4_tos,
> +                       skb->mark);
> +       if (IS_ERR(rt)) {
> +               err = PTR_ERR(rt);
> +               goto error;
> +       }
> +       ip_rt_put(rt);
> +
> +       inet_get_local_port_range(net, &port_min, &port_max);
> +       src_port = vxlan_src_port(port_min, port_max, skb);
> +
> +       /* Now the tun_key should contain the output tun key*/
> +       memset(out_tun_key, 0, sizeof(*out_tun_key));
> +       out_tun_key->tun_flags |= TUNNEL_KEY;
> +       out_tun_key->tun_id = tun_key->tun_id;
> +       out_tun_key->ipv4_src = saddr;
> +       out_tun_key->ipv4_dst = tun_key->ipv4_dst;
> +       out_tun_key->ipv4_tos = tun_key->ipv4_tos;
> +       out_tun_key->ipv4_ttl = tun_key->ipv4_ttl;
> +       out_tun_key->tp_src = src_port;
> +       out_tun_key->tp_dst = dst_port;
> +
> +error:
> +       return err;
> +
> +}
> +
>  static const char *vxlan_get_name(const struct vport *vport)
>  {
>         struct vxlan_port *vxlan_port = vxlan_vport(vport);
> @@ -194,10 +249,11 @@ static const char *vxlan_get_name(const struct vport *vport)
>  }
>
>  const struct vport_ops ovs_vxlan_vport_ops = {
> -       .type           = OVS_VPORT_TYPE_VXLAN,
> -       .create         = vxlan_tnl_create,
> -       .destroy        = vxlan_tnl_destroy,
> -       .get_name       = vxlan_get_name,
> -       .get_options    = vxlan_get_options,
> -       .send           = vxlan_tnl_send,
> +       .type                   = OVS_VPORT_TYPE_VXLAN,
> +       .create                 = vxlan_tnl_create,
> +       .destroy                = vxlan_tnl_destroy,
> +       .get_name               = vxlan_get_name,
> +       .get_options            = vxlan_get_options,
> +       .send                   = vxlan_tnl_send,
> +       .get_out_tun_key        = vxlan_get_out_tun_key,
>  };
> diff --git a/datapath/vport.h b/datapath/vport.h
> index bdd9a89..37a6010 100644
> --- a/datapath/vport.h
> +++ b/datapath/vport.h
> @@ -150,6 +150,7 @@ struct vport_parms {
>   * @get_name: Get the device's name.
>   * @send: Send a packet on the device.  Returns the length of the packet sent,
>   * zero for dropped packets or negative for error.
> + * @get_out_tun_key: Get the output tunnel 5-tuple for a packet.
>   */
>  struct vport_ops {
>         enum ovs_vport_type type;
> @@ -165,6 +166,9 @@ struct vport_ops {
>         const char *(*get_name)(const struct vport *);
>
>         int (*send)(struct vport *, struct sk_buff *);
> +
> +       int (*get_out_tun_key)(struct vport *, struct sk_buff *,
> +                              struct ovs_key_ipv4_tunnel *);
>  };
>
>  enum vport_err_type {
> diff --git a/include/linux/openvswitch.h b/include/linux/openvswitch.h
> index bf27dcb..e593439 100644
> --- a/include/linux/openvswitch.h
> +++ b/include/linux/openvswitch.h
> @@ -180,16 +180,22 @@ enum ovs_packet_cmd {
>   * notification if the %OVS_ACTION_ATTR_USERSPACE action specified an
>   * %OVS_USERSPACE_ATTR_USERDATA attribute, with the same length and content
>   * specified there.
> + * @OVS_PACKET_ATTR_OUT_TUNNEL_KEY: Present for an %OVS_PACKET_CMD_ACTION
> + * notification if the %OVS_ACTION_ATTR_USERSPACE action specified an
> + * %OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT attribute, which is sent only if the
> + * output port is actually a tunnel port. Contains the output tunnel key
> + * extracted from the packet as nested %OVS_TUNNEL_KEY_ATTR_* attributes.
>   *
>   * These attributes follow the &struct ovs_header within the Generic Netlink
>   * payload for %OVS_PACKET_* commands.
>   */
>  enum ovs_packet_attr {
>         OVS_PACKET_ATTR_UNSPEC,
> -       OVS_PACKET_ATTR_PACKET,      /* Packet data. */
> -       OVS_PACKET_ATTR_KEY,         /* Nested OVS_KEY_ATTR_* attributes. */
> -       OVS_PACKET_ATTR_ACTIONS,     /* Nested OVS_ACTION_ATTR_* attributes. */
> -       OVS_PACKET_ATTR_USERDATA,    /* OVS_ACTION_ATTR_USERSPACE arg. */
> +       OVS_PACKET_ATTR_PACKET,         /* Packet data. */
> +       OVS_PACKET_ATTR_KEY,            /* Nested OVS_KEY_ATTR_* attributes. */
> +       OVS_PACKET_ATTR_ACTIONS,        /* Nested OVS_ACTION_ATTR_* attributes. */
> +       OVS_PACKET_ATTR_USERDATA,       /* OVS_ACTION_ATTR_USERSPACE arg. */
> +       OVS_PACKET_ATTR_OUT_TUNNEL_KEY, /* Nested OVS_TUNNEL_KEY_ATTR_* attributes */
>         __OVS_PACKET_ATTR_MAX
>  };
>
> @@ -340,6 +346,8 @@ enum ovs_tunnel_key_attr {
>         OVS_TUNNEL_KEY_ATTR_TTL,                /* u8 Tunnel IP TTL. */
>         OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT,      /* No argument, set DF. */
>         OVS_TUNNEL_KEY_ATTR_CSUM,               /* No argument. CSUM packet. */
> +       OVS_TUNNEL_KEY_ATTR_TP_SRC,             /* u16 Tunnel Source Transport Port. */
> +       OVS_TUNNEL_KEY_ATTR_TP_DST,             /* u16 Tunnel Destination Transport Port. */
>         OVS_TUNNEL_KEY_ATTR_OAM,                /* No argument, OAM frame. */
>         OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS,        /* Array of Geneve options */
>         __OVS_TUNNEL_KEY_ATTR_MAX
> @@ -506,11 +514,16 @@ enum ovs_sample_attr {
>   * message should be sent.  Required.
>   * @OVS_USERSPACE_ATTR_USERDATA: If present, its variable-length argument is
>   * copied to the %OVS_PACKET_CMD_ACTION message as %OVS_PACKET_ATTR_USERDATA.
> + * @OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT: If present, u32 output port id to get
> + * tunnel info.
>   */
>  enum ovs_userspace_attr {
>         OVS_USERSPACE_ATTR_UNSPEC,
>         OVS_USERSPACE_ATTR_PID,       /* u32 Netlink PID to receive upcalls. */
>         OVS_USERSPACE_ATTR_USERDATA,  /* Optional user-specified cookie. */
> +       OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT,  /* Optional, u32 output port id
> +                                              * to get tunnel info.
> +                                              */
>         __OVS_USERSPACE_ATTR_MAX
>  };
>
> diff --git a/lib/dpif-linux.c b/lib/dpif-linux.c
> index 0eac3e7..da31869 100644
> --- a/lib/dpif-linux.c
> +++ b/lib/dpif-linux.c
> @@ -1700,6 +1700,7 @@ parse_odp_packet(struct ofpbuf *buf, struct dpif_upcall *upcall,
>
>          /* OVS_PACKET_CMD_ACTION only. */
>          [OVS_PACKET_ATTR_USERDATA] = { .type = NL_A_UNSPEC, .optional = true },
> +        [OVS_PACKET_ATTR_OUT_TUNNEL_KEY] = { .type = NL_A_NESTED, .optional = true },
>      };
>
>      struct ovs_header *ovs_header;
> @@ -1734,6 +1735,7 @@ parse_odp_packet(struct ofpbuf *buf, struct dpif_upcall *upcall,
>                               nl_attr_get(a[OVS_PACKET_ATTR_KEY]));
>      upcall->key_len = nl_attr_get_size(a[OVS_PACKET_ATTR_KEY]);
>      upcall->userdata = a[OVS_PACKET_ATTR_USERDATA];
> +    upcall->out_tun_key = a[OVS_PACKET_ATTR_OUT_TUNNEL_KEY];
>
>      /* Allow overwriting the netlink attribute header without reallocating. */
>      ofpbuf_use_stub(&upcall->packet,
> diff --git a/lib/dpif.h b/lib/dpif.h
> index f080cde..07cdba8 100644
> --- a/lib/dpif.h
> +++ b/lib/dpif.h
> @@ -668,6 +668,7 @@ struct dpif_upcall {
>
>      /* DPIF_UC_ACTION only. */
>      struct nlattr *userdata;    /* Argument to OVS_ACTION_ATTR_USERSPACE. */
> +    struct nlattr *out_tun_key;    /* Argument to OVS_ACTION_ATTR_USERSPACE. */
>  };
>
>  int dpif_recv_set(struct dpif *, bool enable);
> diff --git a/lib/odp-util.c b/lib/odp-util.c
> index 162d85a..9644a4a 100644
> --- a/lib/odp-util.c
> +++ b/lib/odp-util.c
> @@ -261,9 +261,12 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
>          [OVS_USERSPACE_ATTR_PID] = { .type = NL_A_U32 },
>          [OVS_USERSPACE_ATTR_USERDATA] = { .type = NL_A_UNSPEC,
>                                            .optional = true },
> +        [OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT] = { .type = NL_A_U32,
> +                                                 .optional = true },
>      };
>      struct nlattr *a[ARRAY_SIZE(ovs_userspace_policy)];
>      const struct nlattr *userdata_attr;
> +    const struct nlattr *tunnel_out_port_attr;
>
>      if (!nl_parse_nested(attr, ovs_userspace_policy, a, ARRAY_SIZE(a))) {
>          ds_put_cstr(ds, "userspace(error)");
> @@ -314,7 +317,8 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
>                                cookie.flow_sample.obs_point_id);
>              } else if (userdata_len >= sizeof cookie.ipfix
>                         && cookie.type == USER_ACTION_COOKIE_IPFIX) {
> -                ds_put_format(ds, ",ipfix");
> +                ds_put_format(ds, ",ipfix(output_port=%"PRIu32")",
> +                              cookie.ipfix.output_odp_port);
>              } else {
>                  userdata_unspec = true;
>              }
> @@ -330,6 +334,12 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
>          }
>      }
>
> +    tunnel_out_port_attr = a[OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT];
> +    if (tunnel_out_port_attr) {
> +        ds_put_format(ds, ",tunnel_out_port=%"PRIu32,
> +                      nl_attr_get_u32(tunnel_out_port_attr));
> +    }
> +
>      ds_put_char(ds, ')');
>  }
>
> @@ -506,50 +516,36 @@ format_odp_actions(struct ds *ds, const struct nlattr *actions,
>      }
>  }
>
> +/* Separate out parse_odp_userspace_action() function. */
>  static int
> -parse_odp_action(const char *s, const struct simap *port_names,
> -                 struct ofpbuf *actions)
> +parse_odp_userspace_action(const char *s, struct ofpbuf *actions)
>  {
> -    {
> -        uint32_t port;
> -        int n;
> +    uint32_t pid;
> +    union user_action_cookie cookie;
> +    struct ofpbuf buf;
> +    uint32_t tunnel_out_port;
> +    int n = -1;
> +    void *user_data = NULL;
> +    size_t user_data_size = 0;
>
> -        if (ovs_scan(s, "%"SCNi32"%n", &port, &n)) {
> -            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, port);
> -            return n;
> -        }
> -    }
> -
> -    if (port_names) {
> -        int len = strcspn(s, delimiters);
> -        struct simap_node *node;
> -
> -        node = simap_find_len(port_names, s, len);
> -        if (node) {
> -            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, node->data);
> -            return len;
> -        }
> +    if (!ovs_scan(s, "userspace(pid=%"SCNi32"%n", &pid, &n)){
> +        return -EINVAL;
>      }
>
>      {
> -        uint32_t pid;
>          uint32_t output;
>          uint32_t probability;
>          uint32_t collector_set_id;
>          uint32_t obs_domain_id;
>          uint32_t obs_point_id;
>          int vid, pcp;
> -        int n = -1;
> -
> -        if (ovs_scan(s, "userspace(pid=%"SCNi32")%n", &pid, &n)) {
> -            odp_put_userspace_action(pid, NULL, 0, actions);
> -            return n;
> -        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",sFlow(vid=%i,"
> -                            "pcp=%i,output=%"SCNi32"))%n",
> -                            &pid, &vid, &pcp, &output, &n)) {
> -            union user_action_cookie cookie;
> +        int n1 = -1;
> +        if (ovs_scan(&s[n], ",sFlow(vid=%i,"
> +                     "pcp=%i,output=%"SCNi32")%n",
> +                     &vid, &pcp, &output, &n1)) {
>              uint16_t tci;
>
> +            n += n1;
>              tci = vid | (pcp << VLAN_PCP_SHIFT);
>              if (tci) {
>                  tci |= VLAN_CFI;
> @@ -558,14 +554,13 @@ parse_odp_action(const char *s, const struct simap *port_names,
>              cookie.type = USER_ACTION_COOKIE_SFLOW;
>              cookie.sflow.vlan_tci = htons(tci);
>              cookie.sflow.output = output;
> -            odp_put_userspace_action(pid, &cookie, sizeof cookie.sflow,
> -                                     actions);
> -            return n;
> -        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",slow_path%n",
> -                            &pid, &n)) {
> -            union user_action_cookie cookie;
> +            user_data = &cookie;
> +            user_data_size = sizeof cookie.sflow;
> +        } else if (ovs_scan(&s[n], ",slow_path%n",
> +                            &n1)) {
>              int res;
>
> +            n += n1;
>              cookie.type = USER_ACTION_COOKIE_SLOW_PATH;
>              cookie.slow_path.unused = 0;
>              cookie.slow_path.reason = 0;
> @@ -576,53 +571,91 @@ parse_odp_action(const char *s, const struct simap *port_names,
>                  return res;
>              }
>              n += res;
> -            if (s[n] != ')') {
> -                return -EINVAL;
> -            }
> -            n++;
>
> -            odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path,
> -                                     actions);
> -            return n;
> -        } else if (ovs_scan(s, "userspace(pid=%"SCNi32","
> -                            "flow_sample(probability=%"SCNi32","
> +            user_data = &cookie;
> +            user_data_size = sizeof cookie.slow_path;
> +        } else if (ovs_scan(&s[n], ",flow_sample(probability=%"SCNi32","
>                              "collector_set_id=%"SCNi32","
>                              "obs_domain_id=%"SCNi32","
> -                            "obs_point_id=%"SCNi32"))%n",
> -                            &pid, &probability, &collector_set_id,
> -                            &obs_domain_id, &obs_point_id, &n)) {
> -            union user_action_cookie cookie;
> +                            "obs_point_id=%"SCNi32")%n",
> +                            &probability, &collector_set_id,
> +                            &obs_domain_id, &obs_point_id, &n1)) {
> +            n += n1;
>
>              cookie.type = USER_ACTION_COOKIE_FLOW_SAMPLE;
>              cookie.flow_sample.probability = probability;
>              cookie.flow_sample.collector_set_id = collector_set_id;
>              cookie.flow_sample.obs_domain_id = obs_domain_id;
>              cookie.flow_sample.obs_point_id = obs_point_id;
> -            odp_put_userspace_action(pid, &cookie, sizeof cookie.flow_sample,
> -                                     actions);
> -            return n;
> -        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",ipfix)%n", &pid, &n)) {
> -            union user_action_cookie cookie;
> -
> +            user_data = &cookie;
> +            user_data_size = sizeof cookie.flow_sample;
> +        } else if (ovs_scan(&s[n], ",ipfix(output_port=%"SCNi32")%n",
> +                            &output, &n1) ) {
> +            n += n1;
>              cookie.type = USER_ACTION_COOKIE_IPFIX;
> -            odp_put_userspace_action(pid, &cookie, sizeof cookie.ipfix,
> -                                     actions);
> -            return n;
> -        } else if (ovs_scan(s, "userspace(pid=%"SCNi32",userdata(%n",
> -                            &pid, &n)) {
> -            struct ofpbuf buf;
> +            cookie.ipfix.output_odp_port = output;
> +            user_data = &cookie;
> +            user_data_size = sizeof cookie.ipfix;
> +        } else if (ovs_scan(&s[n], ",userdata(%n",
> +                            &n1)) {
>              char *end;
>
> +            n += n1;
>              ofpbuf_init(&buf, 16);
>              end = ofpbuf_put_hex(&buf, &s[n], NULL);
> -            if (end[0] == ')' && end[1] == ')') {
> -                odp_put_userspace_action(pid, ofpbuf_data(&buf), ofpbuf_size(&buf), actions);
> -                ofpbuf_uninit(&buf);
> -                return (end + 2) - s;
> +            if (end[0] != ')') {
> +                return -EINVAL;
>              }
> +            user_data = ofpbuf_data(&buf),
> +            user_data_size = ofpbuf_size(&buf);
> +            n = (end + 1) - s;
>          }
>      }
>
> +    {
> +        int n1 = -1;
> +        if (ovs_scan(&s[n], ",tunnel_out_port=%"SCNi32")%n",
> +                     &tunnel_out_port, &n1)) {
> +            odp_put_userspace_action(pid, user_data, user_data_size, &tunnel_out_port, actions);
> +            return n + n1;
> +        } else if (s[n] == ')') {
> +            odp_put_userspace_action(pid, user_data, user_data_size, NULL, actions);
> +            return n + 1;
> +        }
> +    }
> +
> +    return -EINVAL;
> +}
> +
> +static int
> +parse_odp_action(const char *s, const struct simap *port_names,
> +                 struct ofpbuf *actions)
> +{
> +    {
> +        uint32_t port;
> +        int n;
> +
> +        if (ovs_scan(s, "%"SCNi32"%n", &port, &n)) {
> +            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, port);
> +            return n;
> +        }
> +    }
> +
> +    if (port_names) {
> +        int len = strcspn(s, delimiters);
> +        struct simap_node *node;
> +
> +        node = simap_find_len(port_names, s, len);
> +        if (node) {
> +            nl_msg_put_u32(actions, OVS_ACTION_ATTR_OUTPUT, node->data);
> +            return len;
> +        }
> +    }
> +
> +    if (!strncmp(s, "userspace(", 10)) {
> +        return parse_odp_userspace_action(s, actions);
> +    }
> +
>      if (!strncmp(s, "set(", 4)) {
>          size_t start_ofs;
>          int retval;
> @@ -832,6 +865,8 @@ tunnel_key_attr_len(int type)
>      case OVS_TUNNEL_KEY_ATTR_TTL: return 1;
>      case OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT: return 0;
>      case OVS_TUNNEL_KEY_ATTR_CSUM: return 0;
> +    case OVS_TUNNEL_KEY_ATTR_TP_SRC: return 2;
> +    case OVS_TUNNEL_KEY_ATTR_TP_DST: return 2;
>      case OVS_TUNNEL_KEY_ATTR_OAM: return 0;
>      case OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS: return -2;
>      case __OVS_TUNNEL_KEY_ATTR_MAX:
> @@ -914,6 +949,12 @@ odp_tun_key_from_attr(const struct nlattr *attr, struct flow_tnl *tun)
>          case OVS_TUNNEL_KEY_ATTR_CSUM:
>              tun->flags |= FLOW_TNL_F_CSUM;
>              break;
> +        case OVS_TUNNEL_KEY_ATTR_TP_SRC:
> +            tun->tp_src = nl_attr_get_be16(a);
> +            break;
> +        case OVS_TUNNEL_KEY_ATTR_TP_DST:
> +            tun->tp_dst = nl_attr_get_be16(a);
> +            break;
>          case OVS_TUNNEL_KEY_ATTR_OAM:
>              tun->flags |= FLOW_TNL_F_OAM;
>              break;
> @@ -970,6 +1011,12 @@ tun_key_to_attr(struct ofpbuf *a, const struct flow_tnl *tun_key)
>      if (tun_key->flags & FLOW_TNL_F_CSUM) {
>          nl_msg_put_flag(a, OVS_TUNNEL_KEY_ATTR_CSUM);
>      }
> +    if (tun_key->tp_src) {
> +        nl_msg_put_be16(a, OVS_TUNNEL_KEY_ATTR_TP_SRC, tun_key->tp_src);
> +    }
> +    if (tun_key->tp_dst) {
> +        nl_msg_put_be16(a, OVS_TUNNEL_KEY_ATTR_TP_DST, tun_key->tp_dst);
> +    }
>      if (tun_key->flags & FLOW_TNL_F_OAM) {
>          nl_msg_put_flag(a, OVS_TUNNEL_KEY_ATTR_OAM);
>      }
> @@ -3503,6 +3550,7 @@ odp_key_fitness_to_string(enum odp_key_fitness fitness)
>  size_t
>  odp_put_userspace_action(uint32_t pid,
>                           const void *userdata, size_t userdata_size,
> +                         const uint32_t *tunnel_out_pid,
>                           struct ofpbuf *odp_actions)
>  {
>      size_t userdata_ofs;
> @@ -3529,6 +3577,9 @@ odp_put_userspace_action(uint32_t pid,
>      } else {
>          userdata_ofs = 0;
>      }
> +    if (tunnel_out_pid) {
> +        nl_msg_put_u32(odp_actions, OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT, *tunnel_out_pid);
> +    }
>      nl_msg_end_nested(odp_actions, offset);
>
>      return userdata_ofs;
> diff --git a/lib/odp-util.h b/lib/odp-util.h
> index 4ce693c..11d76fd 100644
> --- a/lib/odp-util.h
> +++ b/lib/odp-util.h
> @@ -230,12 +230,14 @@ union user_action_cookie {
>
>      struct {
>          uint16_t type;          /* USER_ACTION_COOKIE_IPFIX. */
> +        uint32_t output_odp_port; /* The output odp port on datapath to get tunnel info. */
>      } ipfix;
>  };
>  BUILD_ASSERT_DECL(sizeof(union user_action_cookie) == 16);
>
>  size_t odp_put_userspace_action(uint32_t pid,
>                                  const void *userdata, size_t userdata_size,
> +                                const uint32_t *tunnel_out_pid,
>                                  struct ofpbuf *odp_actions);
>  void odp_put_tunnel_action(const struct flow_tnl *tunnel,
>                             struct ofpbuf *odp_actions);
> diff --git a/lib/packets.h b/lib/packets.h
> index c04e3bb..e54b287 100644
> --- a/lib/packets.h
> +++ b/lib/packets.h
> @@ -33,12 +33,17 @@ struct ds;
>
>  /* Tunnel information used in flow key and metadata. */
>  struct flow_tnl {
> -    ovs_be64 tun_id;
> +    union {
> +        ovs_be64 tun_id;
> +        uint8_t tun_id_octarray[8];
> +    };
>      ovs_be32 ip_src;
>      ovs_be32 ip_dst;
>      uint16_t flags;
>      uint8_t ip_tos;
>      uint8_t ip_ttl;
> +    ovs_be16 tp_src;
> +    ovs_be16 tp_dst;
>  };
>
>  /* Unfortunately, a "struct flow" sometimes has to handle OpenFlow port
> @@ -63,7 +68,7 @@ struct pkt_metadata {
>  };
>
>  #define PKT_METADATA_INITIALIZER(PORT) \
> -    (struct pkt_metadata){ 0, 0, { 0, 0, 0, 0, 0, 0}, 0, 0, {(PORT)} }
> +    (struct pkt_metadata){ 0, 0, { {0}, 0, 0, 0, 0, 0, 0, 0}, 0, 0, {(PORT)} }
>
>  bool dpid_from_string(const char *s, uint64_t *dpidp);
>
> diff --git a/ofproto/automake.mk b/ofproto/automake.mk
> index 22c50d1..0c3521d 100644
> --- a/ofproto/automake.mk
> +++ b/ofproto/automake.mk
> @@ -73,3 +73,7 @@ dist_noinst_SCRIPTS = ofproto/ipfix-gen-entities
>  ofproto/ipfix-entities.def: ofproto/ipfix.xml ofproto/ipfix-gen-entities
>         $(run_python) $(srcdir)/ofproto/ipfix-gen-entities $< > $@.tmp
>         mv $@.tmp $@
> +
> +# IPFIX enterprise entity definition macros.
> +EXTRA_DIST += ofproto/ipfix-enterprise-entities.def
> +
> diff --git a/ofproto/ipfix-enterprise-entities.def b/ofproto/ipfix-enterprise-entities.def
> new file mode 100644
> index 0000000..e14521f
> --- /dev/null
> +++ b/ofproto/ipfix-enterprise-entities.def
> @@ -0,0 +1,19 @@
> +/* IPFIX enterprise entities. */
> +#ifndef IPFIX_ENTERPRISE_ENTITY
> +#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)
> +#endif
> +
> +#ifndef IPFIX_ENTERPRISE_VMWARE
> +#define IPFIX_ENTERPRISE_VMWARE 6876
> +#endif
> +
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_TYPE, 891, 1, tunnelType, IPFIX_ENTERPRISE_VMWARE)
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_KEY, 892, 0, tunnelKey, IPFIX_ENTERPRISE_VMWARE)
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_SOURCE_IPV4_ADDRESS, 893, 4, tunnelSourceIPv4Address, IPFIX_ENTERPRISE_VMWARE)
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_DESTINATION_IPV4_ADDRESS, 894, 4, tunnelDestinationIPv4Address, IPFIX_ENTERPRISE_VMWARE)
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_PROTOCOL_IDENTIFIER, 895, 1, tunnelProtocolIdentifier, IPFIX_ENTERPRISE_VMWARE)
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_SOURCE_TRANSPORT_PORT, 896, 2, tunnelSourceTransportPort, IPFIX_ENTERPRISE_VMWARE)
> +IPFIX_ENTERPRISE_ENTITY(TUNNEL_DESTINATION_TRANSPORT_PORT, 897, 2, tunnelDestinationTransportPort, IPFIX_ENTERPRISE_VMWARE)
> +
> +#undef IPFIX_ENTERPRISE_VMWARE
> +#undef IPFIX_ENTERPRISE_ENTITY
> diff --git a/ofproto/ofproto-dpif-ipfix.c b/ofproto/ofproto-dpif-ipfix.c
> index 08a3fca..a1d9d1a 100644
> --- a/ofproto/ofproto-dpif-ipfix.c
> +++ b/ofproto/ofproto-dpif-ipfix.c
> @@ -25,6 +25,7 @@
>  #include "list.h"
>  #include "ofpbuf.h"
>  #include "ofproto.h"
> +#include "ofproto-dpif.h"
>  #include "packets.h"
>  #include "poll-loop.h"
>  #include "sset.h"
> @@ -41,6 +42,41 @@ static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
>  /* Cf. IETF RFC 5101 Section 10.3.4. */
>  #define IPFIX_DEFAULT_COLLECTOR_PORT 4739
>
> +/* The standard layer2SegmentId (ID 351) element is included in vDS to send
> + * the VxLAN tunnel's VNI. It is 64-bit long, the most significant byte is
> + * used to indicate the type of tunnel (0x01 = VxLAN, 0x02 = GRE) and the three
> + * lowest significant bytes hold the value of the layer 2 overlay network
> + * segment identifier: a 24-bit VxLAN tunnel's VNI or a 24-bit GRE tunnel's
> + * TNI. This is not compatible with STT or GRE-64, as implementd in OVS,
> + * as their tunnel IDs are 64-bit.
> + *
> + * Two new enterprise information elements are defined which are simlar to
> + * laryerSegmentId but support 64-bit IDs:
> + *     tunnelType (ID 891) and tunnelKey (ID 892).
> + *
> + * The enum dpif_ipfix_tunnel_type is to declare the types supported in the
> + * tunnelType element.
> + */
> +enum dpif_ipfix_tunnel_type {
> +    DPIF_IPFIX_TUNNEL_UNKNOWN = 0x00,
> +    DPIF_IPFIX_TUNNEL_VXLAN = 0x01,
> +    DPIF_IPFIX_TUNNEL_GRE = 0x02,
> +    DPIF_IPFIX_TUNNEL_LISP = 0x03,
> +    DPIF_IPFIX_TUNNEL_STTv0 = 0x04,
> +    DPIF_IPFIX_TUNNEL_IPSEC_GRE = 0x05,
> +    DPIF_IPFIX_TUNNEL_IPSEC_STTv0 = 0x06,
> +    DPIF_IPFIX_TUNNEL_GENEVE = 0x07,
> +    NUM_DPIF_IPFIX_TUNNEL
> +};
> +
> +struct dpif_ipfix_port {
> +    struct hmap_node hmap_node; /* In struct dpif_ipfix's "tunnel_ports" hmap. */
> +    struct ofport *ofport;      /* To retrieve port stats. */
> +    odp_port_t odp_port;
> +    enum dpif_ipfix_tunnel_type tunnel_type;
> +    uint8_t tunnel_key_length;
> +};
> +
>  struct dpif_ipfix_exporter {
>      struct collectors *collectors;
>      uint32_t seq_number;
> @@ -70,6 +106,9 @@ struct dpif_ipfix_flow_exporter_map_node {
>  struct dpif_ipfix {
>      struct dpif_ipfix_bridge_exporter bridge_exporter;
>      struct hmap flow_exporter_map;  /* dpif_ipfix_flow_exporter_map_node. */
> +    struct hmap tunnel_ports;       /* Contains "struct dpif_ipfix_port"s.
> +                                     * It makes tunnel port lookups faster in
> +                                     * sampling upcalls. */
>      struct ovs_refcount ref_cnt;
>  };
>
> @@ -121,6 +160,11 @@ enum ipfix_proto_l4 {
>      IPFIX_PROTO_L4_ICMP,
>      NUM_IPFIX_PROTO_L4
>  };
> +enum ipfix_proto_tunnel {
> +    IPFIX_PROTO_TUNNEL_UNKNOWN = 0,
> +    IPFIX_PROTO_TUNNEL_KNOWN,  /* Support gre, lisp and vxlan. */
> +    NUM_IPFIX_PROTO_TUNNEL
> +};
>
>  /* Any Template ID > 255 is usable for Template Records. */
>  #define IPFIX_TEMPLATE_ID_MIN 256
> @@ -134,33 +178,60 @@ struct ipfix_template_record_header {
>  BUILD_ASSERT_DECL(sizeof(struct ipfix_template_record_header) == 4);
>
>  enum ipfix_entity_id {
> +/* standard IPFIX elements */
>  #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ID_##ENUM = ID,
>  #include "ofproto/ipfix-entities.def"
> +/* non-standard IPFIX elements */
> +#define IPFIX_SET_ENTERPRISE(v) (((v) | 0x8000))
> +#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)  IPFIX_ENTITY_ID_##ENUM = IPFIX_SET_ENTERPRISE(ID),
> +#include "ofproto/ipfix-enterprise-entities.def"
>  };
>
>  enum ipfix_entity_size {
> +/* standard IPFIX elements */
>  #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
>  #include "ofproto/ipfix-entities.def"
> +/* non-standard IPFIX elements */
> +#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
> +#include "ofproto/ipfix-enterprise-entities.def"
> +};
> +
> +enum ipfix_entity_enterprise {
> +/* standard IPFIX elements */
> +#define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ENTERPRISE_##ENUM = 0,
> +#include "ofproto/ipfix-entities.def"
> +/* non-standard IPFIX elements */
> +#define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE)  IPFIX_ENTITY_ENTERPRISE_##ENUM = ENTERPRISE,
> +#include "ofproto/ipfix-enterprise-entities.def"
>  };
>
>  OVS_PACKED(
>  struct ipfix_template_field_specifier {
>      ovs_be16 element_id;  /* IPFIX_ENTITY_ID_*. */
> -    ovs_be16 field_length;  /* Length of the field's value, in bytes. */
> -    /* No Enterprise ID, since only standard element IDs are specified. */
> +    ovs_be16 field_length;  /* Length of the field's value, in bytes.
> +                             * For Variable-Length element, it should be 65535.
> +                             */
> +    ovs_be32 enterprise;  /* Enterprise number */
>  });
> -BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 4);
> +BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 8);
> +
> +/* Cf. IETF RFC 5102 Section 5.11.6. */
> +enum ipfix_flow_direction {
> +    INGRESS_FLOW = 0x00,
> +    EGRESS_FLOW = 0x01
> +};
>
>  /* Part of data record flow key for common metadata and Ethernet entities. */
>  OVS_PACKED(
>  struct ipfix_data_record_flow_key_common {
>      ovs_be32 observation_point_id;  /* OBSERVATION_POINT_ID */
> +    uint8_t flow_direction;  /* FLOW_DIRECTION */
>      uint8_t source_mac_address[6];  /* SOURCE_MAC_ADDRESS */
>      uint8_t destination_mac_address[6];  /* DESTINATION_MAC_ADDRESS */
>      ovs_be16 ethernet_type;  /* ETHERNET_TYPE */
>      uint8_t ethernet_header_length;  /* ETHERNET_HEADER_LENGTH */
>  });
> -BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 19);
> +BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 20);
>
>  /* Part of data record flow key for VLAN entities. */
>  OVS_PACKED(
> @@ -217,6 +288,33 @@ struct ipfix_data_record_flow_key_icmp {
>  });
>  BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_icmp) == 2);
>
> +/* For the tunnel type that is on the top of IPSec, the protocol identifier
> + * of the upper tunnel type is used.
> + */
> +static uint8_t tunnel_protocol[NUM_DPIF_IPFIX_TUNNEL] = {
> +   0,              /* reserved */
> +   IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_VXLAN */
> +   IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_GRE */
> +   IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_LISP*/
> +   IPPROTO_TCP,    /* DPIF_IPFIX_TUNNEL_STT*/
> +   IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_IPSEC_GRE */
> +   IPPROTO_TCP,    /* DPIF_IPFIX_TUNNEL_IPSEC_STTv0*/
> +   IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_GENEVE*/
> +};
> +
> +OVS_PACKED(
> +struct ipfix_data_record_flow_key_tunnel {
> +    ovs_be32 tunnel_source_ipv4_address;  /* TUNNEL_SOURCE_IPV4_ADDRESS */
> +    ovs_be32 tunnel_destination_ipv4_address;  /* TUNNEL_DESTINATION_IPV4_ADDRESS */
> +    uint8_t tunnel_protocol_identifier;  /* TUNNEL_PROTOCOL_IDENTIFIER */
> +    ovs_be16 tunnel_source_transport_port;  /* TUNNEL_SOURCE_TRANSPORT_PORT */
> +    ovs_be16 tunnel_destination_transport_port;  /* TUNNEL_DESTINATION_TRANSPORT_PORT */
> +    uint8_t tunnel_type;  /* TUNNEL_TYPE */
> +    uint8_t tunnel_key_length;  /* length of TUNNEL_KEY */
> +    uint8_t tunnel_key[];  /* data of  TUNNEL_KEY */
> +});
> +BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_tunnel) == 15);
> +
>  /* Cf. IETF RFC 5102 Section 5.11.3. */
>  enum ipfix_flow_end_reason {
>      IDLE_TIMEOUT = 0x01,
> @@ -247,6 +345,14 @@ struct ipfix_data_record_aggregated_ip {
>  });
>  BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
>
> +/* support tunnel key for:
> + * VxLAN: 24-bit VIN,
> + * GRE: 32- or 64-bit key,
> + * LISP: 24-bit instance ID
> + * STT: 64-bit key
> + */
> +#define MAX_TUNNEL_KEY_LEN 8
> +
>  #define MAX_FLOW_KEY_LEN                                        \
>      (sizeof(struct ipfix_data_record_flow_key_common)           \
>       + sizeof(struct ipfix_data_record_flow_key_vlan)           \
> @@ -254,7 +360,9 @@ BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
>       + MAX(sizeof(struct ipfix_data_record_flow_key_ipv4),      \
>             sizeof(struct ipfix_data_record_flow_key_ipv6))      \
>       + MAX(sizeof(struct ipfix_data_record_flow_key_icmp),      \
> -           sizeof(struct ipfix_data_record_flow_key_transport)))
> +           sizeof(struct ipfix_data_record_flow_key_transport)) \
> +     + sizeof(struct ipfix_data_record_flow_key_tunnel)         \
> +     + MAX_TUNNEL_KEY_LEN)
>
>  #define MAX_DATA_RECORD_LEN                                 \
>      (MAX_FLOW_KEY_LEN                                       \
> @@ -315,6 +423,9 @@ ofproto_ipfix_bridge_exporter_options_equal(
>              && a->sampling_rate == b->sampling_rate
>              && a->cache_active_timeout == b->cache_active_timeout
>              && a->cache_max_flows == b->cache_max_flows
> +            && a->enable_tunnel_sampling == b->enable_tunnel_sampling
> +            && a->enable_input_sampling == b->enable_input_sampling
> +            && a->enable_output_sampling == b->enable_output_sampling
>              && sset_equals(&a->targets, &b->targets));
>  }
>
> @@ -422,6 +533,114 @@ dpif_ipfix_exporter_set_options(struct dpif_ipfix_exporter *exporter,
>      return true;
>  }
>
> +static struct dpif_ipfix_port *
> +dpif_ipfix_find_port(const struct dpif_ipfix *di,
> +                     odp_port_t odp_port) OVS_REQUIRES(mutex)
> +{
> +    struct dpif_ipfix_port *dip;
> +
> +    HMAP_FOR_EACH_IN_BUCKET (dip, hmap_node, hash_odp_port(odp_port),
> +                             &di->tunnel_ports) {
> +        if (dip->odp_port == odp_port) {
> +            return dip;
> +        }
> +    }
> +    return NULL;
> +}
> +
> +static void
> +dpif_ipfix_del_port(struct dpif_ipfix *di,
> +                      struct dpif_ipfix_port *dip)
> +    OVS_REQUIRES(mutex)
> +{
> +    hmap_remove(&di->tunnel_ports, &dip->hmap_node);
> +    free(dip);
> +}
> +
> +void
> +dpif_ipfix_add_tunnel_port(struct dpif_ipfix *di, struct ofport *ofport,
> +                           odp_port_t odp_port) OVS_EXCLUDED(mutex)
> +{
> +    struct dpif_ipfix_port *dip;
> +    const char *type;
> +
> +    ovs_mutex_lock(&mutex);
> +    dip = dpif_ipfix_find_port(di, odp_port);
> +    if (dip) {
> +        dpif_ipfix_del_port(di, dip);
> +    }
> +
> +    type = netdev_get_type(ofport->netdev);
> +    if (type == NULL) {
> +        goto out;
> +    }
> +
> +    /* Add to table of tunnel ports. */
> +    dip = xmalloc(sizeof *dip);
> +    if (dip == NULL) {
> +        goto out;
> +    }
> +    dip->ofport = ofport;
> +    dip->odp_port = odp_port;
> +    if (strncmp(type, "gre", sizeof "gre") == 0) {
> +        /* 32-bit key gre */
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
> +        dip->tunnel_key_length = 4;
> +    } else if (strncmp(type, "gre64", sizeof "gre64") == 0) {
> +        /* 64-bit key gre */
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
> +        dip->tunnel_key_length = 8;
> +    } else if (strncmp(type, "ipsec_gre", sizeof "ipsec_gre") == 0) {
> +        /* 32-bit key ipsec_gre */
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
> +        dip->tunnel_key_length = 4;
> +    } else if (strncmp(type, "ipsec_gre64", sizeof "ipsec_gre64") == 0) {
> +        /* 64-bit key ipsec_gre */
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
> +        dip->tunnel_key_length = 8;
> +    } else if (strncmp(type, "vxlan", sizeof "vxlan") == 0) {
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_VXLAN;
> +        dip->tunnel_key_length = 3;
> +    } else if (strncmp(type, "lisp", sizeof "lisp") == 0) {
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_LISP;
> +        dip->tunnel_key_length = 3;
> +    } else if (strncmp(type, "geneve", sizeof "geneve") == 0) {
> +        dip->tunnel_type = DPIF_IPFIX_TUNNEL_GENEVE;
> +        dip->tunnel_key_length = 3;
> +    } else {
> +        free(dip);
> +        goto out;
> +    }
> +    hmap_insert(&di->tunnel_ports, &dip->hmap_node, hash_odp_port(odp_port));
> +
> +out:
> +    ovs_mutex_unlock(&mutex);
> +}
> +
> +void
> +dpif_ipfix_del_tunnel_port(struct dpif_ipfix *di, odp_port_t odp_port)
> +    OVS_EXCLUDED(mutex)
> +{
> +    struct dpif_ipfix_port *dip;
> +    ovs_mutex_lock(&mutex);
> +    dip = dpif_ipfix_find_port(di, odp_port);
> +    if (dip) {
> +        dpif_ipfix_del_port(di, dip);
> +    }
> +    ovs_mutex_unlock(&mutex);
> +}
> +
> +bool
> +dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *di, odp_port_t odp_port)
> +    OVS_EXCLUDED(mutex)
> +{
> +    struct dpif_ipfix_port *dip;
> +    ovs_mutex_lock(&mutex);
> +    dip = dpif_ipfix_find_port(di, odp_port);
> +    ovs_mutex_unlock(&mutex);
> +    return (dip != NULL);
> +}
> +
>  static void
>  dpif_ipfix_bridge_exporter_init(struct dpif_ipfix_bridge_exporter *exporter)
>  {
> @@ -652,6 +871,7 @@ dpif_ipfix_create(void)
>      di = xzalloc(sizeof *di);
>      dpif_ipfix_bridge_exporter_init(&di->bridge_exporter);
>      hmap_init(&di->flow_exporter_map);
> +    hmap_init(&di->tunnel_ports);
>      ovs_refcount_init(&di->ref_cnt);
>      return di;
>  }
> @@ -677,10 +897,50 @@ dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *di)
>      return ret;
>  }
>
> +bool
> +dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *di)
> +    OVS_EXCLUDED(mutex)
> +{
> +    bool ret = true;
> +    ovs_mutex_lock(&mutex);
> +    if (di->bridge_exporter.options) {
> +        ret = di->bridge_exporter.options->enable_input_sampling;
> +    }
> +    ovs_mutex_unlock(&mutex);
> +    return ret;
> +}
> +
> +bool
> +dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *di)
> +    OVS_EXCLUDED(mutex)
> +{
> +    bool ret = true;
> +    ovs_mutex_lock(&mutex);
> +    if (di->bridge_exporter.options) {
> +        ret = di->bridge_exporter.options->enable_output_sampling;
> +    }
> +    ovs_mutex_unlock(&mutex);
> +    return ret;
> +}
> +
> +bool
> +dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *di)
> +    OVS_EXCLUDED(mutex)
> +{
> +    bool ret = false;
> +    ovs_mutex_lock(&mutex);
> +    if (di->bridge_exporter.options) {
> +        ret = di->bridge_exporter.options->enable_tunnel_sampling;
> +    }
> +    ovs_mutex_unlock(&mutex);
> +    return ret;
> +}
> +
>  static void
>  dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
>  {
>      struct dpif_ipfix_flow_exporter_map_node *exp_node, *exp_next;
> +    struct dpif_ipfix_port *dip, *next;
>
>      dpif_ipfix_bridge_exporter_clear(&di->bridge_exporter);
>
> @@ -689,6 +949,10 @@ dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
>          dpif_ipfix_flow_exporter_destroy(&exp_node->exporter);
>          free(exp_node);
>      }
> +
> +    HMAP_FOR_EACH_SAFE (dip, next, hmap_node, &di->tunnel_ports) {
> +        dpif_ipfix_del_port(di, dip);
> +    }
>  }
>
>  void
> @@ -699,6 +963,7 @@ dpif_ipfix_unref(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
>          dpif_ipfix_clear(di);
>          dpif_ipfix_bridge_exporter_destroy(&di->bridge_exporter);
>          hmap_destroy(&di->flow_exporter_map);
> +        hmap_destroy(&di->tunnel_ports);
>          free(di);
>          ovs_mutex_unlock(&mutex);
>      }
> @@ -733,42 +998,64 @@ ipfix_send_msg(const struct collectors *collectors, struct ofpbuf *msg)
>
>  static uint16_t
>  ipfix_get_template_id(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
> -                      enum ipfix_proto_l4 l4)
> +                      enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel)
>  {
>      uint16_t template_id;
>      template_id = l2;
>      template_id = template_id * NUM_IPFIX_PROTO_L3 + l3;
>      template_id = template_id * NUM_IPFIX_PROTO_L4 + l4;
> +    template_id = template_id * NUM_IPFIX_PROTO_TUNNEL + tunnel;
>      return IPFIX_TEMPLATE_ID_MIN + template_id;
>  }
>
>  static void
>  ipfix_define_template_entity(enum ipfix_entity_id id,
> -                             enum ipfix_entity_size size, struct ofpbuf *msg)
> +                             enum ipfix_entity_size size,
> +                             enum ipfix_entity_enterprise enterprise,
> +                             struct ofpbuf *msg)
>  {
>      struct ipfix_template_field_specifier *field;
> +    size_t field_size;
>
> -    field = ofpbuf_put_zeros(msg, sizeof *field);
> +    if (enterprise) {
> +        field_size = sizeof *field;
> +    } else {
> +        /* No enterprise number */
> +        field_size = sizeof *field - sizeof(ovs_be32);
> +    }
> +    field = ofpbuf_put_zeros(msg, field_size);
>      field->element_id = htons(id);
> -    field->field_length = htons(size);
> +    if (size) {
> +        field->field_length = htons(size);
> +    } else {
> +        /* RFC 5101, Section 7. Variable-Length Information Element */
> +        field->field_length = 0xffff;
> +    }
> +    if (enterprise) {
> +        field->enterprise = htonl(enterprise);
> +    }
> +
>  }
>
>  static uint16_t
>  ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
> -                             enum ipfix_proto_l4 l4, struct ofpbuf *msg)
> +                             enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel,
> +                             struct ofpbuf *msg)
>  {
>      uint16_t count = 0;
>
>  #define DEF(ID) \
>      { \
>          ipfix_define_template_entity(IPFIX_ENTITY_ID_##ID, \
> -                                     IPFIX_ENTITY_SIZE_##ID, msg); \
> +                                     IPFIX_ENTITY_SIZE_##ID, \
> +                                     IPFIX_ENTITY_ENTERPRISE_##ID, msg); \
>          count++; \
>      }
>
>      /* 1. Flow key. */
>
>      DEF(OBSERVATION_POINT_ID);
> +    DEF(FLOW_DIRECTION);
>
>      /* Common Ethernet entities. */
>      DEF(SOURCE_MAC_ADDRESS);
> @@ -814,6 +1101,16 @@ ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
>          }
>      }
>
> +    if (tunnel != IPFIX_PROTO_TUNNEL_UNKNOWN){
> +        DEF(TUNNEL_SOURCE_IPV4_ADDRESS);
> +        DEF(TUNNEL_DESTINATION_IPV4_ADDRESS);
> +        DEF(TUNNEL_PROTOCOL_IDENTIFIER);
> +        DEF(TUNNEL_SOURCE_TRANSPORT_PORT);
> +        DEF(TUNNEL_DESTINATION_TRANSPORT_PORT);
> +        DEF(TUNNEL_TYPE);
> +        DEF(TUNNEL_KEY);
> +    }
> +
>      /* 2. Flow aggregated data. */
>
>      DEF(FLOW_START_DELTA_MICROSECONDS);
> @@ -828,6 +1125,7 @@ ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
>          DEF(MINIMUM_IP_TOTAL_LENGTH);
>          DEF(MAXIMUM_IP_TOTAL_LENGTH);
>      }
> +
>
>  #undef DEF
>
> @@ -835,8 +1133,8 @@ ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
>  }
>
>  static void
> -ipfix_send_template_msg(struct dpif_ipfix_exporter *exporter,
> -                        uint32_t export_time_sec, uint32_t obs_domain_id)
> +ipfix_send_template_msgs(struct dpif_ipfix_exporter *exporter,
> +                         uint32_t export_time_sec, uint32_t obs_domain_id)
>  {
>      uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
>      struct ofpbuf msg;
> @@ -847,6 +1145,7 @@ ipfix_send_template_msg(struct dpif_ipfix_exporter *exporter,
>      enum ipfix_proto_l2 l2;
>      enum ipfix_proto_l3 l3;
>      enum ipfix_proto_l4 l4;
> +    enum ipfix_proto_tunnel tunnel;
>
>      ofpbuf_use_stub(&msg, msg_stub, sizeof msg_stub);
>
> @@ -867,14 +1166,43 @@ ipfix_send_template_msg(struct dpif_ipfix_exporter *exporter,
>                      l4 != IPFIX_PROTO_L4_UNKNOWN) {
>                      continue;
>                  }
> -                tmpl_hdr_offset = ofpbuf_size(&msg);
> -                tmpl_hdr = ofpbuf_put_zeros(&msg, sizeof *tmpl_hdr);
> -                tmpl_hdr->template_id = htons(
> -                    ipfix_get_template_id(l2, l3, l4));
> -                field_count = ipfix_define_template_fields(l2, l3, l4, &msg);
> -                tmpl_hdr = (struct ipfix_template_record_header*)
> -                    ((uint8_t*)ofpbuf_data(&msg) + tmpl_hdr_offset);
> -                tmpl_hdr->field_count = htons(field_count);
> +                for (tunnel = 0; tunnel < NUM_IPFIX_PROTO_TUNNEL; tunnel++) {
> +                    /* When the size of the tmplate packet reaches
> +                     * MAX_MESSAGE_LEN(1024), send it out.
> +                     * And then reinit the msg to construct a new packet for
> +                     * the following templates.
> +                     */
> +                    if (ofpbuf_size(&msg) >= MAX_MESSAGE_LEN) {
> +                        /* Send template message. */
> +                        set_hdr = (struct ipfix_set_header*)
> +                                  ((uint8_t*)ofpbuf_data(&msg) + set_hdr_offset);
> +                        set_hdr->length = htons(ofpbuf_size(&msg) - set_hdr_offset);
> +
> +                        ipfix_send_msg(exporter->collectors, &msg);
> +                        ofpbuf_uninit(&msg);
> +
> +                        /* Reinit the msg buf. */
> +                        ofpbuf_use_stub(&msg, msg_stub, sizeof msg_stub);
> +
> +                        ipfix_init_header(export_time_sec, exporter->seq_number,
> +                                          obs_domain_id, &msg);
> +                        set_hdr_offset = ofpbuf_size(&msg);
> +
> +                        /* Add a Template Set. */
> +                        set_hdr = ofpbuf_put_zeros(&msg, sizeof *set_hdr);
> +                        set_hdr->set_id = htons(IPFIX_SET_ID_TEMPLATE);
> +                    }
> +
> +                    tmpl_hdr_offset = ofpbuf_size(&msg);
> +                    tmpl_hdr = ofpbuf_put_zeros(&msg, sizeof *tmpl_hdr);
> +                    tmpl_hdr->template_id = htons(
> +                        ipfix_get_template_id(l2, l3, l4, tunnel));
> +                    field_count =
> +                        ipfix_define_template_fields(l2, l3, l4, tunnel, &msg);
> +                    tmpl_hdr = (struct ipfix_template_record_header*)
> +                        ((uint8_t*)ofpbuf_data(&msg) + tmpl_hdr_offset);
> +                    tmpl_hdr->field_count = htons(field_count);
> +                }
>              }
>          }
>      }
> @@ -1021,13 +1349,16 @@ static void
>  ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
>                         struct ofpbuf *packet, const struct flow *flow,
>                         uint64_t packet_delta_count, uint32_t obs_domain_id,
> -                       uint32_t obs_point_id)
> +                       uint32_t obs_point_id, odp_port_t output_odp_port,
> +                       const struct dpif_ipfix_port *tunnel_port,
> +                       const struct flow_tnl *tunnel_key)
>  {
>      struct ipfix_flow_key *flow_key;
>      struct ofpbuf msg;
>      enum ipfix_proto_l2 l2;
>      enum ipfix_proto_l3 l3;
>      enum ipfix_proto_l4 l4;
> +    enum ipfix_proto_tunnel tunnel = IPFIX_PROTO_TUNNEL_UNKNOWN;
>      uint8_t ethernet_header_length;
>      uint16_t ethernet_total_length;
>
> @@ -1074,9 +1405,13 @@ ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
>          l3 = IPFIX_PROTO_L3_UNKNOWN;
>          l4 = IPFIX_PROTO_L4_UNKNOWN;
>      }
> +
> +    if (tunnel_port && tunnel_key) {
> +       tunnel = IPFIX_PROTO_TUNNEL_KNOWN;
> +    }
>
>      flow_key->obs_domain_id = obs_domain_id;
> -    flow_key->template_id = ipfix_get_template_id(l2, l3, l4);
> +    flow_key->template_id = ipfix_get_template_id(l2, l3, l4, tunnel);
>
>      /* The fields defined in the ipfix_data_record_* structs and sent
>       * below must match exactly the templates defined in
> @@ -1092,6 +1427,8 @@ ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
>
>          data_common = ofpbuf_put_zeros(&msg, sizeof *data_common);
>          data_common->observation_point_id = htonl(obs_point_id);
> +        data_common->flow_direction =
> +            (output_odp_port == ODPP_NONE) ? INGRESS_FLOW : EGRESS_FLOW;
>          memcpy(data_common->source_mac_address, flow->dl_src,
>                 sizeof flow->dl_src);
>          memcpy(data_common->destination_mac_address, flow->dl_dst,
> @@ -1154,6 +1491,36 @@ ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
>          data_icmp->icmp_code = ntohs(flow->tp_dst) & 0xff;
>      }
>
> +    if (tunnel == IPFIX_PROTO_TUNNEL_KNOWN) {
> +        struct ipfix_data_record_flow_key_tunnel * data_tunnel;
> +
> +        data_tunnel = ofpbuf_put_zeros(&msg, sizeof(*data_tunnel) +
> +                                             tunnel_port->tunnel_key_length);
> +        data_tunnel->tunnel_source_ipv4_address = tunnel_key->ip_src;
> +        data_tunnel->tunnel_destination_ipv4_address = tunnel_key->ip_dst;
> +        /* The tunnel_protocol_identifier is from tunnel_proto array, which
> +         * contians protocol_identifiers of each tunnel type.
> +         * For the tunnel type on the top of IPSec, which uses the protocol
> +         * identifier of the upper tunnel type is used, the tcp_src and tcp_dst
> +         * are decided based on the protocol identifires.
> +         * E.g:
> +         * The protocol identifier of DPIF_IPFIX_TUNNEL_IPSEC_GRE is IPPROTO_GRE,
> +         * and both tp_src and tp_dst are zero. The protocol identifier of the
> +         * DPIF_IPFIX_TUNNEL_IPSEC_STTv0 is TCP, which is the protocol identifier
> +         * of STT, and the tp_src and tp_dat are the exact TCP ports.
> +         */
> +        data_tunnel->tunnel_protocol_identifier =
> +            tunnel_protocol[tunnel_port->tunnel_type];
> +        data_tunnel->tunnel_source_transport_port = tunnel_key->tp_src;
> +        data_tunnel->tunnel_destination_transport_port = tunnel_key->tp_dst;
> +        data_tunnel->tunnel_type = tunnel_port->tunnel_type;
> +        data_tunnel->tunnel_key_length = tunnel_port->tunnel_key_length;
> +        /* tun_id is in network order, and tunnel key is in low bits*/
> +        memcpy(data_tunnel->tunnel_key,
> +               &tunnel_key->tun_id_octarray[8 - tunnel_port->tunnel_key_length],
> +               tunnel_port->tunnel_key_length);
> +    }
> +
>      flow_key->flow_key_msg_part_size = ofpbuf_size(&msg);
>
>      {
> @@ -1286,31 +1653,51 @@ static void
>  dpif_ipfix_sample(struct dpif_ipfix_exporter *exporter,
>                    struct ofpbuf *packet, const struct flow *flow,
>                    uint64_t packet_delta_count, uint32_t obs_domain_id,
> -                  uint32_t obs_point_id)
> +                  uint32_t obs_point_id, odp_port_t output_odp_port,
> +                  const struct dpif_ipfix_port *tunnel_port,
> +                  const struct flow_tnl *tunnel_key)
>  {
>      struct ipfix_flow_cache_entry *entry;
>
>      /* Create a flow cache entry from the sample. */
>      entry = xmalloc(sizeof *entry);
>      ipfix_cache_entry_init(entry, packet, flow, packet_delta_count,
> -                           obs_domain_id, obs_point_id);
> +                           obs_domain_id, obs_point_id,
> +                           output_odp_port, tunnel_port, tunnel_key);
>      ipfix_cache_update(exporter, entry);
>  }
>
>  void
>  dpif_ipfix_bridge_sample(struct dpif_ipfix *di, struct ofpbuf *packet,
> -                         const struct flow *flow) OVS_EXCLUDED(mutex)
> +                         const struct flow *flow,
> +                         odp_port_t input_odp_port, odp_port_t output_odp_port,
> +                         const struct flow_tnl *output_tunnel_key) OVS_EXCLUDED(mutex)
>  {
>      uint64_t packet_delta_count;
> +    const struct flow_tnl *tunnel_key = NULL;
> +    struct dpif_ipfix_port * tunnel_port = NULL;
>
>      ovs_mutex_lock(&mutex);
>      /* Use the sampling probability as an approximation of the number
>       * of matched packets. */
>      packet_delta_count = UINT32_MAX / di->bridge_exporter.probability;
> +    if (di->bridge_exporter.options->enable_tunnel_sampling) {
> +        if (output_odp_port == ODPP_NONE && flow->tunnel.ip_dst) {
> +            /* Input tunnel. */
> +            tunnel_key = &flow->tunnel;
> +            tunnel_port = dpif_ipfix_find_port(di, input_odp_port);
> +        }
> +        if (output_odp_port != ODPP_NONE && output_tunnel_key) {
> +            /* Output tunnel, output_tunnel_key must valid. */
> +            tunnel_key = output_tunnel_key;
> +            tunnel_port = dpif_ipfix_find_port(di, output_odp_port);
> +        }
> +    }
>      dpif_ipfix_sample(&di->bridge_exporter.exporter, packet, flow,
>                        packet_delta_count,
>                        di->bridge_exporter.options->obs_domain_id,
> -                      di->bridge_exporter.options->obs_point_id);
> +                      di->bridge_exporter.options->obs_point_id,
> +                      output_odp_port, tunnel_port, tunnel_key);
>      ovs_mutex_unlock(&mutex);
>  }
>
> @@ -1329,7 +1716,8 @@ dpif_ipfix_flow_sample(struct dpif_ipfix *di, struct ofpbuf *packet,
>      node = dpif_ipfix_find_flow_exporter_map_node(di, collector_set_id);
>      if (node) {
>          dpif_ipfix_sample(&node->exporter.exporter, packet, flow,
> -                          packet_delta_count, obs_domain_id, obs_point_id);
> +                          packet_delta_count, obs_domain_id, obs_point_id,
> +                          ODPP_NONE, NULL, NULL);
>      }
>      ovs_mutex_unlock(&mutex);
>  }
> @@ -1374,8 +1762,8 @@ dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *exporter,
>          if (!template_msg_sent
>              && (exporter->last_template_set_time + IPFIX_TEMPLATE_INTERVAL)
>                  <= export_time_sec) {
> -            ipfix_send_template_msg(exporter, export_time_sec,
> -                                    entry->flow_key.obs_domain_id);
> +            ipfix_send_template_msgs(exporter, export_time_sec,
> +                                     entry->flow_key.obs_domain_id);
>              exporter->last_template_set_time = export_time_sec;
>              template_msg_sent = true;
>          }
> diff --git a/ofproto/ofproto-dpif-ipfix.h b/ofproto/ofproto-dpif-ipfix.h
> index 6ebf8b0..0222054 100644
> --- a/ofproto/ofproto-dpif-ipfix.h
> +++ b/ofproto/ofproto-dpif-ipfix.h
> @@ -19,24 +19,36 @@
>
>  #include <stddef.h>
>  #include <stdint.h>
> +#include <stdbool.h>
> +#include "lib/odp-util.h"
>
>  struct flow;
>  struct ofpbuf;
>  struct ofproto_ipfix_bridge_exporter_options;
>  struct ofproto_ipfix_flow_exporter_options;
> +struct flow_tnl;
> +struct ofport;
>
>  struct dpif_ipfix *dpif_ipfix_create(void);
>  struct dpif_ipfix *dpif_ipfix_ref(const struct dpif_ipfix *);
>  void dpif_ipfix_unref(struct dpif_ipfix *);
>
> +void dpif_ipfix_add_tunnel_port(struct dpif_ipfix *, struct ofport *, odp_port_t);
> +void dpif_ipfix_del_tunnel_port(struct dpif_ipfix *, odp_port_t);
> +
>  uint32_t dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *);
> +bool dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *);
> +bool dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *);
> +bool dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *);
> +bool dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *, odp_port_t);
>  void dpif_ipfix_set_options(
>      struct dpif_ipfix *,
>      const struct ofproto_ipfix_bridge_exporter_options *,
>      const struct ofproto_ipfix_flow_exporter_options *, size_t);
>
>  void dpif_ipfix_bridge_sample(struct dpif_ipfix *, struct ofpbuf *,
> -                              const struct flow *);
> +                              const struct flow *,
> +                              uint32_t, uint32_t, const struct flow_tnl *);
>  void dpif_ipfix_flow_sample(struct dpif_ipfix *, struct ofpbuf *,
>                              const struct flow *, uint32_t, uint16_t, uint32_t,
>                              uint32_t);
> diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c
> index 4f214f7..1d260ae 100644
> --- a/ofproto/ofproto-dpif-upcall.c
> +++ b/ofproto/ofproto-dpif-upcall.c
> @@ -688,7 +688,7 @@ compose_slow_path(struct udpif *udpif, struct xlate_out *xout,
>          ? ODPP_NONE
>          : odp_in_port;
>      pid = dpif_port_get_pid(udpif->dpif, port, flow_hash_5tuple(flow, 0));
> -    odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path, buf);
> +    odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path, NULL, buf);
>  }
>
>  static void
> @@ -800,7 +800,23 @@ read_upcalls(struct handler *handler,
>              break;
>          case IPFIX_UPCALL:
>              if (ipfix) {
> -                dpif_ipfix_bridge_sample(ipfix, packet, &flow);
> +                union user_action_cookie cookie;
> +                struct flow_tnl output_tunnel_key;
> +
> +                memset(&cookie, 0, sizeof cookie);
> +                memcpy(&cookie, nl_attr_get(dupcall->userdata),
> +                       sizeof cookie.ipfix);
> +
> +                if (dupcall->out_tun_key){
> +                    memset(&output_tunnel_key, 0, sizeof output_tunnel_key);
> +                    odp_tun_key_from_attr(dupcall->out_tun_key,
> +                                          &output_tunnel_key);
> +                }
> +                dpif_ipfix_bridge_sample(ipfix, packet, &flow,
> +                                         odp_in_port,
> +                                         cookie.ipfix.output_odp_port,
> +                                         dupcall->out_tun_key ?
> +                                             &output_tunnel_key : NULL);
>              }
>              break;
>          case FLOW_SAMPLE_UPCALL:
> diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
> index e930521..d376773 100644
> --- a/ofproto/ofproto-dpif-xlate.c
> +++ b/ofproto/ofproto-dpif-xlate.c
> @@ -2153,7 +2153,8 @@ compose_sample_action(const struct xbridge *xbridge,
>                        const struct flow *flow,
>                        const uint32_t probability,
>                        const union user_action_cookie *cookie,
> -                      const size_t cookie_size)
> +                      const size_t cookie_size,
> +                      const uint32_t *tunnel_out_port)
>  {
>      size_t sample_offset, actions_offset;
>      odp_port_t odp_port;
> @@ -2170,7 +2171,7 @@ compose_sample_action(const struct xbridge *xbridge,
>      pid = dpif_port_get_pid(xbridge->dpif, odp_port,
>                              flow_hash_5tuple(flow, 0));
>      cookie_offset = odp_put_userspace_action(pid, cookie, cookie_size,
> -                                             odp_actions);
> +                                             tunnel_out_port, odp_actions);
>
>      nl_msg_end_nested(odp_actions, actions_offset);
>      nl_msg_end_nested(odp_actions, sample_offset);
> @@ -2228,7 +2229,7 @@ compose_sflow_action(const struct xbridge *xbridge,
>                           odp_port == ODPP_NONE ? 0 : 1, &cookie);
>
>      return compose_sample_action(xbridge, odp_actions, flow,  probability,
> -                                 &cookie, sizeof cookie.sflow);
> +                                 &cookie, sizeof cookie.sflow, NULL);
>  }
>
>  static void
> @@ -2244,29 +2245,53 @@ compose_flow_sample_cookie(uint16_t probability, uint32_t collector_set_id,
>  }
>
>  static void
> -compose_ipfix_cookie(union user_action_cookie *cookie)
> +compose_ipfix_cookie(union user_action_cookie *cookie,
> +                     odp_port_t output_odp_port)
>  {
>      cookie->type = USER_ACTION_COOKIE_IPFIX;
> +    cookie->ipfix.output_odp_port = output_odp_port;
>  }
>
>  /* Compose SAMPLE action for IPFIX bridge sampling. */
>  static void
>  compose_ipfix_action(const struct xbridge *xbridge,
>                       struct ofpbuf *odp_actions,
> -                     const struct flow *flow)
> +                     const struct flow *flow,
> +                     odp_port_t output_odp_port)
>  {
>      uint32_t probability;
>      union user_action_cookie cookie;
> +    uint32_t *tunnel_out_port = NULL;
>
>      if (!xbridge->ipfix || flow->in_port.ofp_port == OFPP_NONE) {
>          return;
>      }
>
> +    /* For input case, output_odp_port is ODPP_NONE, which is an invalid port number. */
> +    if (output_odp_port == ODPP_NONE &&
> +        !dpif_ipfix_get_bridge_exporter_input_sampling(xbridge->ipfix)) {
> +        return;
> +    }
> +
> +    /* For output case, output_pid is valid*/
> +    if (output_odp_port != ODPP_NONE) {
> +        if (!dpif_ipfix_get_bridge_exporter_output_sampling(xbridge->ipfix)) {
> +            return;
> +        }
> +        /* If enable-tunnel-sampling is enabled, need put an additional option
> +         * attribute: OVS_USERSPACE_ATTR_TUNNEL_OUT_PORT
> +         */
> +        if (dpif_ipfix_get_bridge_exporter_tunnel_sampling(xbridge->ipfix) &&
> +            dpif_ipfix_get_tunnel_port(xbridge->ipfix, output_odp_port) ) {
> +           tunnel_out_port = &output_odp_port;
> +        }
> +    }
> +
>      probability = dpif_ipfix_get_bridge_exporter_probability(xbridge->ipfix);
> -    compose_ipfix_cookie(&cookie);
> +    compose_ipfix_cookie(&cookie, output_odp_port);
>
>      compose_sample_action(xbridge, odp_actions, flow,  probability,
> -                          &cookie, sizeof cookie.ipfix);
> +                          &cookie, sizeof cookie.ipfix, tunnel_out_port);
>  }
>
>  /* SAMPLE action for sFlow must be first action in any given list of
> @@ -2288,7 +2313,14 @@ static void
>  add_ipfix_action(struct xlate_ctx *ctx)
>  {
>      compose_ipfix_action(ctx->xbridge, &ctx->xout->odp_actions,
> -                         &ctx->xin->flow);
> +                         &ctx->xin->flow, ODPP_NONE);
> +}
> +
> +static void
> +add_ipfix_output_action(struct xlate_ctx *ctx, odp_port_t port)
> +{
> +    compose_ipfix_action(ctx->xbridge, &ctx->xout->odp_actions,
> +                         &ctx->xin->flow, port);
>  }
>
>  /* Fix SAMPLE action according to data collected while composing ODP actions.
> @@ -2519,6 +2551,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
>              nl_msg_put_u32(&ctx->xout->odp_actions, OVS_ACTION_ATTR_RECIRC,
>                             xr->recirc_id);
>          } else {
> +            add_ipfix_output_action(ctx, out_port);
>              nl_msg_put_odp_port(&ctx->xout->odp_actions, OVS_ACTION_ATTR_OUTPUT,
>                                  out_port);
>          }
> @@ -3360,7 +3393,7 @@ xlate_sample_action(struct xlate_ctx *ctx,
>    compose_flow_sample_cookie(os->probability, os->collector_set_id,
>                               os->obs_domain_id, os->obs_point_id, &cookie);
>    compose_sample_action(ctx->xbridge, &ctx->xout->odp_actions, &ctx->xin->flow,
> -                        probability, &cookie, sizeof cookie.flow_sample);
> +                        probability, &cookie, sizeof cookie.flow_sample, NULL);
>  }
>
>  static bool
> diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
> index 8c130e1..1ec1b5a 100644
> --- a/ofproto/ofproto-dpif.c
> +++ b/ofproto/ofproto-dpif.c
> @@ -1318,6 +1318,7 @@ destruct(struct ofproto *ofproto_)
>
>      netflow_unref(ofproto->netflow);
>      dpif_sflow_unref(ofproto->sflow);
> +    dpif_ipfix_unref(ofproto->ipfix);
>      hmap_destroy(&ofproto->bundles);
>      mac_learning_unref(ofproto->ml);
>      mcast_snooping_unref(ofproto->ms);
> @@ -1606,6 +1607,9 @@ port_construct(struct ofport *port_)
>      if (netdev_get_tunnel_config(netdev)) {
>          tnl_port_add(port, port->up.netdev, port->odp_port);
>          port->is_tunnel = true;
> +        if (ofproto->ipfix) {
> +           dpif_ipfix_add_tunnel_port(ofproto->ipfix, port_, port->odp_port);
> +        }
>      } else {
>          /* Sanity-check that a mapping doesn't already exist.  This
>           * shouldn't happen for non-tunnel ports. */
> @@ -1667,6 +1671,10 @@ port_destruct(struct ofport *port_)
>          ovs_rwlock_unlock(&ofproto->backer->odp_to_ofport_lock);
>      }
>
> +    if (port->is_tunnel && ofproto->ipfix) {
> +       dpif_ipfix_del_tunnel_port(ofproto->ipfix, port->odp_port);
> +    }
> +
>      tnl_port_del(port);
>      sset_find_and_delete(&ofproto->ports, devname);
>      sset_find_and_delete(&ofproto->ghost_ports, devname);
> @@ -1768,9 +1776,11 @@ set_ipfix(
>      struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
>      struct dpif_ipfix *di = ofproto->ipfix;
>      bool has_options = bridge_exporter_options || flow_exporters_options;
> +    bool new_di = false;
>
>      if (has_options && !di) {
>          di = ofproto->ipfix = dpif_ipfix_create();
> +        new_di = true;
>      }
>
>      if (di) {
> @@ -1780,6 +1790,16 @@ set_ipfix(
>              di, bridge_exporter_options, flow_exporters_options,
>              n_flow_exporters_options);
>
> +        /* Add tunnel ports only when a new ipfix created */
> +        if (new_di == true) {
> +            struct ofport_dpif *ofport;
> +            HMAP_FOR_EACH (ofport, up.hmap_node, &ofproto->up.ports) {
> +                if (ofport->is_tunnel == true) {
> +                    dpif_ipfix_add_tunnel_port(di, &ofport->up, ofport->odp_port);
> +                }
> +            }
> +        }
> +
>          if (!has_options) {
>              dpif_ipfix_unref(di);
>              ofproto->ipfix = NULL;
> diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h
> index beabcf2..a40b966 100644
> --- a/ofproto/ofproto.h
> +++ b/ofproto/ofproto.h
> @@ -72,6 +72,9 @@ struct ofproto_ipfix_bridge_exporter_options {
>      uint32_t obs_point_id;  /* Bridge-wide Observation Point ID. */
>      uint32_t cache_active_timeout;
>      uint32_t cache_max_flows;
> +    bool enable_tunnel_sampling;
> +    bool enable_input_sampling;
> +    bool enable_output_sampling;
>  };
>
>  struct ofproto_ipfix_flow_exporter_options {
> diff --git a/ofproto/tunnel.c b/ofproto/tunnel.c
> index 2b5aa50..46b0719 100644
> --- a/ofproto/tunnel.c
> +++ b/ofproto/tunnel.c
> @@ -341,6 +341,10 @@ tnl_xlate_init(const struct flow *base_flow, struct flow *flow,
>                                    FLOW_TNL_F_KEY);
>          wc->masks.tunnel.ip_tos = UINT8_MAX;
>          wc->masks.tunnel.ip_ttl = UINT8_MAX;
> +        /* The tp_src and tp_dst members in flow_tnl are set to be always
> +         * wildcarded, not to unwildcard them here. */
> +        wc->masks.tunnel.tp_src = 0;
> +        wc->masks.tunnel.tp_dst = 0;
>
>          memset(&wc->masks.pkt_mark, 0xff, sizeof wc->masks.pkt_mark);
>
> diff --git a/tests/odp.at b/tests/odp.at
> index e725f70..5c00764 100644
> --- a/tests/odp.at
> +++ b/tests/odp.at
> @@ -230,12 +230,19 @@ AT_SETUP([OVS datapath actions parsing and formatting - valid forms])
>  AT_DATA([actions.txt], [dnl
>  1,2,3
>  userspace(pid=555666777)
> +userspace(pid=555666777,tunnel_out_port=10)
>  userspace(pid=6633,sFlow(vid=9,pcp=7,output=10))
> +userspace(pid=6633,sFlow(vid=9,pcp=7,output=10),tunnel_out_port=10)
>  userspace(pid=9765,slow_path())
> +userspace(pid=9765,slow_path(),tunnel_out_port=10)
>  userspace(pid=9765,slow_path(cfm))
> +userspace(pid=9765,slow_path(cfm),tunnel_out_port=10)
>  userspace(pid=1234567,userdata(0102030405060708090a0b0c0d0e0f))
> +userspace(pid=1234567,userdata(0102030405060708090a0b0c0d0e0f),tunnel_out_port=10)
>  userspace(pid=6633,flow_sample(probability=123,collector_set_id=1234,obs_domain_id=2345,obs_point_id=3456))
> -userspace(pid=6633,ipfix)
> +userspace(pid=6633,flow_sample(probability=123,collector_set_id=1234,obs_domain_id=2345,obs_point_id=3456),tunnel_out_port=10)
> +userspace(pid=6633,ipfix(output_port=10))
> +userspace(pid=6633,ipfix(output_port=10),tunnel_out_port=10)
>  set(in_port(2))
>  set(eth(src=00:01:02:03:04:05,dst=10:11:12:13:14:15))
>  set(eth_type(0x1234))
> diff --git a/utilities/ovs-vsctl.8.in b/utilities/ovs-vsctl.8.in
> index d397721..62631c7 100644
> --- a/utilities/ovs-vsctl.8.in
> +++ b/utilities/ovs-vsctl.8.in
> @@ -940,11 +940,15 @@ Deconfigure sFlow from \fBbr0\fR, which also destroys the sFlow record
>  Configure bridge \fBbr0\fR to send one IPFIX flow record per packet
>  sample to UDP port 4739 on host 192.168.0.34, with Observation Domain
>  ID 123 and Observation Point ID 456, a flow cache active timeout of 1
> -minute (60 seconds), and a maximum flow cache size of 13 flows:
> +minute (60 seconds), maximum flow cache size of 13 flows, and flows
> +sampled on output port with tunnel info(sampling on input and output
> +port is enabled by default if not disabled) :
>  .IP
>  .B "ovs\-vsctl \-\- set Bridge br0 ipfix=@i \(rs"
>  .IP
> -.B "\-\- \-\-id=@i create IPFIX targets=\(rs\(dq192.168.0.34:4739\(rs\(dq obs_domain_id=123 obs_point_id=456 cache_active_timeout=60 cache_max_flows=13"
> +.B "\-\- \-\-id=@i create IPFIX targets=\(rs\(dq192.168.0.34:4739\(rs\(dq obs_domain_id=123 obs_point_id=456 cache_active_timeout=60 cache_max_flows=13 \(rs"
> +.IP
> +.B "other_config:enable-input-sampling=false other_config:enable-tunnel-sampling=true"
>  .PP
>  Deconfigure the IPFIX settings from \fBbr0\fR, which also destroys the
>  IPFIX record (since it is now unreferenced):
> diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
> index 25e3279..1f10b59 100644
> --- a/vswitchd/bridge.c
> +++ b/vswitchd/bridge.c
> @@ -1160,6 +1160,15 @@ bridge_configure_ipfix(struct bridge *br)
>          if (be_cfg->cache_max_flows) {
>              be_opts.cache_max_flows = *be_cfg->cache_max_flows;
>          }
> +
> +        be_opts.enable_tunnel_sampling = smap_get_bool(&be_cfg->other_config,
> +                                             "enable-tunnel-sampling", true);
> +
> +        be_opts.enable_input_sampling = !smap_get_bool(&be_cfg->other_config,
> +                                              "enable-input-sampling", false);
> +
> +        be_opts.enable_output_sampling = !smap_get_bool(&be_cfg->other_config,
> +                                              "enable-output-sampling", false);
>      }
>
>      if (n_fe_opts > 0) {
> diff --git a/vswitchd/vswitch.ovsschema b/vswitchd/vswitch.ovsschema
> index bc9ea73..bf86f20 100644
> --- a/vswitchd/vswitch.ovsschema
> +++ b/vswitchd/vswitch.ovsschema
> @@ -1,6 +1,6 @@
>  {"name": "Open_vSwitch",
>   "version": "7.8.0",
> - "cksum": "2676751133 20740",
> + "cksum": "4147598271 20869",
>   "tables": {
>     "Open_vSwitch": {
>       "columns": {
> @@ -449,6 +449,9 @@
>                            "minInteger": 0,
>                            "maxInteger": 4294967295},
>                    "min": 0, "max": 1}},
> +       "other_config": {
> +         "type": {"key": "string", "value": "string",
> +                  "min": 0, "max": "unlimited"}},
>         "external_ids": {
>           "type": {"key": "string", "value": "string",
>                    "min": 0, "max": "unlimited"}}}},
> diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
> index 69b3567..28bab8b 100644
> --- a/vswitchd/vswitch.xml
> +++ b/vswitchd/vswitch.xml
> @@ -3877,6 +3877,33 @@
>        disabled.
>      </column>
>
> +    <column name="other_config" key="enable-tunnel-sampling"
> +            type='{"type": "boolean"}'>
> +      For per-bridge packet sampling, i.e. when this row is referenced
> +      from a <ref table="Bridge"/>, enable sampling and reporting tunnel
> +      header 7-tuples in IPFIX flow records. Disabled by default.
> +      Ignored for per-flow sampling, i.e. when this row is referenced
> +      from a <ref table="Flow_Sample_Collector_Set"/>.
> +    </column>
> +
> +    <column name="other_config" key="enable-input-sampling"
> +            type='{"type": "boolean"}'>
> +      For per-bridge packet sampling, i.e. when this row is referenced
> +      from a <ref table="Bridge"/>, enable sampling and reporting flows
> +      at bridge port input in IPFIX flow records. Enabled by default.
> +      Ignored for per-flow sampling, i.e. when this row is referenced
> +      from a <ref table="Flow_Sample_Collector_Set"/>.
> +    </column>
> +
> +    <column name="other_config" key="enable-output-sampling"
> +            type='{"type": "boolean"}'>
> +      For per-bridge packet sampling, i.e. when this row is referenced
> +      from a <ref table="Bridge"/>, enable sampling and reporting flows
> +      at bridge port output in IPFIX flow records. Enabled by default.
> +      Ignored for per-flow sampling, i.e. when this row is referenced
> +      from a <ref table="Flow_Sample_Collector_Set"/>.
> +    </column>
> +
>      <group title="Common Columns">
>        The overall purpose of these columns is described under <code>Common
>        Columns</code> at the beginning of this document.
> --
> 1.7.9.5
> _______________________________________________
> dev mailing list
> dev at openvswitch.org
> http://openvswitch.org/mailman/listinfo/dev



More information about the dev mailing list