[ovs-dev] [bond megaflow v2 5/5] ofproto/bond: Implement bond megaflow using recirculation

Simon Horman horms at verge.net.au
Wed Mar 19 01:12:39 UTC 2014


On Tue, Mar 11, 2014 at 04:56:21PM -0700, Andy Zhou wrote:
> Infrastructure to enable megaflow support for bond ports using
> recirculation. This patch adds the following features:
> * Generate RECIRC action when bond can benefit from recirculation.
> * Populate post recirculation rules in table 254.
> * Uses post recirculation rules for bond rebalancing
> * Logic to detect whether data path supports recirculation.
> 
> Bond port using recirculation is currently turned off by always
> detect the data path as not supporting recirculation.
> 
> Signed-off-by: Andy Zhou <azhou at nicira.com>
> 
> ---
> v1->v2:  Rewritten
> ---
>  lib/dpif-netdev.c             |   50 ++++----
>  lib/flow.c                    |    2 +
>  lib/odp-util.c                |   25 ++--
>  lib/odp-util.h                |    4 +-
>  ofproto/bond.c                |  271 ++++++++++++++++++++++++++++++++++++++---
>  ofproto/bond.h                |   33 ++++-
>  ofproto/ofproto-dpif-upcall.c |   13 +-
>  ofproto/ofproto-dpif-xlate.c  |   78 +++++++++---
>  ofproto/ofproto-dpif-xlate.h  |   21 +++-
>  ofproto/ofproto-dpif.c        |  263 +++++++++++++++++++++++++++++++--------
>  ofproto/ofproto-dpif.h        |   15 ++-
>  ofproto/ofproto.c             |   47 ++++---
>  ofproto/ofproto.h             |    1 +
>  tests/lacp.at                 |   24 +++-
>  tests/ofproto-dpif.at         |  207 ++++++++++++++++++++++---------
>  tests/test-odp.c              |    3 +-
>  16 files changed, 835 insertions(+), 222 deletions(-)
> 
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index 58bcfc1..43cf08f 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -126,6 +126,10 @@ struct dp_netdev {
>      struct ovs_refcount ref_cnt;
>      atomic_flag destroyed;
>  
> +    /* Set to false to simulate older datapath that do not support
> +     * circulation. */
> +    bool enable_recirc;
> +
>      /* Flows.
>       *
>       * Readers of 'cls' and 'flow_table' must take a 'cls->rwlock' read lock.
> @@ -449,6 +453,7 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
>      *CONST_CAST(const char **, &dp->name) = xstrdup(name);
>      ovs_refcount_init(&dp->ref_cnt);
>      atomic_flag_init(&dp->destroyed);
> +    dp->enable_recirc = true;
>  
>      ovs_mutex_init(&dp->flow_mutex);
>      classifier_init(&dp->cls, NULL);
> @@ -1082,7 +1087,7 @@ dpif_netdev_mask_from_nlattrs(const struct nlattr *key, uint32_t key_len,
>  
>  static int
>  dpif_netdev_flow_from_nlattrs(const struct nlattr *key, uint32_t key_len,
> -                              struct flow *flow)
> +                              struct flow *flow, bool enable_recirc)
>  {
>      odp_port_t in_port;
>  
> @@ -1097,7 +1102,7 @@ dpif_netdev_flow_from_nlattrs(const struct nlattr *key, uint32_t key_len,
>              struct ds s;
>  
>              ds_init(&s);
> -            odp_flow_format(key, key_len, NULL, 0, NULL, &s, true);
> +            odp_flow_format(key, key_len, NULL, 0, NULL, &s, enable_recirc);
>              VLOG_ERR("internal error parsing flow key %s", ds_cstr(&s));
>              ds_destroy(&s);
>          }
> @@ -1123,7 +1128,8 @@ dpif_netdev_flow_get(const struct dpif *dpif,
>      struct flow key;
>      int error;
>  
> -    error = dpif_netdev_flow_from_nlattrs(nl_key, nl_key_len, &key);
> +    error = dpif_netdev_flow_from_nlattrs(nl_key, nl_key_len, &key,
> +                                          dp->enable_recirc);
>      if (error) {
>          return error;
>      }
> @@ -1157,19 +1163,6 @@ dpif_netdev_flow_get(const struct dpif *dpif,
>      return error;
>  }
>  
> -/* To support recirculation, user space datapath requires all rules
> - * have exact match to the metadata field, which will store the
> - * recirculation ID. */
> -static void
> -dp_netdev_match_init(struct match *match, const struct flow *flow,
> -                                const struct flow_wildcards *wc)
> -{
> -    struct flow_wildcards netdev_wc = *wc;
> -
> -    netdev_wc.masks.metadata = UINT64_MAX;
> -    match_init(match, flow, &netdev_wc);
> -}
> -
>  static int
>  dp_netdev_flow_add(struct dp_netdev *dp, const struct flow *flow,
>                     const struct flow_wildcards *wc,
> @@ -1189,7 +1182,7 @@ dp_netdev_flow_add(struct dp_netdev *dp, const struct flow *flow,
>  
>      netdev_flow->actions = dp_netdev_actions_create(actions, actions_len);
>  
> -    dp_netdev_match_init(&match, flow, wc);
> +    match_init(&match, flow, wc);

This seems to reverse the change made in patch 2.
Likewise dp_netdev_match_init() was added in patch 2 and then removed here.
I'm not sure I understand why dp_netdev_match_init() is needed at all.

>      cls_rule_init(CONST_CAST(struct cls_rule *, &netdev_flow->cr),
>                    &match, NETDEV_RULE_PRIORITY);
>      fat_rwlock_wrlock(&dp->cls.rwlock);
> @@ -1224,7 +1217,8 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
>      struct flow_wildcards wc;
>      int error;
>  
> -    error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &flow);
> +    error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &flow,
> +                                          dp->enable_recirc);
>      if (error) {
>          return error;
>      }
> @@ -1235,6 +1229,12 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
>          return error;
>      }
>  
> +    if (!dp->enable_recirc) {
> +        if ((flow.recirc_id | flow.dp_hash)  != 0) {
> +            return  EINVAL;
> +        }
> +    }
> +
>      ovs_mutex_lock(&dp->flow_mutex);
>      netdev_flow = dp_netdev_lookup_flow(dp, &flow);
>      if (!netdev_flow) {
> @@ -1293,7 +1293,8 @@ dpif_netdev_flow_del(struct dpif *dpif, const struct dpif_flow_del *del)
>      struct flow key;
>      int error;
>  
> -    error = dpif_netdev_flow_from_nlattrs(del->key, del->key_len, &key);
> +    error = dpif_netdev_flow_from_nlattrs(del->key, del->key_len, &key,
> +                                          dp->enable_recirc);
>      if (error) {
>          return error;
>      }
> @@ -1402,7 +1403,8 @@ dpif_netdev_flow_dump_next(const struct dpif *dpif, void *iter_, void *state_,
>  
>          ofpbuf_use_stack(&buf, &state->keybuf, sizeof state->keybuf);
>          odp_flow_key_from_flow(&buf, &netdev_flow->flow,
> -                               netdev_flow->flow.in_port.odp_port);
> +                               netdev_flow->flow.in_port.odp_port,
> +                               dp->enable_recirc);
>  
>          *key = buf.data;
>          *key_len = buf.size;
> @@ -1416,7 +1418,7 @@ dpif_netdev_flow_dump_next(const struct dpif *dpif, void *iter_, void *state_,
>          minimask_expand(&netdev_flow->cr.match.mask, &wc);
>          odp_flow_key_from_mask(&buf, &wc.masks, &netdev_flow->flow,
>                                 odp_to_u32(wc.masks.in_port.odp_port),
> -                               SIZE_MAX);
> +                               SIZE_MAX, dp->enable_recirc);
>  
>          *mask = buf.data;
>          *mask_len = buf.size;
> @@ -1787,7 +1789,8 @@ dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *packet,
>          ofpbuf_init(buf, buf_size);
>  
>          /* Put ODP flow. */
> -        odp_flow_key_from_flow(buf, flow, flow->in_port.odp_port);
> +        odp_flow_key_from_flow(buf, flow, flow->in_port.odp_port,
> +                               dp->enable_recirc);
>          upcall->key = buf->data;
>          upcall->key_len = buf->size;
>  
> @@ -1858,13 +1861,14 @@ dp_execute_cb(void *aux_, struct ofpbuf *packet,
>          const struct ovs_action_recirc *act;
>          act = nl_attr_get(a);
>          md->recirc_id = ntohl(act->recirc_id);
> -        md->dp_hash = 0;
>  
>          if (act->hash_alg == OVS_RECIRC_HASH_ALG_L4) {
>              struct flow flow;
>  
>              flow_extract(packet, md, &flow);
>              md->dp_hash = flow_hash_symmetric_l4(&flow, ntohl(act->hash_bias));
> +        } else {
> +            md->dp_hash = 0;
>          }
>  
>          dp_netdev_port_input(aux->dp, packet, md);
> diff --git a/lib/flow.c b/lib/flow.c
> index 6c60f04..d2387af 100644
> --- a/lib/flow.c
> +++ b/lib/flow.c
> @@ -398,6 +398,8 @@ flow_extract(struct ofpbuf *packet, const struct pkt_metadata *md,
>          };
>          flow->skb_priority = md->skb_priority;
>          flow->pkt_mark = md->pkt_mark;
> +        flow->recirc_id = md->recirc_id;
> +        flow->dp_hash = md->dp_hash;
>      }
>  
>      packet->l2   = b.data;
> diff --git a/lib/odp-util.c b/lib/odp-util.c
> index 6e46dff..36132c7 100644
> --- a/lib/odp-util.c
> +++ b/lib/odp-util.c
> @@ -2481,7 +2481,7 @@ ovs_to_odp_frag_mask(uint8_t nw_frag_mask)
>  static void
>  odp_flow_key_from_flow__(struct ofpbuf *buf, const struct flow *data,
>                           const struct flow *flow, odp_port_t odp_in_port,
> -                         size_t max_mpls_depth)
> +                         size_t max_mpls_depth, bool enable_recirc)
>  {
>      bool is_mask;
>      struct ovs_key_ethernet *eth_key;
> @@ -2499,12 +2499,16 @@ odp_flow_key_from_flow__(struct ofpbuf *buf, const struct flow *data,
>  
>      nl_msg_put_u32(buf, OVS_KEY_ATTR_SKB_MARK, data->pkt_mark);
>  
> -    if (!is_mask && data->recirc_id)  {
> -        nl_msg_put_u32(buf, OVS_KEY_ATTR_RECIRC_ID, data->recirc_id);
> -    }
> +    if (enable_recirc) {
> +        if (is_mask) {
> +            nl_msg_put_u32(buf, OVS_KEY_ATTR_RECIRC_ID, UINT32_MAX);
> +        } else {
> +            nl_msg_put_u32(buf, OVS_KEY_ATTR_RECIRC_ID, data->recirc_id);
> +        }
>  
> -    if (data->dp_hash != 0) {
> -        nl_msg_put_u32(buf, OVS_KEY_ATTR_DP_HASH, data->dp_hash);
> +        if (data->dp_hash) {
> +            nl_msg_put_u32(buf, OVS_KEY_ATTR_DP_HASH, data->dp_hash);
> +        }
>      }
>  
>      /* Add an ingress port attribute if this is a mask or 'odp_in_port'
> @@ -2679,9 +2683,10 @@ unencap:
>   * capable of being expanded to allow for that much space. */
>  void
>  odp_flow_key_from_flow(struct ofpbuf *buf, const struct flow *flow,
> -                       odp_port_t odp_in_port)
> +                       odp_port_t odp_in_port, bool enable_recirc)
>  {
> -    odp_flow_key_from_flow__(buf, flow, flow, odp_in_port, SIZE_MAX);
> +    odp_flow_key_from_flow__(buf, flow, flow, odp_in_port, SIZE_MAX,
> +                             enable_recirc);
>  }
>  
>  /* Appends a representation of 'mask' as OVS_KEY_ATTR_* attributes to
> @@ -2695,10 +2700,10 @@ odp_flow_key_from_flow(struct ofpbuf *buf, const struct flow *flow,
>  void
>  odp_flow_key_from_mask(struct ofpbuf *buf, const struct flow *mask,
>                         const struct flow *flow, uint32_t odp_in_port_mask,
> -                       size_t max_mpls_depth)
> +                       size_t max_mpls_depth, bool enable_recirc)
>  {
>      odp_flow_key_from_flow__(buf, mask, flow, u32_to_odp(odp_in_port_mask),
> -                             max_mpls_depth);
> +                             max_mpls_depth, enable_recirc);
>  }
>  
>  /* Generate ODP flow key from the given packet metadata */
> diff --git a/lib/odp-util.h b/lib/odp-util.h
> index 7bc64c7..e0b26ac 100644
> --- a/lib/odp-util.h
> +++ b/lib/odp-util.h
> @@ -144,10 +144,10 @@ int odp_flow_from_string(const char *s,
>                           struct ofpbuf *, struct ofpbuf *);
>  
>  void odp_flow_key_from_flow(struct ofpbuf *, const struct flow *,
> -                            odp_port_t odp_in_port);
> +                            odp_port_t odp_in_port, bool enable_recirc);
>  void odp_flow_key_from_mask(struct ofpbuf *, const struct flow *mask,
>                              const struct flow *flow, uint32_t odp_in_port,
> -                            size_t max_mpls_depth);
> +                            size_t max_mpls_depth, bool enable_recirc);
>  
>  uint32_t odp_flow_key_hash(const struct nlattr *, size_t);
>  
> diff --git a/ofproto/bond.c b/ofproto/bond.c
> index c4cfa45..2824731 100644
> --- a/ofproto/bond.c
> +++ b/ofproto/bond.c
> @@ -1,5 +1,5 @@
>  /*
> - * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc.
> + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014 Nicira, Inc.
>   *
>   * Licensed under the Apache License, Version 2.0 (the "License");
>   * you may not use this file except in compliance with the License.
> @@ -23,6 +23,11 @@
>  #include <stdlib.h>
>  #include <math.h>
>  
> +#include "ofp-util.h"
> +#include "ofp-actions.h"
> +#include "ofpbuf.h"
> +#include "ofproto/ofproto-provider.h"
> +#include "ofproto/ofproto-dpif.h"
>  #include "connectivity.h"
>  #include "coverage.h"
>  #include "dynamic-string.h"
> @@ -36,6 +41,7 @@
>  #include "packets.h"
>  #include "poll-loop.h"
>  #include "seq.h"
> +#include "match.h"
>  #include "shash.h"
>  #include "timeval.h"
>  #include "unixctl.h"
> @@ -50,6 +56,7 @@ static struct hmap *const all_bonds OVS_GUARDED_BY(rwlock) = &all_bonds__;
>  /* Bit-mask for hashing a flow down to a bucket.
>   * There are (BOND_MASK + 1) buckets. */
>  #define BOND_MASK 0xff
> +#define RECIRC_RULE_PRIORITY 20   /* Priority level for internal rules */
>  
>  /* A hash bucket for mapping a flow to a slave.
>   * "struct bond" has an array of (BOND_MASK + 1) of these. */
> @@ -57,6 +64,12 @@ struct bond_entry {
>      struct bond_slave *slave;   /* Assigned slave, NULL if unassigned. */
>      uint64_t tx_bytes;          /* Count of bytes recently transmitted. */
>      struct list list_node;      /* In bond_slave's 'entries' list. */
> +
> +    /* Recirculation. */
> +    struct rule *pr_rule;       /* Post recirculation rule for this entry.*/
> +    uint64_t pr_tx_bytes;       /* Record the rule tx_bytes to figure out
> +                                   the delta to update the tx_bytes entry
> +                                   above.*/
>  };
>  
>  /* A bond slave, that is, one of the links comprising a bond. */
> @@ -68,6 +81,7 @@ struct bond_slave {
>  
>      struct netdev *netdev;      /* Network device, owned by the client. */
>      unsigned int change_seq;    /* Tracks changes in 'netdev'. */
> +    ofp_port_t  ofp_port;       /* Open flow port number */
>      char *name;                 /* Name (a copy of netdev_get_name(netdev)). */
>  
>      /* Link status. */
> @@ -86,6 +100,7 @@ struct bond_slave {
>  struct bond {
>      struct hmap_node hmap_node; /* In 'all_bonds' hmap. */
>      char *name;                 /* Name provided by client. */
> +    struct ofproto_dpif *ofproto; /* The bridge this bond belongs to. */
>  
>      /* Slaves. */
>      struct hmap slaves;
> @@ -109,8 +124,10 @@ struct bond {
>      /* SLB specific bonding info. */
>      struct bond_entry *hash;     /* An array of (BOND_MASK + 1) elements. */
>      int rebalance_interval;      /* Interval between rebalances, in ms. */
> -    long long int next_rebalance; /* Next rebalancing time. */
> +    long long int next_rebalance;/* Next rebalancing time. */
>      bool send_learning_packets;
> +    uint32_t recirc_id;          /* Non zero if recirculation can be used.*/
> +    struct hmap pr_rule_ops;     /* Helps to maintain post recirculation rules.*/
>  
>      /* Legacy compatibility. */
>      long long int next_fake_iface_update; /* LLONG_MAX if disabled. */
> @@ -119,6 +136,22 @@ struct bond {
>      struct ovs_refcount ref_cnt;
>  };
>  
> +/* What to do with an bond_recirc_rule. */
> +enum bond_op {
> +    ADD,        /* Add the rule to ofproto's flow table. */
> +    DELETE,     /* Delete the rule from the ofproto's flow table. */
> +
> +};
> +
> +/* A rule to add to or delete from ofproto's internal flow table. */
> +struct bond_pr_rule_op {
> +    struct hmap_node hmap_node;
> +    struct match match;
> +    ofp_port_t out_ofport;
> +    enum bond_op op;
> +    struct rule *pr_rule;
> +};
> +
>  static void bond_entry_reset(struct bond *) OVS_REQ_WRLOCK(rwlock);
>  static struct bond_slave *bond_slave_lookup(struct bond *, const void *slave_)
>      OVS_REQ_RDLOCK(rwlock);
> @@ -185,17 +218,21 @@ bond_mode_to_string(enum bond_mode balance) {
>   * The caller should register each slave on the new bond by calling
>   * bond_slave_register().  */
>  struct bond *
> -bond_create(const struct bond_settings *s)
> +bond_create(const struct bond_settings *s, struct ofproto_dpif *ofproto)
>  {
>      struct bond *bond;
>  
>      bond = xzalloc(sizeof *bond);
> +    bond->ofproto = ofproto;
>      hmap_init(&bond->slaves);
>      list_init(&bond->enabled_slaves);
>      ovs_mutex_init(&bond->mutex);
>      bond->next_fake_iface_update = LLONG_MAX;
>      ovs_refcount_init(&bond->ref_cnt);
>  
> +    bond->recirc_id = 0;
> +    hmap_init(&bond->pr_rule_ops);
> +
>      bond_reconfigure(bond, s);
>      return bond;
>  }
> @@ -216,6 +253,7 @@ void
>  bond_unref(struct bond *bond)
>  {
>      struct bond_slave *slave, *next_slave;
> +    struct bond_pr_rule_op *pr_op, *next_op;
>  
>      if (!bond || ovs_refcount_unref(&bond->ref_cnt) != 1) {
>          return;
> @@ -236,10 +274,116 @@ bond_unref(struct bond *bond)
>      ovs_mutex_destroy(&bond->mutex);
>      free(bond->hash);
>      free(bond->name);
> -    ovs_refcount_destroy(&bond->ref_cnt);
> +
> +    HMAP_FOR_EACH_SAFE(pr_op, next_op, hmap_node, &bond->pr_rule_ops) {
> +        hmap_remove(&bond->pr_rule_ops, &pr_op->hmap_node);
> +        free(pr_op);
> +    }
> +    hmap_destroy(&bond->pr_rule_ops);
> +
> +    if (bond->recirc_id) {
> +        ofproto_dpif_free_recirc_id(bond->ofproto, bond->recirc_id);
> +    }
> +
>      free(bond);
> +
> +    ovs_refcount_destroy(&bond->ref_cnt);
> +}
> +
> +static void
> +add_pr_rule(struct bond *bond, const struct match *match,
> +            ofp_port_t out_ofport, struct rule *rule)
> +{
> +    uint32_t hash = match_hash(match, 0);
> +    struct bond_pr_rule_op *pr_op;
> +
> +    HMAP_FOR_EACH_WITH_HASH(pr_op, hmap_node, hash, &bond->pr_rule_ops) {
> +        if (match_equal(&pr_op->match, match)) {
> +            pr_op->op = ADD;
> +            return;
> +        }
> +    }
> +
> +    pr_op = xmalloc(sizeof *pr_op);
> +    pr_op->match = *match;
> +    pr_op->op = ADD;
> +    pr_op->out_ofport = out_ofport;
> +    pr_op->pr_rule = rule;
> +    hmap_insert(&bond->pr_rule_ops, &pr_op->hmap_node, hash);
>  }
>  
> +static void
> +update_recirc_rules(struct bond *bond)
> +{
> +    struct match match;
> +    struct bond_pr_rule_op *pr_op, *next_op;
> +    uint64_t ofpacts_stub[128 / 8];
> +    struct ofpbuf ofpacts;
> +    int i;
> +
> +    ofpbuf_use_stub(&ofpacts, ofpacts_stub, sizeof ofpacts_stub);
> +
> +    HMAP_FOR_EACH(pr_op, hmap_node, &bond->pr_rule_ops) {
> +        pr_op->op = DELETE;
> +    }
> +
> +    if ((bond->hash == NULL) || (!bond->recirc_id))
> +        return;
> +
> +    for (i = 0; i < BOND_MASK + 1; i++) {
> +        struct bond_slave *slave = bond->hash[i].slave;
> +
> +        if (slave) {
> +            match_init_catchall(&match);
> +            /* recirc_id -> metadata to speed up look ups. */
> +            match_set_metadata(&match, htonll(bond->recirc_id));
> +            match_set_recirc_id(&match, bond->recirc_id);
> +            match_set_dp_hash_masked(&match, i, BOND_MASK);
> +
> +            add_pr_rule(bond, &match, slave->ofp_port,
> +                            bond->hash[i].pr_rule);
> +        }
> +    }
> +
> +    HMAP_FOR_EACH_SAFE(pr_op, next_op, hmap_node, &bond->pr_rule_ops) {
> +        int error;
> +        struct rule_dpif *rule_dpif;
> +        switch (pr_op->op) {
> +        case ADD:
> +            ofpbuf_clear(&ofpacts);
> +            ofpact_put_OUTPUT(&ofpacts)->port = pr_op->out_ofport;
> +            error = ofproto_dpif_add_internal_flow(bond->ofproto,
> +                                                   &pr_op->match,
> +                                                   RECIRC_RULE_PRIORITY,
> +                                                   &ofpacts, &rule_dpif);
> +            if (error) {
> +                VLOG_ERR("Bond: Failed to add post recirculation flow %s \n",
> +                    match_to_string(&pr_op->match, 0));
> +                pr_op->pr_rule = NULL;
> +            } else {
> +                pr_op->pr_rule = (struct rule *)(rule_dpif); /* Can we do better ? XXXX */
> +            }
> +            break;
> +
> +        case DELETE:
> +            error = ofproto_dpif_delete_internal_flow(bond->ofproto,
> +                                                      &pr_op->match, 0);
> +            hmap_remove(&bond->pr_rule_ops, &pr_op->hmap_node);
> +            pr_op->pr_rule = NULL;
> +            free(pr_op);
> +
> +            if (error) {
> +                VLOG_ERR("Bond: Failed to remove post recirculation flow %s\n",
> +                    match_to_string(&pr_op->match, 0));
> +            }
> +            break;
> +        }
> +    }
> +
> +    ofpbuf_uninit(&ofpacts);
> +}
> +
> +
>  /* Updates 'bond''s overall configuration to 's'.
>   *
>   * The caller should register each slave on 'bond' by calling
> @@ -300,6 +444,15 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s)
>          bond->bond_revalidate = false;
>      }
>  
> +    if (bond->balance != BM_AB) {
> +        if (!bond->recirc_id) {
> +            bond->recirc_id = ofproto_dpif_alloc_recirc_id(bond->ofproto);
> +        }
> +    } else if (bond -> recirc_id) {
> +        ofproto_dpif_free_recirc_id(bond->ofproto, bond->recirc_id);
> +        bond->recirc_id = 0;
> +    }
> +
>      if (bond->balance == BM_AB || !bond->hash || revalidate) {
>          bond_entry_reset(bond);
>      }
> @@ -328,7 +481,8 @@ bond_slave_set_netdev__(struct bond_slave *slave, struct netdev *netdev)
>   * 'slave_' or destroying 'bond'.
>   */
>  void
> -bond_slave_register(struct bond *bond, void *slave_, struct netdev *netdev)
> +bond_slave_register(struct bond *bond, void *slave_,
> +                    ofp_port_t ofport, struct netdev *netdev)
>  {
>      struct bond_slave *slave;
>  
> @@ -340,6 +494,7 @@ bond_slave_register(struct bond *bond, void *slave_, struct netdev *netdev)
>          hmap_insert(&bond->slaves, &slave->hmap_node, hash_pointer(slave_, 0));
>          slave->bond = bond;
>          slave->aux = slave_;
> +        slave->ofp_port = ofport;
>          slave->delay_expires = LLONG_MAX;
>          slave->name = xstrdup(netdev_get_name(netdev));
>          bond->bond_revalidate = true;
> @@ -689,6 +844,82 @@ bond_choose_output_slave(struct bond *bond, const struct flow *flow,
>      return aux;
>  }
>  
> +/* Recirculation. */
> +static void bond_entry_acount(struct bond_entry *entry, uint64_t rule_tx_bytes)
> +    OVS_REQ_RDLOCK(rwlock)
> +{
> +    if (entry->slave) {
> +        uint64_t delta;
> +
> +        delta = rule_tx_bytes - entry->pr_tx_bytes;
> +        entry->tx_bytes += delta;
> +        entry->pr_tx_bytes = rule_tx_bytes;
> +    }
> +}
> +
> +/* Maintain bond stats using post recirculation rule byte counters.*/
> +void
> +bond_recirculation_account(struct bond *bond)
> +{
> +    int i;
> +
> +    ovs_rwlock_rdlock(&rwlock);
> +    for (i =0; i< BOND_MASK + 1; i++) {
> +        struct bond_entry *entry = &bond->hash[i];
> +        struct rule *rule;
> +
> +        rule = entry->pr_rule;
> +        if (rule) {
> +            uint64_t n_packets OVS_UNUSED;
> +            long long int used OVS_UNUSED;
> +            uint64_t n_bytes;
> +
> +            rule->ofproto->ofproto_class->rule_get_stats(
> +                rule, &n_packets, &n_bytes, &used);
> +            bond_entry_acount(entry, n_bytes);
> +        }
> +    }
> +    ovs_rwlock_unlock(&rwlock);
> +}
> +
> +bool
> +bond_may_recirc(const struct bond *bond, uint32_t *recirc_id,
> +                uint32_t *hash_bias)
> +{
> +    bool rv = false;
> +
> +    if ((bond->balance == BM_TCP)) {
> +        if (recirc_id) {
> +            *recirc_id = bond->recirc_id;
> +        }
> +        if (hash_bias) {
> +            *hash_bias = bond->basis;
> +        }
> +        rv = true;
> +    }
> +
> +    return rv;
> +}
> +
> +void
> +bond_update_post_recirc_rules(struct bond* bond)
> +{
> +   struct bond_entry *e;
> +
> +   /* Make sure all bond entries are populated */
> +   for (e = bond->hash; e <= &bond->hash[BOND_MASK]; e++) {
> +       if (!e->slave || !e->slave->enabled) {
> +            e->slave = CONTAINER_OF(hmap_random_node(&bond->slaves),
> +                                    struct bond_slave, hmap_node);
> +            if (!e->slave->enabled) {
> +                e->slave = bond->active_slave;
> +            }
> +        }
> +   }
> +
> +   update_recirc_rules(bond);
> +}
> +
>  /* Rebalancing. */
>  
>  static bool
> @@ -715,7 +946,6 @@ bond_slave_from_bal_node(struct list *bal) OVS_REQ_RDLOCK(rwlock)
>  {
>      return CONTAINER_OF(bal, struct bond_slave, bal_node);
>  }
> -
>  static void
>  log_bals(struct bond *bond, const struct list *bals)
>  {
> @@ -846,19 +1076,22 @@ reinsert_bal(struct list *bals, struct bond_slave *slave)
>  
>  /* If 'bond' needs rebalancing, does so.
>   *
> - * The caller should have called bond_account() for each active flow, to ensure
> - * that flow data is consistently accounted at this point. */
> -void
> + * The caller should have called bond_account() for each active flow, or in case
> + * of recirculation is used, have called bond_recirculation_account(bond),
> + * to ensure that flow data is consistently accounted at this point.
> + *
> + * Return whether rebalancing took place.*/
> +bool
>  bond_rebalance(struct bond *bond)
>  {
>      struct bond_slave *slave;
>      struct bond_entry *e;
>      struct list bals;
> +    bool rebalanced = false;
>  
>      ovs_rwlock_wrlock(&rwlock);
>      if (!bond_is_balanced(bond) || time_msec() < bond->next_rebalance) {
> -        ovs_rwlock_unlock(&rwlock);
> -        return;
> +        goto done;
>      }
>      bond->next_rebalance = time_msec() + bond->rebalance_interval;
>  
> @@ -917,6 +1150,7 @@ bond_rebalance(struct bond *bond)
>              /* Re-sort 'bals'. */
>              reinsert_bal(&bals, from);
>              reinsert_bal(&bals, to);
> +	    rebalanced = true;
>          } else {
>              /* Can't usefully migrate anything away from 'from'.
>               * Don't reconsider it. */
> @@ -933,7 +1167,10 @@ bond_rebalance(struct bond *bond)
>              e->slave = NULL;
>          }
>      }
> +
> +done:
>      ovs_rwlock_unlock(&rwlock);
> +    return rebalanced;
>  }
>  
>  /* Bonding unixctl user interface functions. */
> @@ -973,15 +1210,15 @@ bond_unixctl_list(struct unixctl_conn *conn,
>      struct ds ds = DS_EMPTY_INITIALIZER;
>      const struct bond *bond;
>  
> -    ds_put_cstr(&ds, "bond\ttype\tslaves\n");
> +    ds_put_cstr(&ds, "bond\ttype\trecircID\tslaves\n");
>  
>      ovs_rwlock_rdlock(&rwlock);
>      HMAP_FOR_EACH (bond, hmap_node, all_bonds) {
>          const struct bond_slave *slave;
>          size_t i;
>  
> -        ds_put_format(&ds, "%s\t%s\t",
> -                      bond->name, bond_mode_to_string(bond->balance));
> +        ds_put_format(&ds, "%s\t%s\t%d\t", bond->name,
> +                      bond_mode_to_string(bond->balance), bond->recirc_id);
>  
>          i = 0;
>          HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
> @@ -1004,12 +1241,18 @@ bond_print_details(struct ds *ds, const struct bond *bond)
>      struct shash slave_shash = SHASH_INITIALIZER(&slave_shash);
>      const struct shash_node **sorted_slaves = NULL;
>      const struct bond_slave *slave;
> +    bool may_recirc;
> +    uint32_t recirc_id;
>      int i;
>  
>      ds_put_format(ds, "---- %s ----\n", bond->name);
>      ds_put_format(ds, "bond_mode: %s\n",
>                    bond_mode_to_string(bond->balance));
>  
> +    may_recirc = bond_may_recirc(bond, &recirc_id , NULL);
> +    ds_put_format(ds, "bond may use recirculation: %s, Recirc-ID : %d\n",
> +                  may_recirc ? "yes" : "no", may_recirc ? recirc_id: -1);
> +
>      ds_put_format(ds, "bond-hash-basis: %"PRIu32"\n", bond->basis);
>  
>      ds_put_format(ds, "updelay: %d ms\n", bond->updelay);
> diff --git a/ofproto/bond.h b/ofproto/bond.h
> index 5b3814e..fc7df71 100644
> --- a/ofproto/bond.h
> +++ b/ofproto/bond.h
> @@ -1,5 +1,5 @@
>  /*
> - * Copyright (c) 2008, 2009, 2010, 2011 Nicira, Inc.
> + * Copyright (c) 2008, 2009, 2010, 2011, 2014 Nicira, Inc.
>   *
>   * Licensed under the Apache License, Version 2.0 (the "License");
>   * you may not use this file except in compliance with the License.
> @@ -19,12 +19,13 @@
>  
>  #include <stdbool.h>
>  #include <stdint.h>
> -
> +#include "ofproto-provider.h"
>  #include "packets.h"
>  
>  struct flow;
>  struct netdev;
>  struct ofpbuf;
> +struct ofproto_dpif;
>  enum lacp_status;
>  
>  /* How flows are balanced among bond slaves. */
> @@ -60,12 +61,13 @@ struct bond_settings {
>  void bond_init(void);
>  
>  /* Basics. */
> -struct bond *bond_create(const struct bond_settings *);
> +struct bond *bond_create(const struct bond_settings *,
> +                         struct ofproto_dpif *ofproto);
>  void bond_unref(struct bond *);
>  struct bond *bond_ref(const struct bond *);
>  
>  bool bond_reconfigure(struct bond *, const struct bond_settings *);
> -void bond_slave_register(struct bond *, void *slave_, struct netdev *);
> +void bond_slave_register(struct bond *, void *slave_, ofp_port_t ofport, struct netdev *);
>  void bond_slave_set_netdev(struct bond *, void *slave_, struct netdev *);
>  void bond_slave_unregister(struct bond *, const void *slave);
>  
> @@ -94,6 +96,27 @@ void *bond_choose_output_slave(struct bond *, const struct flow *,
>  /* Rebalancing. */
>  void bond_account(struct bond *, const struct flow *, uint16_t vlan,
>                    uint64_t n_bytes);
> -void bond_rebalance(struct bond *);
> +bool bond_rebalance(struct bond *);
> +
> +/* Recirculation */
> +
> +/*
> +Only balance_tcp mode uses recirculation.
> +
> +When recirculation is used, each bond port is assigned with a unique recirc_id.
> +The output action to the bond port will be replaced by a RECIRC action.
> +
> +    ... actions= ... RECIRC(L4_HASH, bond_recirc_id) ....
> +
> +On handling first output packet, 256 post recirculation flows are installed:
> +
> +   recirc_id=<bond_recirc_id>, dp_hash=<[0..255]>/0xff, actions: output<slave>
> +
> +Bond module pulls stats from those post recirculation rules. If rebalancing
> +is needed, those rules are updated with new output actions.
> +*/
>  
> +void bond_update_post_recirc_rules(struct bond *);
> +bool bond_may_recirc(const struct bond *, uint32_t *recirc_id, uint32_t *hash_bias);
> +void bond_recirculation_account(struct bond *);
>  #endif /* bond.h */
> diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c
> index 7dbd7f7..fffe713 100644
> --- a/ofproto/ofproto-dpif-upcall.c
> +++ b/ofproto/ofproto-dpif-upcall.c
> @@ -1083,7 +1083,7 @@ handle_upcalls(struct handler *handler, struct list *upcalls)
>      HMAP_FOR_EACH (miss, hmap_node, &misses) {
>          struct xlate_in xin;
>  
> -        xlate_in_init(&xin, miss->ofproto, &miss->flow, NULL,
> +        xlate_in_init(&xin, miss->ofproto, &miss->flow, NULL, 0,
>                        miss->stats.tcp_flags, NULL);
>          xin.may_learn = true;
>  
> @@ -1125,7 +1125,7 @@ handle_upcalls(struct handler *handler, struct list *upcalls)
>          if (miss->xout.slow) {
>              struct xlate_in xin;
>  
> -            xlate_in_init(&xin, miss->ofproto, &miss->flow, NULL, 0, packet);
> +            xlate_in_init(&xin, miss->ofproto, &miss->flow, NULL, 0, 0, packet);
>              xlate_actions_for_side_effects(&xin);
>          }
>  
> @@ -1168,10 +1168,13 @@ handle_upcalls(struct handler *handler, struct list *upcalls)
>              ofpbuf_use_stack(&mask, &miss->mask_buf, sizeof miss->mask_buf);
>              if (megaflow) {
>                  size_t max_mpls;
> +                bool enable_recirc;
>  
> +                enable_recirc = ofproto_dpif_get_enable_recirc(miss->ofproto);
>                  max_mpls = ofproto_dpif_get_max_mpls_depth(miss->ofproto);
>                  odp_flow_key_from_mask(&mask, &miss->xout.wc.masks,
> -                                       &miss->flow, UINT32_MAX, max_mpls);
> +                                       &miss->flow, UINT32_MAX, max_mpls,
> +                                       enable_recirc);
>              }
>  
>              op = &ops[n_ops++];
> @@ -1352,7 +1355,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_flow_dump *udump,
>          goto exit;
>      }
>  
> -    xlate_in_init(&xin, ofproto, &flow, NULL, push.tcp_flags, NULL);
> +    xlate_in_init(&xin, ofproto, &flow, NULL, 0, push.tcp_flags, NULL);
>      xin.resubmit_stats = push.n_packets ? &push : NULL;
>      xin.may_learn = push.n_packets > 0;
>      xin.skip_wildcards = !udump->need_revalidate;
> @@ -1459,7 +1462,7 @@ push_dump_ops(struct revalidator *revalidator,
>                                 NULL, NULL, &netflow, NULL)) {
>                  struct xlate_in xin;
>  
> -                xlate_in_init(&xin, ofproto, &flow, NULL, push->tcp_flags,
> +                xlate_in_init(&xin, ofproto, &flow, NULL, 0, push->tcp_flags,
>                                NULL);
>                  xin.resubmit_stats = push->n_packets ? push : NULL;
>                  xin.may_learn = push->n_packets > 0;
> diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
> index 75728ce..53350bc 100644
> --- a/ofproto/ofproto-dpif-xlate.c
> +++ b/ofproto/ofproto-dpif-xlate.c
> @@ -58,6 +58,8 @@ VLOG_DEFINE_THIS_MODULE(ofproto_dpif_xlate);
>  /* Maximum depth of flow table recursion (due to resubmit actions) in a
>   * flow translation. */
>  #define MAX_RESUBMIT_RECURSION 64
> +#define MAX_INTERNAL_RESUBMITS 1   /* Max resbmits allowed using rules in
> +                                      internal table. */
>  
>  /* Maximum number of resubmit actions in a flow translation, whether they are
>   * recursive or not. */
> @@ -84,11 +86,13 @@ struct xbridge {
>      /* Special rules installed by ofproto-dpif. */
>      struct rule_dpif *miss_rule;
>      struct rule_dpif *no_packet_in_rule;
> -
>      enum ofp_config_flags frag;   /* Fragmentation handling. */
>      bool has_in_band;             /* Bridge has in band control? */
>      bool forward_bpdu;            /* Bridge forwards STP BPDUs? */
>  
> +    /* True if the datapath supports recirculation. */
> +    bool enable_recirc;
> +
>      /* True if the datapath supports variable-length
>       * OVS_USERSPACE_ATTR_USERDATA in OVS_ACTION_ATTR_USERSPACE actions.
>       * False if the datapath supports only 8-byte (or shorter) userdata. */
> @@ -97,6 +101,9 @@ struct xbridge {
>      /* Number of MPLS label stack entries that the datapath supports
>       * in matches. */
>      size_t max_mpls_depth;
> +
> +    /* Number of open flow tables visible to the controller. */
> +    uint8_t n_visible_tables;
>  };
>  
>  struct xbundle {
> @@ -224,10 +231,10 @@ static void do_xlate_actions(const struct ofpact *, size_t ofpacts_len,
>                               struct xlate_ctx *);
>  static void xlate_actions__(struct xlate_in *, struct xlate_out *)
>      OVS_REQ_RDLOCK(xlate_rwlock);
> -    static void xlate_normal(struct xlate_ctx *);
> -    static void xlate_report(struct xlate_ctx *, const char *);
> -    static void xlate_table_action(struct xlate_ctx *, ofp_port_t in_port,
> -                                   uint8_t table_id, bool may_packet_in);
> +static void xlate_normal(struct xlate_ctx *);
> +static void xlate_report(struct xlate_ctx *, const char *);
> +static void xlate_table_action(struct xlate_ctx *, ofp_port_t in_port,
> +                               uint8_t table_id, bool may_packet_in);
>  static bool input_vid_is_valid(uint16_t vid, struct xbundle *, bool warn);
>  static uint16_t input_vid_to_vlan(const struct xbundle *, uint16_t vid);
>  static void output_normal(struct xlate_ctx *, const struct xbundle *,
> @@ -254,8 +261,11 @@ xlate_ofproto_set(struct ofproto_dpif *ofproto, const char *name,
>                    const struct dpif_ipfix *ipfix,
>                    const struct netflow *netflow, enum ofp_config_flags frag,
>                    bool forward_bpdu, bool has_in_band,
> +                  bool enable_recirc,
>                    bool variable_length_userdata,
> -                  size_t max_mpls_depth)
> +                  size_t max_mpls_depth,
> +                  uint8_t n_visible_tables)
> +
>  {
>      struct xbridge *xbridge = xbridge_lookup(ofproto);
>  
> @@ -307,8 +317,10 @@ xlate_ofproto_set(struct ofproto_dpif *ofproto, const char *name,
>      xbridge->frag = frag;
>      xbridge->miss_rule = miss_rule;
>      xbridge->no_packet_in_rule = no_packet_in_rule;
> +    xbridge->enable_recirc = enable_recirc;
>      xbridge->variable_length_userdata = variable_length_userdata;
>      xbridge->max_mpls_depth = max_mpls_depth;
> +    xbridge->n_visible_tables = n_visible_tables;
>  }
>  
>  void
> @@ -1127,6 +1139,18 @@ output_normal(struct xlate_ctx *ctx, const struct xbundle *out_xbundle,
>                               bundle_node);
>      } else {
>          struct ofport_dpif *ofport;
> +        struct xlate_recirc *xr = &ctx->xout->recirc;
> +
> +        if (ctx->xbridge->enable_recirc) {
> +            ctx->xout->use_recirc = bond_may_recirc(
> +                out_xbundle->bond, &xr->recirc_id, &xr->hash_bias);
> +
> +            if (ctx->xout->use_recirc) {
> +                /* Only TCP mode uses recirculation. */
> +                xr->hash_alg = OVS_RECIRC_HASH_ALG_L4;
> +                bond_update_post_recirc_rules(out_xbundle->bond);
> +            }
> +        }
>  
>          ofport = bond_choose_output_slave(out_xbundle->bond, &ctx->xin->flow,
>                                            &ctx->xout->wc, vid);
> @@ -1800,8 +1824,21 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
>          ctx->xout->slow |= commit_odp_actions(flow, &ctx->base_flow,
>                                                &ctx->xout->odp_actions,
>                                                &ctx->xout->wc);
> -        nl_msg_put_odp_port(&ctx->xout->odp_actions, OVS_ACTION_ATTR_OUTPUT,
> -                            out_port);
> +
> +        if (ctx->xout->use_recirc) {
> +            struct ovs_action_recirc *act_recirc;
> +            struct xlate_recirc *xr = &ctx->xout->recirc;
> +
> +            act_recirc = (struct ovs_action_recirc *)
> +                nl_msg_put_unspec_uninit(&ctx->xout->odp_actions,
> +                    OVS_ACTION_ATTR_RECIRC, sizeof *act_recirc);
> +            act_recirc->recirc_id = htonl(xr->recirc_id);
> +            act_recirc->hash_alg = xr->hash_alg;
> +            act_recirc->hash_bias = htonl(xr->hash_bias);
> +        } else {
> +            nl_msg_put_odp_port(&ctx->xout->odp_actions, OVS_ACTION_ATTR_OUTPUT,
> +                                out_port);
> +        }
>  
>          ctx->sflow_odp_port = odp_port;
>          ctx->sflow_n_outputs++;
> @@ -1846,10 +1883,10 @@ xlate_resubmit_resource_check(struct xlate_ctx *ctx)
>  {
>      static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
>  
> -    if (ctx->recurse >= MAX_RESUBMIT_RECURSION) {
> +    if (ctx->recurse >= MAX_RESUBMIT_RECURSION + MAX_INTERNAL_RESUBMITS) {
>          VLOG_ERR_RL(&rl, "resubmit actions recursed over %d times",
>                      MAX_RESUBMIT_RECURSION);
> -    } else if (ctx->resubmits >= MAX_RESUBMITS) {
> +    } else if (ctx->resubmits >= MAX_RESUBMITS + MAX_INTERNAL_RESUBMITS) {
>          VLOG_ERR_RL(&rl, "over %d resubmit actions", MAX_RESUBMITS);
>      } else if (ctx->xout->odp_actions.size > UINT16_MAX) {
>          VLOG_ERR_RL(&rl, "resubmits yielded over 64 kB of actions");
> @@ -2050,6 +2087,13 @@ xlate_ofpact_resubmit(struct xlate_ctx *ctx,
>  {
>      ofp_port_t in_port;
>      uint8_t table_id;
> +    bool may_packet_in = false;
> +
> +    if (ctx->table_id >= ctx->xbridge->n_visible_tables ) {
> +        /* Still allow missed packets to be sent to the controller
> +         * if Resubmiting from an internal table  */
> +        may_packet_in = true;
> +    }
>  
>      in_port = resubmit->in_port;
>      if (in_port == OFPP_IN_PORT) {
> @@ -2061,7 +2105,7 @@ xlate_ofpact_resubmit(struct xlate_ctx *ctx,
>          table_id = ctx->table_id;
>      }
>  
> -    xlate_table_action(ctx, in_port, table_id, false);
> +    xlate_table_action(ctx, in_port, table_id, may_packet_in);
>  }
>  
>  static void
> @@ -2795,7 +2839,8 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
>  
>  void
>  xlate_in_init(struct xlate_in *xin, struct ofproto_dpif *ofproto,
> -              const struct flow *flow, struct rule_dpif *rule,
> +              const struct flow *flow,
> +              struct rule_dpif *rule, uint8_t table_id,
>                uint16_t tcp_flags, const struct ofpbuf *packet)
>  {
>      xin->ofproto = ofproto;
> @@ -2803,6 +2848,7 @@ xlate_in_init(struct xlate_in *xin, struct ofproto_dpif *ofproto,
>      xin->packet = packet;
>      xin->may_learn = packet != NULL;
>      xin->rule = rule;
> +    xin->table_id = table_id;
>      xin->ofpacts = NULL;
>      xin->ofpacts_len = 0;
>      xin->tcp_flags = tcp_flags;
> @@ -2982,6 +3028,7 @@ xlate_actions__(struct xlate_in *xin, struct xlate_out *xout)
>      }
>  
>      ctx.rule = xin->rule;
> +    ctx.table_id = xin->table_id;
>  
>      ctx.base_flow = *flow;
>      memset(&ctx.base_flow.tunnel, 0, sizeof ctx.base_flow.tunnel);
> @@ -3005,18 +3052,21 @@ xlate_actions__(struct xlate_in *xin, struct xlate_out *xout)
>      ctx.resubmits = 0;
>      ctx.in_group = false;
>      ctx.orig_skb_priority = flow->skb_priority;
> -    ctx.table_id = 0;
>      ctx.exit = false;
>  
>      if (!xin->ofpacts && !ctx.rule) {
> +        uint8_t table_id;
> +
>          rule_dpif_lookup(ctx.xbridge->ofproto, flow,
> -                         !xin->skip_wildcards ? wc : NULL, &rule);
> +                         !xin->skip_wildcards ? wc : NULL, &rule, &table_id);
>          if (ctx.xin->resubmit_stats) {
>              rule_dpif_credit_stats(rule, ctx.xin->resubmit_stats);
>          }
>          ctx.rule = rule;
> +        ctx.table_id = table_id;
>      }
>      xout->fail_open = ctx.rule && rule_dpif_is_fail_open(ctx.rule);
> +    xout->use_recirc = false;
>  
>      if (xin->ofpacts) {
>          ofpacts = xin->ofpacts;
> diff --git a/ofproto/ofproto-dpif-xlate.h b/ofproto/ofproto-dpif-xlate.h
> index 8b01d4e..1e9355d 100644
> --- a/ofproto/ofproto-dpif-xlate.h
> +++ b/ofproto/ofproto-dpif-xlate.h
> @@ -32,6 +32,12 @@ struct dpif_ipfix;
>  struct dpif_sflow;
>  struct mac_learning;
>  
> +struct xlate_recirc {
> +    uint32_t recirc_id;  /* !0 Use recirculation instead of output.*/
> +    uint8_t  hash_alg;   /* !0 Compute hash for recirc before.*/
> +    uint32_t hash_bias;  /* Compute hash for recirc before.*/
> +};
> +
>  struct xlate_out {
>      /* Wildcards relevant in translation.  Any fields that were used to
>       * calculate the action must be set for caching and kernel
> @@ -50,6 +56,9 @@ struct xlate_out {
>      ofp_port_t nf_output_iface; /* Output interface index for NetFlow. */
>      mirror_mask_t mirrors;      /* Bitmap of associated mirrors. */
>  
> +    bool use_recirc;            /* Should generate recirc? */
> +    struct xlate_recirc recirc; /* Information used for generating
> +                                 * recirculation actions */
>      uint64_t odp_actions_stub[256 / 8];
>      struct ofpbuf odp_actions;
>  };
> @@ -83,6 +92,9 @@ struct xlate_in {
>       * are NULL, xlate_actions() will do the initial rule lookup itself. */
>      struct rule_dpif *rule;
>  
> +    /* The table contains the rule above. */
> +    uint8_t table_id;
> +
>      /* The actions to translate.  If 'rule' is not NULL, these may be NULL. */
>      const struct ofpact *ofpacts;
>      size_t ofpacts_len;
> @@ -129,8 +141,9 @@ void xlate_ofproto_set(struct ofproto_dpif *, const char *name,
>                         const struct mbridge *, const struct dpif_sflow *,
>                         const struct dpif_ipfix *, const struct netflow *,
>                         enum ofp_config_flags, bool forward_bpdu,
> -                       bool has_in_band, bool variable_length_userdata,
> -                       size_t mpls_label_stack_length)
> +                       bool has_in_band, bool enable_recirc,
> +                       bool variable_length_userdata,
> +                       size_t mpls_label_stack_length, uint8_t n_oftables)
>      OVS_REQ_WRLOCK(xlate_rwlock);
>  void xlate_remove_ofproto(struct ofproto_dpif *) OVS_REQ_WRLOCK(xlate_rwlock);
>  
> @@ -161,8 +174,8 @@ int xlate_receive(const struct dpif_backer *, struct ofpbuf *packet,
>  void xlate_actions(struct xlate_in *, struct xlate_out *)
>      OVS_EXCLUDED(xlate_rwlock);
>  void xlate_in_init(struct xlate_in *, struct ofproto_dpif *,
> -                   const struct flow *, struct rule_dpif *, uint16_t tcp_flags,
> -                   const struct ofpbuf *packet);
> +                   const struct flow *, struct rule_dpif *, uint8_t table_id,
> +                   uint16_t tcp_flags, const struct ofpbuf *packet);
>  void xlate_out_uninit(struct xlate_out *);
>  void xlate_actions_for_side_effects(struct xlate_in *);
>  void xlate_out_copy(struct xlate_out *dst, const struct xlate_out *src);
> diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
> index 4018129..8dae9be 100644
> --- a/ofproto/ofproto-dpif.c
> +++ b/ofproto/ofproto-dpif.c
> @@ -74,6 +74,7 @@ COVERAGE_DEFINE(packet_in_overflow);
>  
>  /* Number of implemented OpenFlow tables. */
>  enum { N_TABLES = 255 };
> +
>  enum { TBL_INTERNAL = N_TABLES - 1 };    /* Used for internal hidden rules. */
>  BUILD_ASSERT_DECL(N_TABLES >= 2 && N_TABLES <= 255);
>  

The hunk immediately above seems spurious..

> @@ -278,6 +279,7 @@ struct dpif_backer {
>      bool recv_set_enable; /* Enables or disables receiving packets. */
>  
>      /* Recirculation. */
> +    bool enable_recirc;   /* True if the datapath supports recirculation */
>      struct ovs_mutex recirc_id_lock; /* lock for recirc_ids */
>      struct recirc_id_set rids;       /* Managing recirculation ID. */
>  
> @@ -358,6 +360,12 @@ ofproto_dpif_get_max_mpls_depth(const struct ofproto_dpif *ofproto)
>      return ofproto->backer->max_mpls_depth;
>  }
>  
> +bool
> +ofproto_dpif_get_enable_recirc(const struct ofproto_dpif *ofproto)
> +{
> +    return ofproto->backer->enable_recirc;
> +}
> +
>  static struct ofport_dpif *get_ofp_port(const struct ofproto_dpif *ofproto,
>                                          ofp_port_t ofp_port);
>  static void ofproto_trace(struct ofproto_dpif *, const struct flow *,
> @@ -597,8 +605,10 @@ type_run(const char *type)
>                                ofproto->netflow, ofproto->up.frag_handling,
>                                ofproto->up.forward_bpdu,
>                                connmgr_has_in_band(ofproto->up.connmgr),
> +                              ofproto->backer->enable_recirc,
>                                ofproto->backer->variable_length_userdata,
> -                              ofproto->backer->max_mpls_depth);
> +                              ofproto->backer->max_mpls_depth,
> +                              ofproto_get_n_visible_tables(&ofproto->up));
>  
>              HMAP_FOR_EACH (bundle, hmap_node, &ofproto->bundles) {
>                  xlate_bundle_set(ofproto, bundle, bundle->name,
> @@ -822,6 +832,7 @@ struct odp_garbage {
>  
>  static bool check_variable_length_userdata(struct dpif_backer *backer);
>  static size_t check_max_mpls_depth(struct dpif_backer *backer);
> +static bool check_recirc(struct dpif_backer *backer);
>  
>  static int
>  open_dpif_backer(const char *type, struct dpif_backer **backerp)
> @@ -835,6 +846,7 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
>      struct sset names;
>      char *backer_name;
>      const char *name;
> +    int recirc_id_size;
>      int error;
>  
>      backer = shash_find_data(&all_dpif_backers, type);
> @@ -921,19 +933,71 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
>          close_dpif_backer(backer);
>          return error;
>      }
> +    backer->enable_recirc = check_recirc(backer);
>      backer->variable_length_userdata = check_variable_length_userdata(backer);
>      backer->max_mpls_depth = check_max_mpls_depth(backer);
>  
> +    backer->enable_recirc = false;  /* XXX Do not enable recirc yet */
> +
>      if (backer->recv_set_enable) {
>          udpif_set_threads(backer->udpif, n_handlers, n_revalidators);
>      }
>  
>      ovs_mutex_init(&backer->recirc_id_lock);
> -    recirc_id_set_init(&backer->rids, 0, 0);
> +    recirc_id_size = (backer->enable_recirc) ? RECIRC_ID_SIZE : 0;
> +    recirc_id_set_init(&backer->rids, RECIRC_ID_BASE, recirc_id_size);
>  
>      return error;
>  }
>  
> +/* Tests whether 'backer''s datapath supports recirculation
> + * Only newer datapath supports OVS_KEY_ATTR in OVS_ACTION_ATTR_USERSPACE actions.  We need
> + * to disable some features on older datapaths that don't support this
> + * feature.
> + *
> + * Returns false if 'backer' definitely does not support variable-length
> + * userdata, true if it seems to support them or if at least the error we get
> + * is ambiguous. */
> +static bool
> +check_recirc(struct dpif_backer *backer)
> +{
> +    struct flow flow;
> +    struct odputil_keybuf keybuf;
> +    struct ofpbuf key;
> +    int error;
> +    bool enable_recirc = false;
> +
> +    memset(&flow, 0, sizeof flow);
> +    flow.recirc_id = 1;
> +    flow.dp_hash = 1;
> +
> +    ofpbuf_use_stack(&key, &keybuf, sizeof keybuf);
> +    odp_flow_key_from_flow(&key, &flow, 0, true);
> +
> +    error = dpif_flow_put(backer->dpif, DPIF_FP_CREATE | DPIF_FP_MODIFY,
> +                          key.data, key.size, NULL, 0, NULL, 0, NULL);
> +    if (error && error != EEXIST) {
> +        if (error != EINVAL) {
> +            VLOG_WARN("%s: Reciculation flow probe failed (%s)",
> +                      dpif_name(backer->dpif), ovs_strerror(error));
> +        }
> +        goto done;
> +    }
> +
> +    error = dpif_flow_del(backer->dpif, key.data, key.size, NULL);
> +    if (error) {
> +        VLOG_WARN("%s: failed to delete recirculation feature probe flow",
> +                  dpif_name(backer->dpif));
> +    }
> +
> +    enable_recirc = true;
> +
> +done:
> +    VLOG_INFO("%s: Recirculation is %s", dpif_name(backer->dpif),
> +              enable_recirc ? "enabled" : "not enabled");
> +    return enable_recirc;
> +}
> +
>  /* Tests whether 'backer''s datapath supports variable-length
>   * OVS_USERSPACE_ATTR_USERDATA in OVS_ACTION_ATTR_USERSPACE actions.  We need
>   * to disable some features on older datapaths that don't support this
> @@ -1033,7 +1097,7 @@ check_max_mpls_depth(struct dpif_backer *backer)
>          flow_set_mpls_bos(&flow, n, 1);
>  
>          ofpbuf_use_stack(&key, &keybuf, sizeof keybuf);
> -        odp_flow_key_from_flow(&key, &flow, 0);
> +        odp_flow_key_from_flow(&key, &flow, 0, false);
>  
>          error = dpif_flow_put(backer->dpif, DPIF_FP_CREATE | DPIF_FP_MODIFY,
>                                key.data, key.size, NULL, 0, NULL, 0, NULL);
> @@ -1117,50 +1181,25 @@ construct(struct ofproto *ofproto_)
>  
>      ofproto_init_tables(ofproto_, N_TABLES);
>      error = add_internal_flows(ofproto);
> -    ofproto->up.tables[TBL_INTERNAL].flags = OFTABLE_HIDDEN | OFTABLE_READONLY;
> +
> +    ofproto->up.tables[TBL_INTERNAL].flags = OFTABLE_HIDDEN;
>  
>      return error;
>  }
>  
>  static int
> -add_internal_flow(struct ofproto_dpif *ofproto, int id,
> +add_internal_miss_flow(struct ofproto_dpif *ofproto, int id,
>                    const struct ofpbuf *ofpacts, struct rule_dpif **rulep)
>  {
> -    struct ofputil_flow_mod fm;
> +    struct match match;
>      int error;
>  
> -    match_init_catchall(&fm.match);
> -    fm.priority = 0;
> -    match_set_reg(&fm.match, 0, id);
> -    fm.new_cookie = htonll(0);
> -    fm.cookie = htonll(0);
> -    fm.cookie_mask = htonll(0);
> -    fm.modify_cookie = false;
> -    fm.table_id = TBL_INTERNAL;
> -    fm.command = OFPFC_ADD;
> -    fm.idle_timeout = 0;
> -    fm.hard_timeout = 0;
> -    fm.buffer_id = 0;
> -    fm.out_port = 0;
> -    fm.flags = 0;
> -    fm.ofpacts = ofpacts->data;
> -    fm.ofpacts_len = ofpacts->size;
> -
> -    error = ofproto_flow_mod(&ofproto->up, &fm);
> -    if (error) {
> -        VLOG_ERR_RL(&rl, "failed to add internal flow %d (%s)",
> -                    id, ofperr_to_string(error));
> -        return error;
> -    }
> +    match_init_catchall(&match);
> +    match_set_reg(&match, 0, id);
>  
> -    if (rule_dpif_lookup_in_table(ofproto, &fm.match.flow, NULL, TBL_INTERNAL,
> -                                  rulep)) {
> -        rule_dpif_unref(*rulep);
> -    } else {
> -        OVS_NOT_REACHED();
> -    }
> +    error = ofproto_dpif_add_internal_flow(ofproto, &match, 0, ofpacts, rulep);
>  
> -    return 0;
> +    return error;
>  }
>  
>  static int
> @@ -1181,20 +1220,42 @@ add_internal_flows(struct ofproto_dpif *ofproto)
>      controller->reason = OFPR_NO_MATCH;
>      ofpact_pad(&ofpacts);
>  
> -    error = add_internal_flow(ofproto, id++, &ofpacts, &ofproto->miss_rule);
> +    error = add_internal_miss_flow(ofproto, id++, &ofpacts, &ofproto->miss_rule);
>      if (error) {
>          return error;
>      }
>  
>      ofpbuf_clear(&ofpacts);
> -    error = add_internal_flow(ofproto, id++, &ofpacts,
> +    error = add_internal_miss_flow(ofproto, id++, &ofpacts,
>                                &ofproto->no_packet_in_rule);
>      if (error) {
>          return error;
>      }
>  
> -    error = add_internal_flow(ofproto, id++, &ofpacts,
> +    error = add_internal_miss_flow(ofproto, id++, &ofpacts,
>                                &ofproto->drop_frags_rule);
> +    if (error) {
> +        return error;
> +    }
> +
> +    {
> +        struct ofpact_resubmit *resubmit;
> +        struct match match;
> +        struct rule_dpif *resubmit_rule OVS_UNUSED;
> +
> +        /* Continue non-recirculation rule lookups from table 0:
> +         *
> +         * (priority=1), *, actions=resubmit(, 0)
> +         */
> +        resubmit = ofpact_put_RESUBMIT(&ofpacts);
> +        resubmit->ofpact.compat = OFPUTIL_NXAST_RESUBMIT_TABLE;
> +        resubmit->in_port = OFPP_IN_PORT;
> +        resubmit->table_id = 0;
> +        match_init_catchall(&match);
> +        error = ofproto_dpif_add_internal_flow(ofproto, &match, 1,  &ofpacts,
> +                                               &resubmit_rule);
> +    }
> +
>      return error;
>  }
>  
> @@ -1262,6 +1323,7 @@ run(struct ofproto *ofproto_)
>  {
>      struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
>      uint64_t new_seq, new_dump_seq;
> +    const bool enable_recirc = ofproto_dpif_get_enable_recirc(ofproto);
>  
>      if (mbridge_need_revalidate(ofproto->mbridge)) {
>          ofproto->backer->need_revalidate = REV_RECONFIGURE;
> @@ -1339,12 +1401,17 @@ run(struct ofproto *ofproto_)
>  
>          /* All outstanding data in existing flows has been accounted, so it's a
>           * good time to do bond rebalancing. */
> -        if (ofproto->has_bonded_bundles) {
> +        if (enable_recirc && ofproto->has_bonded_bundles) {
>              struct ofbundle *bundle;
>  
>              HMAP_FOR_EACH (bundle, hmap_node, &ofproto->bundles) {
> -                if (bundle->bond) {
> -                    bond_rebalance(bundle->bond);
> +                struct bond *bond = bundle->bond;
> +
> +                if (bond && bond_may_recirc(bond, NULL, NULL)) {
> +                    bond_recirculation_account(bond);
> +                    if (bond_rebalance(bundle->bond)) {
> +                        bond_update_post_recirc_rules(bond);
> +                    }
>                  }
>              }
>          }
> @@ -2357,12 +2424,13 @@ bundle_set(struct ofproto *ofproto_, void *aux,
>                  ofproto->backer->need_revalidate = REV_RECONFIGURE;
>              }
>          } else {
> -            bundle->bond = bond_create(s->bond);
> +            bundle->bond = bond_create(s->bond, ofproto);
>              ofproto->backer->need_revalidate = REV_RECONFIGURE;
>          }
>  
>          LIST_FOR_EACH (port, bundle_node, &bundle->ports) {
> -            bond_slave_register(bundle->bond, port, port->up.netdev);
> +            bond_slave_register(bundle->bond, port,
> +                                port->up.ofp_port, port->up.netdev);
>          }
>      } else {
>          bond_unref(bundle->bond);
> @@ -3001,17 +3069,21 @@ ofproto_dpif_execute_actions(struct ofproto_dpif *ofproto,
>      struct xlate_out xout;
>      struct xlate_in xin;
>      ofp_port_t in_port;
> +    uint8_t rule_table_id = 0;
>      struct dpif_execute execute;
>      int error;
>  
>      ovs_assert((rule != NULL) != (ofpacts != NULL));
>  
>      dpif_flow_stats_extract(flow, packet, time_msec(), &stats);
> +
>      if (rule) {
>          rule_dpif_credit_stats(rule, &stats);
> +        rule_table_id = rule->up.table_id;
>      }
>  
> -    xlate_in_init(&xin, ofproto, flow, rule, stats.tcp_flags, packet);
> +    xlate_in_init(&xin, ofproto, flow, rule, rule_table_id,
> +                  stats.tcp_flags, packet);
>      xin.ofpacts = ofpacts;
>      xin.ofpacts_len = ofpacts_len;
>      xin.resubmit_stats = &stats;
> @@ -3087,11 +3159,26 @@ rule_dpif_get_actions(const struct rule_dpif *rule)
>   * the fields that were relevant as part of the lookup. */
>  void
>  rule_dpif_lookup(struct ofproto_dpif *ofproto, const struct flow *flow,
> -                 struct flow_wildcards *wc, struct rule_dpif **rule)
> +                 struct flow_wildcards *wc, struct rule_dpif **rule,
> +                 uint8_t *table_id)
>  {
>      struct ofport_dpif *port;
>  
> -    if (rule_dpif_lookup_in_table(ofproto, flow, wc, 0, rule)) {
> +    if (table_id) {
> +        *table_id = TBL_INTERNAL;
> +    }
> +
> +    if (flow->recirc_id) {
> +        struct flow recirc_flow = *flow;
> +
> +        /* Reflect recirc_id into metadata to match with inernal
> +         * rules.
> +         */
> +        recirc_flow.metadata = htonll(flow->recirc_id);
> +        flow = &recirc_flow;
> +    }
> +
> +    if (rule_dpif_lookup_in_table(ofproto, flow, wc, TBL_INTERNAL, rule)) {
>          return;
>      }
>      port = get_ofp_port(ofproto, flow->in_port.ofp_port);
> @@ -3968,6 +4055,7 @@ ofproto_trace(struct ofproto_dpif *ofproto, const struct flow *flow,
>                struct ds *ds)
>  {
>      struct rule_dpif *rule;
> +    uint8_t table_id;
>      struct trace_ctx trace;
>  
>      ds_put_format(ds, "Bridge: %s\n", ofproto->up.name);
> @@ -3979,7 +4067,7 @@ ofproto_trace(struct ofproto_dpif *ofproto, const struct flow *flow,
>      if (ofpacts) {
>          rule = NULL;
>      } else {
> -        rule_dpif_lookup(ofproto, flow, &trace.wc, &rule);
> +        rule_dpif_lookup(ofproto, flow, &trace.wc, &rule, &table_id);
>  
>          trace_format_rule(ds, 0, rule);
>          if (rule == ofproto->miss_rule) {
> @@ -3999,7 +4087,7 @@ ofproto_trace(struct ofproto_dpif *ofproto, const struct flow *flow,
>          tcp_flags = packet ? packet_get_tcp_flags(packet, flow) : 0;
>          trace.result = ds;
>          trace.flow = *flow;
> -        xlate_in_init(&trace.xin, ofproto, flow, rule, tcp_flags, packet);
> +        xlate_in_init(&trace.xin, ofproto, flow, rule, table_id, tcp_flags, packet);
>          if (ofpacts) {
>              trace.xin.ofpacts = ofpacts;
>              trace.xin.ofpacts_len = ofpacts_len;
> @@ -4308,7 +4396,7 @@ set_realdev(struct ofport *ofport_, ofp_port_t realdev_ofp_port, int vid)
>      if (realdev_ofp_port && ofport->bundle) {
>          /* vlandevs are enslaved to their realdevs, so they are not allowed to
>           * themselves be part of a bundle. */
> -        bundle_set(ofport->up.ofproto, ofport->bundle, NULL);
> +        bundle_set(ofport_->ofproto, ofport->bundle, NULL);
>      }
>  
>      ofport->realdev_ofp_port = realdev_ofp_port;
> @@ -4600,9 +4688,8 @@ recirc_id_free(struct recirc_id_set *set, uint32_t recirc_id)
>  }
>  
>  uint32_t
> -ofproto_dpif_alloc_recirc_id(struct ofproto *ofproto_)
> +ofproto_dpif_alloc_recirc_id(struct ofproto_dpif *ofproto)
>  {
> -    struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
>      struct dpif_backer *backer = ofproto->backer;
>      uint32_t id;
>  
> @@ -4614,9 +4701,8 @@ ofproto_dpif_alloc_recirc_id(struct ofproto *ofproto_)
>  }
>  
>  void
> -ofproto_dpif_free_recirc_id(struct ofproto *ofproto_, uint32_t recirc_id)
> +ofproto_dpif_free_recirc_id(struct ofproto_dpif *ofproto, uint32_t recirc_id)
>  {
> -    struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
>      struct dpif_backer *backer = ofproto->backer;
>  
>      ovs_mutex_lock(&backer->recirc_id_lock);
> @@ -4662,6 +4748,75 @@ odp_port_to_ofp_port(const struct ofproto_dpif *ofproto, odp_port_t odp_port)
>      }
>  }
>  
> +int
> +ofproto_dpif_add_internal_flow(struct ofproto_dpif *ofproto,
> +                               struct match *match, int priority,
> +                               const struct ofpbuf *ofpacts,
> +                               struct rule_dpif **rulep)
> +{
> +    struct ofputil_flow_mod fm;
> +    int error;
> +
> +    fm.match = *match;
> +    fm.priority = priority;
> +    fm.new_cookie = htonll(0);
> +    fm.cookie = htonll(0);
> +    fm.cookie_mask = htonll(0);
> +    fm.modify_cookie = false;
> +    fm.table_id = TBL_INTERNAL;
> +    fm.command = OFPFC_ADD;
> +    fm.idle_timeout = 0;
> +    fm.hard_timeout = 0;
> +    fm.buffer_id = 0;
> +    fm.out_port = 0;
> +    fm.flags = 0;
> +    fm.ofpacts = ofpacts->data;
> +    fm.ofpacts_len = ofpacts->size;
> +
> +    error = ofproto_flow_mod(&ofproto->up, &fm);
> +    if (error) {
> +        VLOG_ERR_RL(&rl, "failed to add internal flow (%s)",
> +                    ofperr_to_string(error));
> +        *rulep = NULL;
> +        return error;
> +    }
> +
> +    if (rule_dpif_lookup_in_table(ofproto, &match->flow, NULL, TBL_INTERNAL,
> +                                  rulep)) {
> +        rule_dpif_unref(*rulep);
> +    } else {
> +        OVS_NOT_REACHED();
> +    }
> +    return 0;
> +}
> +
> +
> +int
> +ofproto_dpif_delete_internal_flow(struct ofproto_dpif *ofproto,
> +                               struct match *match, int priority)
> +{
> +    struct ofputil_flow_mod fm;
> +    int error;
> +
> +    fm.match = *match;
> +    fm.priority = priority;
> +    fm.new_cookie = htonll(0);
> +    fm.cookie = htonll(0);
> +    fm.cookie_mask = htonll(0);
> +    fm.modify_cookie = false;
> +    fm.table_id = TBL_INTERNAL;
> +    fm.command = OFPFC_DELETE_STRICT;
> +
> +    error = ofproto_flow_mod(&ofproto->up, &fm);
> +    if (error) {
> +        VLOG_ERR_RL(&rl, "failed to delete internal flow (%s)",
> +                    ofperr_to_string(error));
> +        return error;
> +    }
> +
> +    return 0;
> +}
> +
>  const struct ofproto_class ofproto_dpif_class = {
>      init,
>      enumerate_types,
> diff --git a/ofproto/ofproto-dpif.h b/ofproto/ofproto-dpif.h
> index f020c7d..fc5d031 100644
> --- a/ofproto/ofproto-dpif.h
> +++ b/ofproto/ofproto-dpif.h
> @@ -67,9 +67,10 @@ extern struct ovs_rwlock xlate_rwlock;
>   *   actions into datapath actions. */
>  
>  size_t ofproto_dpif_get_max_mpls_depth(const struct ofproto_dpif *);
> +bool ofproto_dpif_get_enable_recirc(const struct ofproto_dpif *);
>  
>  void rule_dpif_lookup(struct ofproto_dpif *, const struct flow *,
> -                      struct flow_wildcards *, struct rule_dpif **rule);
> +                      struct flow_wildcards *, struct rule_dpif **rule, uint8_t *table_id);
>  
>  bool rule_dpif_lookup_in_table(struct ofproto_dpif *, const struct flow *,
>                                 struct flow_wildcards *, uint8_t table_id,
> @@ -121,7 +122,6 @@ int ofproto_dpif_send_packet(const struct ofport_dpif *, struct ofpbuf *);
>  void ofproto_dpif_flow_mod(struct ofproto_dpif *, struct ofputil_flow_mod *);
>  
>  struct ofport_dpif *odp_port_to_ofport(const struct dpif_backer *, odp_port_t);
> -
>  /*
>  Recirculation
>  =============
> @@ -178,8 +178,13 @@ They are created on demand. Miss handling, stats collection and revalidation
>  work the same way as regular flows.
>  */
>  
> -uint32_t ofproto_dpif_alloc_recirc_id(struct ofproto *ofproto);
> -void ofproto_dpif_free_recirc_id(struct ofproto *ofproto, uint32_t recirc_id);
> -
> +uint32_t ofproto_dpif_alloc_recirc_id(struct ofproto_dpif *ofproto);
> +void ofproto_dpif_free_recirc_id(struct ofproto_dpif *ofproto, uint32_t recirc_id);
> +int ofproto_dpif_add_internal_flow(struct ofproto_dpif *ofproto,
> +                                   struct match *match, int proiroty,
> +                                   const struct ofpbuf *ofpacts,
> +                                   struct rule_dpif **rulep);
> +int ofproto_dpif_delete_internal_flow(struct ofproto_dpif *ofproto, struct match *match,
> +                      int proiroty);
>  
>  #endif /* ofproto-dpif.h */
> diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c
> index 19e7091..d8526ec 100644
> --- a/ofproto/ofproto.c
> +++ b/ofproto/ofproto.c
> @@ -1133,6 +1133,24 @@ ofproto_get_n_tables(const struct ofproto *ofproto)
>      return ofproto->n_tables;
>  }
>  
> +/* Returns the number of Controller visible OpenFlow tables
> + * in 'ofproto'. This number will exclude Hidden tables.
> + * This funtion's return value should be less or equal to that of
> + * ofproto_get_n_tables() . */
> +uint8_t
> +ofproto_get_n_visible_tables(const struct ofproto *ofproto)
> +{
> +    uint8_t n = ofproto->n_tables;
> +
> +    /* Count only non-hidden tables in the number of tables.  (Hidden tables,
> +     * if present, are always at the end.) */
> +    while(n && (ofproto->tables[n - 1].flags & OFTABLE_HIDDEN)) {
> +        n--;
> +    }
> +
> +    return n;
> +}
> +
>  /* Configures the OpenFlow table in 'ofproto' with id 'table_id' with the
>   * settings from 's'.  'table_id' must be in the range 0 through the number of
>   * OpenFlow tables in 'ofproto' minus 1, inclusive.
> @@ -2738,6 +2756,13 @@ destroy_rule_executes(struct ofproto *ofproto)
>      }
>  }
>  
> +
> +static enum oftable_flags
> +rule_get_flags(const struct rule *rule)
> +{
> +    return rule->ofproto->tables[rule->table_id].flags;
> +}
> +
>  /* Returns true if 'rule' should be hidden from the controller.
>   *
>   * Rules with priority higher than UINT16_MAX are set up by ofproto itself
> @@ -2746,13 +2771,7 @@ destroy_rule_executes(struct ofproto *ofproto)
>  static bool
>  ofproto_rule_is_hidden(const struct rule *rule)
>  {
> -    return rule->cr.priority > UINT16_MAX;
> -}
> -
> -static enum oftable_flags
> -rule_get_flags(const struct rule *rule)
> -{
> -    return rule->ofproto->tables[rule->table_id].flags;
> +    return (rule->cr.priority > UINT16_MAX);
>  }
>  
>  static bool
> @@ -2776,26 +2795,14 @@ handle_features_request(struct ofconn *ofconn, const struct ofp_header *oh)
>      struct ofport *port;
>      bool arp_match_ip;
>      struct ofpbuf *b;
> -    int n_tables;
> -    int i;
>  
>      ofproto->ofproto_class->get_features(ofproto, &arp_match_ip,
>                                           &features.actions);
>      ovs_assert(features.actions & OFPUTIL_A_OUTPUT); /* sanity check */
>  
> -    /* Count only non-hidden tables in the number of tables.  (Hidden tables,
> -     * if present, are always at the end.) */
> -    n_tables = ofproto->n_tables;
> -    for (i = 0; i < ofproto->n_tables; i++) {
> -        if (ofproto->tables[i].flags & OFTABLE_HIDDEN) {
> -            n_tables = i;
> -            break;
> -        }
> -    }
> -
>      features.datapath_id = ofproto->datapath_id;
>      features.n_buffers = pktbuf_capacity();
> -    features.n_tables = n_tables;
> +    features.n_tables = ofproto_get_n_visible_tables(ofproto);
>      features.capabilities = (OFPUTIL_C_FLOW_STATS | OFPUTIL_C_TABLE_STATS |
>                               OFPUTIL_C_PORT_STATS | OFPUTIL_C_QUEUE_STATS);
>      if (arp_match_ip) {
> diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h
> index 1f9cb15..1f0e03f 100644
> --- a/ofproto/ofproto.h
> +++ b/ofproto/ofproto.h
> @@ -380,6 +380,7 @@ struct ofproto_table_settings {
>  };
>  
>  int ofproto_get_n_tables(const struct ofproto *);
> +uint8_t ofproto_get_n_visible_tables(const struct ofproto *);
>  void ofproto_configure_table(struct ofproto *, int table_id,
>                               const struct ofproto_table_settings *);
>  
> diff --git a/tests/lacp.at b/tests/lacp.at
> index d44bee0..0db2077 100644
> --- a/tests/lacp.at
> +++ b/tests/lacp.at
> @@ -1,5 +1,10 @@
>  AT_BANNER([lacp])
>  
> +# Strips out Reciulation ID information since it may change over time.
> +m4_define([STRIP_RECIRC_ID], [[sed '
> +    s/Recirc-ID.*$/<del>/
> +' ]])
> +
>  AT_SETUP([lacp - config])
>  OVS_VSWITCHD_START([\
>          add-port br0 p1 --\
> @@ -113,6 +118,7 @@ slave: p2: expired attached
>  AT_CHECK([ovs-appctl bond/show], [0], [dnl
>  ---- bond ----
>  bond_mode: active-backup
> +bond may use recirculation: no, Recirc-ID : -1
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> @@ -182,8 +188,8 @@ done
>  AT_CHECK(
>    [ovs-appctl lacp/show bond0
>  ovs-appctl lacp/show bond1
> -ovs-appctl bond/show bond0
> -ovs-appctl bond/show bond1], [0], [stdout])
> +ovs-appctl bond/show bond0 | STRIP_RECIRC_ID
> +ovs-appctl bond/show bond1 | STRIP_RECIRC_ID ], [0], [stdout])
>  AT_CHECK([sed '/active slave/d' stdout], [0], [dnl
>  ---- bond0 ----
>  	status: active negotiated
> @@ -275,6 +281,7 @@ slave: p3: current attached
>  	partner state: activity timeout aggregation synchronized collecting distributing
>  ---- bond0 ----
>  bond_mode: balance-tcp
> +bond may use recirculation: yes, <del>
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> @@ -288,6 +295,7 @@ slave p1: enabled
>  
>  ---- bond1 ----
>  bond_mode: balance-tcp
> +bond may use recirculation: yes, <del>
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> @@ -316,8 +324,8 @@ for i in `seq 0 40`; do ovs-appctl time/warp 100; done
>  AT_CHECK(
>    [ovs-appctl lacp/show bond0
>  ovs-appctl lacp/show bond1
> -ovs-appctl bond/show bond0
> -ovs-appctl bond/show bond1], [0], [dnl
> +ovs-appctl bond/show bond0 | STRIP_RECIRC_ID
> +ovs-appctl bond/show bond1 | STRIP_RECIRC_ID ], [0], [dnl
>  ---- bond0 ----
>  	status: active negotiated
>  	sys_id: aa:55:aa:55:00:00
> @@ -408,6 +416,7 @@ slave: p3: current attached
>  	partner state: activity timeout aggregation synchronized collecting distributing
>  ---- bond0 ----
>  bond_mode: balance-tcp
> +bond may use recirculation: yes, <del>
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> @@ -422,6 +431,7 @@ slave p1: enabled
>  
>  ---- bond1 ----
>  bond_mode: balance-tcp
> +bond may use recirculation: yes, <del>
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> @@ -442,8 +452,8 @@ for i in `seq 0 40`; do ovs-appctl time/warp 100; done
>  AT_CHECK(
>    [ovs-appctl lacp/show bond0
>  ovs-appctl lacp/show bond1
> -ovs-appctl bond/show bond0
> -ovs-appctl bond/show bond1], [0], [dnl
> +ovs-appctl bond/show bond0 | STRIP_RECIRC_ID
> +ovs-appctl bond/show bond1 | STRIP_RECIRC_ID ], [0], [dnl
>  ---- bond0 ----
>  	status: active negotiated
>  	sys_id: aa:55:aa:55:00:00
> @@ -534,6 +544,7 @@ slave: p3: current attached
>  	partner state: activity timeout aggregation synchronized collecting distributing
>  ---- bond0 ----
>  bond_mode: balance-tcp
> +bond may use recirculation: yes, <del>
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> @@ -548,6 +559,7 @@ slave p1: enabled
>  
>  ---- bond1 ----
>  bond_mode: balance-tcp
> +bond may use recirculation: yes, <del>
>  bond-hash-basis: 0
>  updelay: 0 ms
>  downdelay: 0 ms
> diff --git a/tests/ofproto-dpif.at b/tests/ofproto-dpif.at
> index ad23e9d..2b79d33 100644
> --- a/tests/ofproto-dpif.at
> +++ b/tests/ofproto-dpif.at
> @@ -36,17 +36,106 @@ ovs-appctl time/warp 100
>  sleep 1  # wait for forwarders process packets
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3/0.0.0.0,dst=10.0.0.4/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3/0.0.0.0,dst=10.0.0.4/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows br1 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(2),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(8),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3/0.0.0.0,dst=10.0.0.4/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(2),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(8),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3/0.0.0.0,dst=10.0.0.4/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
>  
> +AT_SETUP([ofproto-dpif, active-backup bonding])
> +# Create br0 with interfaces p1, p2 and p7, creating bond0 with p1 and p2
> +#    and br1 with interfaces p3, p4 and p8.
> +# toggle p1,p2 of bond0 up and down to test bonding in active-backup mode.
> +OVS_VSWITCHD_START(
> +  [add-bond br0 bond0 p1 p2 bond_mode=active-backup --\
> +   set interface p1 type=dummy options:pstream=punix:$OVS_RUNDIR/p1.sock ofport_request=1 -- \
> +   set interface p2 type=dummy options:pstream=punix:$OVS_RUNDIR/p2.sock ofport_request=2 -- \
> +   add-port br0 p7 -- set interface p7 ofport_request=7 type=dummy -- \
> +   add-br br1 -- \
> +   set bridge br1 other-config:hwaddr=aa:66:aa:66:00:00 -- \
> +   set bridge br1 datapath-type=dummy other-config:datapath-id=1234 \
> +                  fail-mode=secure -- \
> +   add-port br1 p3 -- set interface p3 type=dummy options:stream=unix:$OVS_RUNDIR/p1.sock ofport_request=3 -- \
> +   add-port br1 p4 -- set interface p4 type=dummy options:stream=unix:$OVS_RUNDIR/p2.sock ofport_request=4 -- \
> +   add-port br1 p8 -- set interface p8 ofport_request=8 type=dummy --])
> +
> +AT_CHECK([ovs-ofctl add-flow br0 action=normal])
> +AT_CHECK([ovs-ofctl add-flow br1 action=normal])
> +ovs-appctl netdev-dummy/set-admin-state up
> +ovs-appctl time/warp 100
> +ovs-appctl netdev-dummy/set-admin-state p2 down
> +ovs-appctl time/stop
> +ovs-appctl time/warp 100
> +AT_CHECK([ovs-appctl netdev-dummy/receive p7 'in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
> +AT_CHECK([ovs-appctl netdev-dummy/receive p7 'in_port(7),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3,dst=10.0.0.4,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
> +ovs-appctl time/warp 100
> +ovs-appctl netdev-dummy/set-admin-state p2 up
> +ovs-appctl netdev-dummy/set-admin-state p1 down
> +ovs-appctl time/warp 100
> +AT_CHECK([ovs-appctl netdev-dummy/receive p7 'in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0d),eth_type(0x0800),ipv4(src=10.0.0.5,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
> +AT_CHECK([ovs-appctl netdev-dummy/receive p7 'in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0e),eth_type(0x0800),ipv4(src=10.0.0.6,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
> +ovs-appctl time/warp 100
> +ovs-appctl time/warp 100
> +AT_CHECK([ovs-appctl dpif/dump-flows br1 | STRIP_XOUT], [0], [dnl
> +skb_priority(0),recirc_id(0),in_port(3),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(3),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.3/0.0.0.0,dst=10.0.0.4/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(4),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0d),eth_type(0x0800),ipv4(src=10.0.0.5/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(4),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0e),eth_type(0x0800),ipv4(src=10.0.0.6/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(4),eth(src=50:54:00:00:00:09,dst=ff:ff:ff:ff:ff:ff),eth_type(0x8035), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(4),eth(src=50:54:00:00:00:0b,dst=ff:ff:ff:ff:ff:ff),eth_type(0x8035), packets:0, bytes:0, used:never, actions: <del>
> +])
> +OVS_VSWITCHD_STOP
> +AT_CLEANUP
> +
> +AT_SETUP([ofproto-dpif, balance-slb bonding])
> +# Create br0 with interfaces bond0(p1, p2, p3) and p7,
> +#    and br1 with interfaces p4, p5, p6 and p8.
> +#    p1 <-> p4, p2 <-> p5, p3 <-> p6
> +# Send some traffic, make sure the traffic are spread based on source mac.
> +OVS_VSWITCHD_START(
> +  [add-bond br0 bond0 p1 p2 p3 bond_mode=balance-slb --\
> +   set interface p1 type=dummy options:pstream=punix:$OVS_RUNDIR/p1.sock ofport_request=1 -- \
> +   set interface p2 type=dummy options:pstream=punix:$OVS_RUNDIR/p2.sock ofport_request=2 -- \
> +   set interface p3 type=dummy options:pstream=punix:$OVS_RUNDIR/p3.sock ofport_request=3 -- \
> +   add-port br0 p7 -- set interface p7 ofport_request=7 type=dummy -- \
> +   add-br br1 -- \
> +   set bridge br1 other-config:hwaddr=aa:66:aa:66:00:00 -- \
> +   set bridge br1 datapath-type=dummy other-config:datapath-id=1234 \
> +                  fail-mode=secure -- \
> +   add-port br1 p4 -- set interface p4 type=dummy options:stream=unix:$OVS_RUNDIR/p1.sock ofport_request=4 -- \
> +   add-port br1 p5 -- set interface p5 type=dummy options:stream=unix:$OVS_RUNDIR/p2.sock ofport_request=5 -- \
> +   add-port br1 p6 -- set interface p6 type=dummy options:stream=unix:$OVS_RUNDIR/p3.sock ofport_request=6 -- \
> +   add-port br1 p8 -- set interface p8 ofport_request=8 type=dummy --])
> +
> +AT_CHECK([ovs-ofctl add-flow br0 action=normal])
> +AT_CHECK([ovs-ofctl add-flow br1 action=normal])
> +AT_CHECK([ovs-appctl netdev-dummy/set-admin-state up], 0, [OK
> +])
> +ovs-appctl netdev-dummy/set-admin-state up
> +ovs-appctl time/stop
> +ovs-appctl time/warp 100
> +(
> +for i in `seq 0 100 |xargs printf '%02x\n'`;
> +    do
> +    pkt="in_port(7),eth(src=50:54:00:00:00:$i,dst=50:54:00:00:01:00),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)"
> +    AT_CHECK([ovs-appctl netdev-dummy/receive p7 $pkt])
> +    done
> +)
> +ovs-appctl time/warp 100
> +AT_CHECK([ovs-appctl dpif/dump-flows br1 > br1_flows.txt])
> +# Make sure there is resonable distribution to all three ports.
> +# We don't want to make this check precise, in case hash function changes.
> +AT_CHECK([test `egrep 'in_port\(4\)' br1_flows.txt |wc -l` -gt 3])
> +AT_CHECK([test `egrep 'in_port\(5\)' br1_flows.txt |wc -l` -gt 3])
> +AT_CHECK([test `egrep 'in_port\(6\)' br1_flows.txt |wc -l` -gt 3])
> +OVS_VSWITCHD_STOP
> +AT_CLEANUP
> +
>  AT_SETUP([ofproto-dpif - resubmit])
>  OVS_VSWITCHD_START
>  ADD_OF_PORTS([br0], [1], [10], [11], [12], [13], [14], [15],
> @@ -2996,21 +3085,21 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p2 'in_port(2),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p3 'in_port(3),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | sort | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=192.168.0.1/0.0.0.0,dst=192.168.0.2/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> -skb_priority(0),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.2/0.0.0.0,dst=192.168.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=192.168.0.1/0.0.0.0,dst=192.168.0.2/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),recirc_id(0),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.2/0.0.0.0,dst=192.168.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
>  ])
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows br1 | sort | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(3),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),recirc_id(0),in_port(3),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
>  ])
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows -m br0 | sort | STRIP_USED], [0], [dnl
> -skb_priority(0),skb_mark(0/0),in_port(p1),eth(src=50:54:00:00:00:05/00:00:00:00:00:00,dst=50:54:00:00:00:07/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=192.168.0.1/0.0.0.0,dst=192.168.0.2/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8/0,code=0/0), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> -skb_priority(0),skb_mark(0/0),in_port(p2),eth(src=50:54:00:00:00:07/00:00:00:00:00:00,dst=50:54:00:00:00:05/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=192.168.0.2/0.0.0.0,dst=192.168.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=0/0,code=0/0), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),skb_mark(0/0),recirc_id(0),in_port(p1),eth(src=50:54:00:00:00:05/00:00:00:00:00:00,dst=50:54:00:00:00:07/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=192.168.0.1/0.0.0.0,dst=192.168.0.2/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8/0,code=0/0), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),skb_mark(0/0),recirc_id(0),in_port(p2),eth(src=50:54:00:00:00:07/00:00:00:00:00:00,dst=50:54:00:00:00:05/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=192.168.0.2/0.0.0.0,dst=192.168.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=0/0,code=0/0), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
>  ])
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows -m br1 | sort | STRIP_USED], [0], [dnl
> -skb_priority(0),skb_mark(0/0),in_port(p3),eth(src=50:54:00:00:00:09/00:00:00:00:00:00,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8/0,code=0/0), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),skb_mark(0/0),recirc_id(0),in_port(p3),eth(src=50:54:00:00:00:09/00:00:00:00:00:00,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8/0,code=0/0), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
>  ])
>  
>  OVS_VSWITCHD_STOP
> @@ -3042,7 +3131,7 @@ for dl_src in 00 01; do
>      AT_CHECK([ovs-appctl netdev-dummy/receive p1 "505400000007 6066666666$dl_src 8847 00014020 00014120 45 00 00 2c 00 00 00 00 40 06 3b 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45"])
>  
>      AT_CHECK_UNQUOTED([ovs-appctl dpif/dump-flows br0 | grep ":$dl_src/" | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=60:66:66:66:66:$dl_src/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:07/00:00:00:00:00:00),eth_type(0x8847),mpls(lse0=0x14020,lse1=0x14120), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=60:66:66:66:66:$dl_src/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:07/00:00:00:00:00:00),eth_type(0x8847),mpls(lse0=0x14020,lse1=0x14120), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
>  ])
>  done
>  
> @@ -3079,7 +3168,7 @@ for dl_src in 00 01; do
>      AT_CHECK([ovs-appctl netdev-dummy/receive p1 "505400000007 6066666666$dl_src 8847 00014020 00014120 45 00 00 2c 00 00 00 00 40 06 3b 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45"])
>  
>      AT_CHECK_UNQUOTED([ovs-appctl dpif/dump-flows br0 | grep ":$dl_src/" | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=60:66:66:66:66:$dl_src/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:07/00:00:00:00:00:00),eth_type(0x8847),mpls(lse0=0x14020,lse1=0x14120), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=60:66:66:66:66:$dl_src/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:07/00:00:00:00:00:00),eth_type(0x8847),mpls(lse0=0x14020,lse1=0x14120), packets:0, bytes:0, used:never, actions:userspace(pid=0,slow_path(controller))
>  ])
>  done
>  
> @@ -3131,10 +3220,10 @@ dummy at ovs-dummy: hit:13 missed:2
>  ])
>  
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(100),eth_type(0x0800),ipv4(src=192.168.0.1/0.0.0.0,dst=192.168.0.2/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:9, bytes:540, used:0.0s, actions:101,3,2
> +skb_priority(0),recirc_id(0),in_port(100),eth_type(0x0800),ipv4(src=192.168.0.1/0.0.0.0,dst=192.168.0.2/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:9, bytes:540, used:0.0s, actions:101,3,2
>  ]),
>  AT_CHECK([ovs-appctl dpif/dump-flows br1 | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(101),eth_type(0x0800),ipv4(src=192.168.0.2/0.0.0.0,dst=192.168.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:4, bytes:240, used:0.0s, actions:100,2,3
> +skb_priority(0),recirc_id(0),in_port(101),eth_type(0x0800),ipv4(src=192.168.0.2/0.0.0.0,dst=192.168.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:4, bytes:240, used:0.0s, actions:100,2,3
>  ])
>  
>  AT_CHECK([ovs-ofctl dump-ports br0 pbr0], [0], [dnl
> @@ -3183,7 +3272,7 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3199,8 +3288,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3217,8 +3306,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.252,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/0.0.0.0,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.252,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/0.0.0.0,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3235,8 +3324,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x86dd),ipv6(src=2001:db8:3c4d:5:4:3:2:1,dst=2001:db8:3c4d:1:2:3:4:1,label=0,proto=99,tclass=0x70,hlimit=64,frag=no)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x86dd),ipv6(src=2001:db8:3c4d:1:2:3:4:5/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff,dst=fe80::2/::,label=0/0,proto=10/0,tclass=0x70/0,hlimit=128/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth_type(0x86dd),ipv6(src=2001:db8:3c4d:5:4:3:2:1/ffff:ffff:ffff:fffc::,dst=2001:db8:3c4d:1:2:3:4:1/::,label=0/0,proto=99/0,tclass=0x70/0,hlimit=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x86dd),ipv6(src=2001:db8:3c4d:1:2:3:4:5/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff,dst=fe80::2/::,label=0/0,proto=10/0,tclass=0x70/0,hlimit=128/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x86dd),ipv6(src=2001:db8:3c4d:5:4:3:2:1/ffff:ffff:ffff:fffc::,dst=2001:db8:3c4d:1:2:3:4:1/::,label=0/0,proto=99/0,tclass=0x70/0,hlimit=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3251,7 +3340,7 @@ AT_CHECK([ovs-ofctl add-flows br0 flows.txt])
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8/0xff,code=0/0), packets:0, bytes:0, used:0.0s, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8/0xff,code=0/0), packets:0, bytes:0, used:0.0s, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3264,8 +3353,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3282,8 +3371,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0a),eth_type(0x8847),mpls(label=11,tc=3,ttl=64,bos=1)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x8847),mpls(label=11,tc=3,ttl=64,bos=1), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x8847),mpls(label=11,tc=3,ttl=64,bos=1), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x8847),mpls(label=11,tc=3,ttl=64,bos=1), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x8847),mpls(label=11,tc=3,ttl=64,bos=1), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3311,8 +3400,8 @@ m4_define([CHECK_MEGAFLOW_NETFLOW],
>    AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>    sleep 1
>    AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/255.255.255.255,proto=1/0xff,tos=0/0xfc,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/255.255.255.255,proto=1/0xff,tos=0/0xfc,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/255.255.255.255,proto=1/0xff,tos=0/0xfc,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/255.255.255.255,proto=1/0xff,tos=0/0xfc,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
>  ])
>    OVS_VSWITCHD_STOP
>    AT_CLEANUP])
> @@ -3334,8 +3423,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3354,8 +3443,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3389,10 +3478,10 @@ ovs-appctl time/stop
>  ovs-appctl time/warp 5000
>  AT_CHECK([ovs-appctl netdev-dummy/receive p7 'in_port(1),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  AT_CHECK([ovs-appctl netdev-dummy/receive p7 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
> -sleep 1
> +
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/255.255.255.255,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(7),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/255.255.255.255,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(7),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/255.255.255.255,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(7),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/255.255.255.255,proto=1/0xff,tos=0/0,ttl=64/0,frag=no/0xff),icmp(type=8,code=0), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3409,8 +3498,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3428,8 +3517,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3446,8 +3535,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3468,7 +3557,7 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3489,8 +3578,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth_type(0x8100),vlan(vid=11/0xfff,pcp=7/0x0,cfi=1/1),encap(eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff)), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x8100),vlan(vid=11/0xfff,pcp=7/0x0,cfi=1/1),encap(eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff)), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3508,8 +3597,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3525,8 +3614,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.255,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4/255.255.255.255,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3551,8 +3640,8 @@ done
>  sleep 1
>  dnl The original flow is missing due to a revalidation.
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0a/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b/ff:ff:ff:ff:ff:ff,dst=50:54:00:00:00:0c/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3580,9 +3669,9 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p3 'in_port(3),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p3 'in_port(3),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0x1,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0xfd/0x3,ttl=128/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> -skb_priority(0),in_port(3),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0xfd/0xff,ttl=128/0xff,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(3),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0x1/0xff,ttl=64/0xff,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0xfd/0x3,ttl=128/0,frag=no/0xff), packets:0, bytes:0, used:0.0s, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(3),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0xfd/0xff,ttl=128/0xff,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(3),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0x1/0xff,ttl=64/0xff,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3599,8 +3688,8 @@ AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:
>  AT_CHECK([ovs-appctl netdev-dummy/receive p1 'in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0)'])
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_XOUT], [0], [dnl
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.252,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> -skb_priority(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.2/255.255.255.252,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions: <del>
> +skb_priority(0),recirc_id(0),in_port(1),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no), packets:0, bytes:0, used:never, actions: <del>
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3626,8 +3715,8 @@ dnl MAC, so both the source and destination MAC addresses are
>  dnl un-wildcarded, since the ODP commit functions update both the source
>  dnl and destination MAC addresses.
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_USED], [0], [dnl
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:09/00:00:00:00:00:00,dst=50:54:00:00:00:0a/ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:2
> -skb_priority(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:set(eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0a)),2
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:09/00:00:00:00:00:00,dst=50:54:00:00:00:0a/ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(src=10.0.0.2/0.0.0.0,dst=10.0.0.1/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:2
> +skb_priority(0),recirc_id(0),in_port(1),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4/0.0.0.0,dst=10.0.0.3/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no/0xff), packets:0, bytes:0, used:never, actions:set(eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0a)),2
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> @@ -3649,8 +3738,8 @@ for i in 1 2 3 4; do
>  done
>  sleep 1
>  AT_CHECK([ovs-appctl dpif/dump-flows br0 | STRIP_USED], [0], [dnl
> -skb_priority(0),skb_mark(0),in_port(1/0xffff),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0), packets:3, bytes:180, used:0.0s, actions:2
> -skb_priority(0),skb_mark(0),in_port(1/0xffff),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0), packets:3, bytes:180, used:0.0s, actions:drop
> +skb_priority(0),skb_mark(0),recirc_id(0),in_port(1/0xffff),eth(src=50:54:00:00:00:09,dst=50:54:00:00:00:0a),eth_type(0x0800),ipv4(src=10.0.0.2,dst=10.0.0.1,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0), packets:3, bytes:180, used:0.0s, actions:2
> +skb_priority(0),skb_mark(0),recirc_id(0),in_port(1/0xffff),eth(src=50:54:00:00:00:0b,dst=50:54:00:00:00:0c),eth_type(0x0800),ipv4(src=10.0.0.4,dst=10.0.0.3,proto=1,tos=0,ttl=64,frag=no),icmp(type=8,code=0), packets:3, bytes:180, used:0.0s, actions:drop
>  ])
>  OVS_VSWITCHD_STOP
>  AT_CLEANUP
> diff --git a/tests/test-odp.c b/tests/test-odp.c
> index a27bf7f..6c12494 100644
> --- a/tests/test-odp.c
> +++ b/tests/test-odp.c
> @@ -75,7 +75,8 @@ parse_keys(bool wc_keys)
>              /* Convert cls_rule back to odp_key. */
>              ofpbuf_uninit(&odp_key);
>              ofpbuf_init(&odp_key, 0);
> -            odp_flow_key_from_flow(&odp_key, &flow, flow.in_port.odp_port);
> +            odp_flow_key_from_flow(&odp_key, &flow, flow.in_port.odp_port,
> +                                   false);
>  
>              if (odp_key.size > ODPUTIL_FLOW_KEY_BYTES) {
>                  printf ("too long: %"PRIuSIZE" > %d\n",
> -- 
> 1.7.9.5
> 
> _______________________________________________
> dev mailing list
> dev at openvswitch.org
> http://openvswitch.org/mailman/listinfo/dev
> 



More information about the dev mailing list