[ovs-dev] [PATCH 7/7] ofproto-dpif: Restore metadata and registers on recirculation.

Jarno Rajahalme jrajahalme at nicira.com
Tue Feb 24 23:42:06 UTC 2015


xlate_actions() now takes an optional recirclulation context as a
parameter and restores OpenFlow pipeline metadata based on it.  The
recirculation context is returned by the xlate_lookup() and may
contain an action set to be restored, and further actions to be
executed upon recirculation.

The pipeline metadata is restored using two new internal actions.  The
UNROLL_METADATA action stores the non-packet data in struct flow.
This is used to restore metadata right after recirculation, and also
after returning from a patch port, if the patch port recirculated.
This is needed as pipeline metadata restored for recirculation in this
case belongs to a different bridge than the bridge that initially
received the packet.

The UNROLL_XLATE action stores the translation context data visible to
OpenFlow controllers via PACKET_IN messages.  This includes the
current table number and the current rule cookie.  UNROLL_XLATE
actions are always inserted right after the UNROLL_METADATA actions,
but also whenever returning back from recursive translation when
recirculating.

These changes allow the post-MPLS recirculation to properly continue
with the pipeline metadata that existed at the time of recirculation.
The internal table is still consulted for bonds.

Signed-off-by: Jarno Rajahalme <jrajahalme at nicira.com>Summary:
Signed-off-by: Jarno Rajahalme <jrajahalme at nicira.com>
---
 lib/ofp-actions.c             |   64 +++++++
 lib/ofp-actions.h             |   53 ++++++
 ofproto/bond.c                |    7 +-
 ofproto/ofproto-dpif-rid.c    |  343 ++++++++++++++++++++++++++++++----
 ofproto/ofproto-dpif-rid.h    |  121 ++++++++++--
 ofproto/ofproto-dpif-upcall.c |  120 +++++++++---
 ofproto/ofproto-dpif-xlate.c  |  408 +++++++++++++++++++++++++++--------------
 ofproto/ofproto-dpif-xlate.h  |   72 +++++++-
 ofproto/ofproto-dpif.c        |  165 +++++------------
 ofproto/ofproto-dpif.h        |   79 +-------
 tests/mpls-xlate.at           |   16 +-
 tests/ofproto-dpif.at         |  203 ++++++++++----------
 12 files changed, 1129 insertions(+), 522 deletions(-)

diff --git a/lib/ofp-actions.c b/lib/ofp-actions.c
index e694fd9..b8f6f40 100644
--- a/lib/ofp-actions.c
+++ b/lib/ofp-actions.c
@@ -4237,6 +4237,56 @@ format_EXIT(const struct ofpact_null *a OVS_UNUSED, struct ds *s)
     ds_put_cstr(s, "exit");
 }
 
+/* Unroll metadata action. */
+
+static void
+encode_UNROLL_METADATA(const struct ofpact_unroll_metadata *unroll OVS_UNUSED,
+                       enum ofp_version ofp_version OVS_UNUSED,
+                       struct ofpbuf *out OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+}
+
+static char * OVS_WARN_UNUSED_RESULT
+parse_UNROLL_METADATA(char *arg OVS_UNUSED, struct ofpbuf *ofpacts OVS_UNUSED,
+                      enum ofputil_protocol *usable_protocols OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+    return NULL;
+}
+
+static void
+format_UNROLL_METADATA(const struct ofpact_unroll_metadata *a OVS_UNUSED,
+                       struct ds *s)
+{
+    ds_put_cstr(s, "unroll_metadata");
+}
+
+/* Unroll xlate action. */
+
+static void
+encode_UNROLL_XLATE(const struct ofpact_unroll_xlate *unroll OVS_UNUSED,
+                    enum ofp_version ofp_version OVS_UNUSED,
+                    struct ofpbuf *out OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+}
+
+static char * OVS_WARN_UNUSED_RESULT
+parse_UNROLL_XLATE(char *arg OVS_UNUSED, struct ofpbuf *ofpacts OVS_UNUSED,
+                   enum ofputil_protocol *usable_protocols OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+    return NULL;
+}
+
+static void
+format_UNROLL_XLATE(const struct ofpact_unroll_xlate *a OVS_UNUSED,
+                    struct ds *s)
+{
+    ds_put_cstr(s, "unroll_xlate");
+}
+
 /* Action structure for NXAST_SAMPLE.
  *
  * Samples matching packets with the given probability and sends them
@@ -4726,6 +4776,8 @@ ofpact_is_set_or_move_action(const struct ofpact *a)
     case OFPACT_DEC_TTL:
     case OFPACT_ENQUEUE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_FIN_TIMEOUT:
     case OFPACT_GOTO_TABLE:
     case OFPACT_GROUP:
@@ -4795,6 +4847,8 @@ ofpact_is_allowed_in_actions_set(const struct ofpact *a)
     case OFPACT_CONTROLLER:
     case OFPACT_ENQUEUE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_FIN_TIMEOUT:
     case OFPACT_LEARN:
     case OFPACT_CONJUNCTION:
@@ -5017,6 +5071,8 @@ ovs_instruction_type_from_ofpact_type(enum ofpact_type type)
     case OFPACT_MULTIPATH:
     case OFPACT_NOTE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_SAMPLE:
     default:
         return OVSINST_OFPIT11_APPLY_ACTIONS;
@@ -5607,6 +5663,12 @@ ofpact_check__(enum ofputil_protocol *usable_protocols, struct ofpact *a,
     case OFPACT_GROUP:
         return 0;
 
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
+        /* UNROLL is an internal action that should never be seen via
+         * OpenFlow. */
+        return OFPERR_OFPBAC_BAD_TYPE;
+
     default:
         OVS_NOT_REACHED();
     }
@@ -5998,6 +6060,8 @@ ofpact_outputs_to_port(const struct ofpact *ofpact, ofp_port_t port)
     case OFPACT_MULTIPATH:
     case OFPACT_NOTE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_PUSH_MPLS:
     case OFPACT_POP_MPLS:
     case OFPACT_SAMPLE:
diff --git a/lib/ofp-actions.h b/lib/ofp-actions.h
index a1a5bb1..929ffc1 100644
--- a/lib/ofp-actions.h
+++ b/lib/ofp-actions.h
@@ -105,6 +105,8 @@
     OFPACT(NOTE,            ofpact_note,        data, "note")           \
     OFPACT(EXIT,            ofpact_null,        ofpact, "exit")         \
     OFPACT(SAMPLE,          ofpact_sample,      ofpact, "sample")       \
+    OFPACT(UNROLL_METADATA, ofpact_unroll_metadata, ofpact, "unroll_metadata") \
+    OFPACT(UNROLL_XLATE,    ofpact_unroll_xlate, ofpact, "unroll_xlate") \
                                                                         \
     /* Instructions. */                                                 \
     OFPACT(METER,           ofpact_meter,       ofpact, "meter")        \
@@ -715,6 +717,57 @@ struct ofpact_group {
     uint32_t group_id;
 };
 
+/* OFPACT_UNROLL_METADATA.
+ *
+ * Used only internally.  Update when new metadata is added.  Helpers
+ * are inlined below to keep them together with the definition for easier
+ * updates. */
+BUILD_ASSERT_DECL(FLOW_WC_SEQ == 31);
+
+struct ofpact_unroll_metadata {
+    struct ofpact ofpact;
+
+    /* Metadata in struct flow. */
+    struct flow_tnl tunnel;       /* Encapsulating tunnel parameters. */
+    ovs_be64 metadata;            /* OpenFlow Metadata. */
+    uint64_t regs[FLOW_N_XREGS];  /* Registers. */
+    ofp_port_t in_port;           /* Incoming port. */
+    ofp_port_t actset_output;     /* Output port in action set. */
+};
+
+static inline void
+ofpact_unroll_metadata_from_flow(struct ofpact_unroll_metadata *unroll,
+                                 const struct flow *flow)
+{
+    unroll->tunnel = flow->tunnel;
+    unroll->metadata = flow->metadata;
+    memcpy(unroll->regs, flow->regs, sizeof unroll->regs);
+    unroll->in_port = flow->in_port.ofp_port;
+    unroll->actset_output = flow->actset_output;
+}
+
+static inline void
+ofpact_unroll_metadata_to_flow(const struct ofpact_unroll_metadata *unroll,
+                               struct flow *flow)
+{
+    flow->tunnel = unroll->tunnel;
+    flow->metadata = unroll->metadata;
+    memcpy(flow->regs, unroll->regs, sizeof flow->regs);
+    flow->in_port.ofp_port = unroll->in_port;
+    flow->actset_output = unroll->actset_output;
+}
+
+/* OFPACT_UNROLL_XLATE.
+ *
+ * Used only internally. */
+struct ofpact_unroll_xlate {
+    struct ofpact ofpact;
+
+    /* Metadata in xlate context, visible to controller via PACKET_INs. */
+    uint8_t  rule_table_id;       /* 0xFF if none. */
+    ovs_be64 rule_cookie;         /* OVS_BE64_MAX if none. */
+};
+
 /* Converting OpenFlow to ofpacts. */
 enum ofperr ofpacts_pull_openflow_actions(struct ofpbuf *openflow,
                                           unsigned int actions_len,
diff --git a/ofproto/bond.c b/ofproto/bond.c
index 89979a0..ba91b7a 100644
--- a/ofproto/bond.c
+++ b/ofproto/bond.c
@@ -28,6 +28,7 @@
 #include "ofpbuf.h"
 #include "ofproto/ofproto-provider.h"
 #include "ofproto/ofproto-dpif.h"
+#include "ofproto/ofproto-dpif-rid.h"
 #include "connectivity.h"
 #include "coverage.h"
 #include "dynamic-string.h"
@@ -288,7 +289,7 @@ bond_unref(struct bond *bond)
     hmap_destroy(&bond->pr_rule_ops);
 
     if (bond->recirc_id) {
-        ofproto_dpif_free_recirc_id(bond->ofproto, bond->recirc_id);
+        recirc_free_id(bond->recirc_id);
     }
 
     free(bond);
@@ -445,10 +446,10 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s)
 
     if (bond->balance != BM_AB) {
         if (!bond->recirc_id) {
-            bond->recirc_id = ofproto_dpif_alloc_recirc_id(bond->ofproto);
+            bond->recirc_id = recirc_alloc_id(bond->ofproto);
         }
     } else if (bond->recirc_id) {
-        ofproto_dpif_free_recirc_id(bond->ofproto, bond->recirc_id);
+        recirc_free_id(bond->recirc_id);
         bond->recirc_id = 0;
     }
 
diff --git a/ofproto/ofproto-dpif-rid.c b/ofproto/ofproto-dpif-rid.c
index afad3ce..db69d8f 100644
--- a/ofproto/ofproto-dpif-rid.c
+++ b/ofproto/ofproto-dpif-rid.c
@@ -16,59 +16,338 @@
 
 #include <config.h>
 
-#include "id-pool.h"
-#include "ovs-thread.h"
 #include "ofproto-dpif-rid.h"
+#include "ofproto-provider.h"
+#include "openvswitch/vlog.h"
 
-struct recirc_id_pool {
-    struct ovs_mutex lock;
-    struct id_pool *rids;
-};
+VLOG_DEFINE_THIS_MODULE(ofproto_dpif_rid);
 
-#define RECIRC_ID_BASE  300
-#define RECIRC_ID_N_IDS  1024
+static struct ovs_mutex mutex;
 
-struct recirc_id_pool *
-recirc_id_pool_create(void)
+static struct cmap id_map;
+static struct cmap metadata_map;
+
+static struct ovs_list expiring;
+static struct ovs_list expired;
+
+static uint32_t next_id; /* Possible next free id. */
+
+#define RECIRC_POOL_STATIC_IDS 1024
+
+void
+recirc_init()
 {
-    struct recirc_id_pool *pool;
+    static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
 
-    pool = xmalloc(sizeof *pool);
-    pool->rids = id_pool_create(RECIRC_ID_BASE, RECIRC_ID_N_IDS);
-    ovs_mutex_init(&pool->lock);
+    if (ovsthread_once_start(&once)) {
+        next_id = 1; /* 0 is not a valid ID. */
+        ovs_mutex_init(&mutex);
+        cmap_init(&id_map);
+        cmap_init(&metadata_map);
+        list_init(&expiring);
+        list_init(&expired);
+
+        ovsthread_once_done(&once);
+    }
 
-    return pool;
 }
 
+/* This should be called by the revalidator once at each round (every 500ms or
+ * more). */
 void
-recirc_id_pool_destroy(struct recirc_id_pool *pool)
+recirc_run(void)
 {
-    id_pool_destroy(pool->rids);
-    ovs_mutex_destroy(&pool->lock);
-    free(pool);
+    static long long int last = 0;
+    long long int now = time_msec();
+
+    /* Do maintenance at most 4 times / sec. */
+    ovs_mutex_lock(&mutex);
+    if (now - last > 250) {
+        struct recirc_id_node *node, *next;
+
+        last = now;
+
+        /* Nodes in 'expiring' and 'expired' lists have the refcount of zero,
+         * which means that while they can still be found (by id), no new
+         * references can be taken on them.  We have removed the entry from the
+         * 'metadata_map', at the time when refcount reached zero, causing any
+         * new translations to allocate a new ID.  This allows the expiring
+         * entry to be safely deleted while any sudden new use of the similar
+         * recirculation will safely start using a new recirculation ID.  When
+         * the refcount gets to zero, the node is also added to the 'expiring'
+         * list.  At any time after that the nodes in the 'expiring' list can
+         * be moved to the 'expired' list, from which they are deleted at least
+         * 250ms afterwards. */
+
+        /* Delete the expired.  These have been lingering for at least 250 ms,
+         * which should be enough for any ongoing recirculations to be
+         * finished. */
+        LIST_FOR_EACH_SAFE (node, next, exp_node, &expired) {
+            list_remove(&node->exp_node);
+            cmap_remove(&id_map, &node->id_node, node->id);
+            ovsrcu_postpone(free, node);
+        }
+
+        if (!list_is_empty(&expiring)) {
+            /* 'expired' is now empty, move nodes in 'expiring' to it. */
+            list_splice(&expired, list_front(&expiring), &expiring);
+        }
+    }
+    ovs_mutex_unlock(&mutex);
+}
+
+/* We use the id as the hash value, which works due to cmap internal rehashing.
+ * We also only insert nodes with unique IDs, so all possible hash collisions
+ * remain internal to the cmap. */
+static struct recirc_id_node *
+recirc_find__(uint32_t id)
+{
+    struct cmap_node *node = cmap_find_protected(&id_map, id);
+
+    return node ? CONTAINER_OF(node, struct recirc_id_node, id_node) : NULL;
+}
+
+/* Lockless RCU protected lookup.  If node is needed accross RCU quiescent
+ * state, caller should copy the contents. */
+const struct recirc_id_node *
+recirc_id_node_find(uint32_t id)
+{
+    const struct cmap_node *node = cmap_find(&id_map, id);
+
+    return node
+        ? CONTAINER_OF(node, const struct recirc_id_node, id_node)
+        : NULL;
 }
 
+ofp_port_t
+recirc_id_node_get_in_port(const struct recirc_id_node *n)
+{
+    const struct ofpact *a;
+    const struct ofpact_unroll_metadata *unroll;
+
+    if (n->ofpacts_len >= n->action_set_len + sizeof *unroll) {
+        a = (const struct ofpact *)((const char *)n->ofpacts
+                                    + n->action_set_len);
+        if (a->type == OFPACT_UNROLL_METADATA) {
+            unroll = ofpact_get_UNROLL_METADATA(a);
+            return unroll->in_port;
+        }
+    }
+    return OFPP_NONE;
+}
+
+static uint32_t
+recirc_metadata_hash(const struct ofproto_dpif *ofproto,
+                     uint32_t action_set_len, uint32_t ofpacts_len,
+                     const struct ofpact *ofpacts)
+{
+    uint32_t hash;
+
+    hash = hash_pointer(ofproto, 0);
+    if (ofpacts_len) {
+        BUILD_ASSERT(OFPACT_ALIGNTO == sizeof(uint64_t));
+
+        hash = hash_add(hash, action_set_len);
+        hash = hash_words64((const uint64_t *)ofpacts,
+                            OFPACT_ALIGN(ofpacts_len) / OFPACT_ALIGNTO, hash);
+    }
+    return hash;
+}
+
+static bool
+recirc_metadata_equal(const struct recirc_id_node *node,
+                      struct ofproto_dpif *ofproto,
+                      uint32_t action_set_len, uint32_t ofpacts_len,
+                      const struct ofpact *ofpacts)
+{
+    return node->ofproto == ofproto
+        && node->action_set_len == action_set_len
+        && node->ofpacts_len == ofpacts_len
+        && !memcmp(node->ofpacts, ofpacts, ofpacts_len);
+}
+
+/* Lockless RCU protected lookup.  If node is needed accross RCU quiescent
+ * state, caller should take a reference. */
+static struct recirc_id_node *
+recirc_find_equal(struct ofproto_dpif *ofproto, uint32_t action_set_len,
+                  uint32_t ofpacts_len, const struct ofpact *ofpacts,
+                  uint32_t hash)
+{
+    struct recirc_id_node *node;
+
+    CMAP_FOR_EACH_WITH_HASH(node, metadata_node, hash, &metadata_map) {
+        if (recirc_metadata_equal(node, ofproto, action_set_len, ofpacts_len,
+                                  ofpacts)) {
+            return node;
+        }
+    }
+    return NULL;
+}
+
+static struct recirc_id_node *
+recirc_ref_equal(struct ofproto_dpif *ofproto, uint32_t action_set_len,
+                 uint32_t ofpacts_len, const struct ofpact *ofpacts,
+                 uint32_t hash)
+{
+    struct recirc_id_node *node;
+
+    do {
+        node = recirc_find_equal(ofproto, action_set_len, ofpacts_len, ofpacts,
+                                 hash);
+
+        /* Try again if the node was released before we get the reference. */
+    } while (node && !ovs_refcount_try_ref_rcu(&node->refcount));
+
+    return node;
+}
+
+/* Allocate a unique recirculation id for the given set of flow metadata.
+ * The ID space is 2^^32, so there should never be a situation in which all
+ * the IDs are used up.  We loop until we find a free one.
+ * hash is recomputed if it is passed in as 0. */
+static struct recirc_id_node *
+recirc_alloc_id__(struct ofproto_dpif *ofproto, uint32_t action_set_len,
+                  uint32_t ofpacts_len, const struct ofpact *ofpacts,
+                  uint32_t hash)
+{
+    struct recirc_id_node *node = xzalloc(sizeof *node +
+                                          OFPACT_ALIGN(ofpacts_len));
+
+    /* All the data must be initialized before the node is made visible to
+     * lookups. */
+    ovs_refcount_init(&node->refcount);
+    node->ofproto = ofproto;
+    if (ofpacts_len) {
+        node->action_set_len = action_set_len;
+        node->ofpacts_len = ofpacts_len;
+        memcpy(node->ofpacts, ofpacts, ofpacts_len);
+    }
+    node->hash = hash;
+    ovs_mutex_lock(&mutex);
+    for (;;) {
+        /* Claim the next ID.  The ID space should be sparse enough for the
+           allocation to succeed at the first try.  We do skip the first
+           RECIRC_POOL_STATIC_IDS IDs on the later rounds, though, as some of
+           the initial allocations may be for long term uses (like bonds). */
+        node->id = next_id++;
+        if (OVS_UNLIKELY(!node->id)) {
+            next_id = RECIRC_POOL_STATIC_IDS + 1;
+            node->id = next_id++;
+        }
+        /* Find if the id is free. */
+        if (OVS_LIKELY(!recirc_find__(node->id))) {
+            break;
+        }
+    }
+    cmap_insert(&id_map, &node->id_node, node->id);
+    cmap_insert(&metadata_map, &node->metadata_node, node->hash);
+    ovs_mutex_unlock(&mutex);
+    return node;
+}
+
+/* Look up an existing ID for the given flow's metadata and optional actions.
+ */
+uint32_t
+recirc_find_id(struct ofproto_dpif *ofproto, uint32_t action_set_len,
+               uint32_t ofpacts_len, const struct ofpact *ofpacts)
+{
+    /* Check if an ID with the given metadata already exists. */
+    struct recirc_id_node *node;
+    uint32_t hash;
+
+    hash = recirc_metadata_hash(ofproto, action_set_len, ofpacts_len, ofpacts);
+    node = recirc_find_equal(ofproto, action_set_len, ofpacts_len, ofpacts,
+                             hash);
+
+    return node ? node->id : 0;
+}
+
+/* Allocate a unique recirculation id for the given set of flow metadata and
+   optional actions. */
+uint32_t
+recirc_alloc_id_ctx(struct ofproto_dpif *ofproto, uint32_t action_set_len,
+                    uint32_t ofpacts_len, const struct ofpact *ofpacts)
+{
+    struct recirc_id_node *node;
+    uint32_t hash;
+
+    /* Look up an existing ID. */
+    hash = recirc_metadata_hash(ofproto, action_set_len, ofpacts_len, ofpacts);
+    node = recirc_ref_equal(ofproto, action_set_len, ofpacts_len, ofpacts,
+                            hash);
+
+    /* Allocate a new recirc ID if needed. */
+    if (!node) {
+        ovs_assert(action_set_len <= ofpacts_len);
+
+        node = recirc_alloc_id__(ofproto, action_set_len, ofpacts_len, ofpacts,
+                                 hash);
+    }
+
+    return node->id;
+}
+
+/* Allocate a unique recirculation id. */
 uint32_t
-recirc_id_alloc(struct recirc_id_pool *pool)
+recirc_alloc_id(struct ofproto_dpif *ofproto)
 {
-    uint32_t id;
-    bool ret;
+    struct flow flow;
+    struct recirc_id_node *node;
+    uint32_t hash;
+
+    memset(&flow, 0, sizeof flow);
+
+    hash = recirc_metadata_hash(ofproto, 0, 0, NULL);
+    node = recirc_alloc_id__(ofproto, 0, 0, NULL, hash);
+
+    return node->id;
+}
 
-    ovs_mutex_lock(&pool->lock);
-    ret = id_pool_alloc_id(pool->rids, &id);
-    ovs_mutex_unlock(&pool->lock);
+void
+recirc_id_node_unref(const struct recirc_id_node *node_)
+    OVS_EXCLUDED(mutex)
+{
+    struct recirc_id_node *node = CONST_CAST(struct recirc_id_node *, node_);
 
-    if (!ret) {
-        return 0;
+    if (node && ovs_refcount_unref(&node->refcount) == 1) {
+        ovs_mutex_lock(&mutex);
+        /* Prevent re-use of this node by removing the node from 'metadata_map'
+         */
+        cmap_remove(&metadata_map, &node->metadata_node, node->hash);
+        /* We keep the node in the 'id_map' so that it can be found as long
+         * as it lingers, and add it to the 'expiring' list. */
+        list_insert(&expiring, &node->exp_node);
+        ovs_mutex_unlock(&mutex);
     }
+}
+
+void
+recirc_free_id(uint32_t id)
+{
+    struct recirc_id_node *node;
 
-    return id;
+    node = recirc_find__(id);
+    if (node) {
+        recirc_id_node_unref(node);
+    } else {
+        VLOG_ERR("Freeing non-exiting recirculation ID: %"PRIu32, id);
+    }
 }
 
+/* Called when 'ofproto' is destructed.  Checks for and clears any
+ * recirc_id leak.
+ * No other thread may have access to the 'ofproto' being destructed.
+ * All related datapath flows must be deleted before calling this. */
 void
-recirc_id_free(struct recirc_id_pool *pool, uint32_t id)
+recirc_free_ofproto(struct ofproto_dpif *ofproto, const char *ofproto_name)
 {
-    ovs_mutex_lock(&pool->lock);
-    id_pool_free_id(pool->rids, id);
-    ovs_mutex_unlock(&pool->lock);
+    struct recirc_id_node *node;
+
+    CMAP_FOR_EACH (node, id_node, &id_map) {
+        if (node->ofproto == ofproto) {
+            VLOG_ERR("recirc_id %"PRIu32" left allocated when ofproto (%s) "
+                     "is destructed", node->id, ofproto_name);
+            recirc_id_node_unref(node);
+        }
+    }
 }
diff --git a/ofproto/ofproto-dpif-rid.h b/ofproto/ofproto-dpif-rid.h
index 3344e2a..75c61d6 100644
--- a/ofproto/ofproto-dpif-rid.h
+++ b/ofproto/ofproto-dpif-rid.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Nicira, Inc.
+ * Copyright (c) 2014, 2015 Nicira, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,29 +20,120 @@
 #include <stddef.h>
 #include <stdint.h>
 
-struct recirc_id_pool;
+#include "cmap.h"
+#include "list.h"
+#include "ofp-actions.h"
+#include "ovs-thread.h"
+
+struct ofproto_dpif;
+struct rule;
 
 /*
- * Recirculation ID pool.
- * ======================
+ * Recirculation
+ * =============
+ *
+ * Recirculation is a technique to allow a frame to re-enter the packet
+ * processing path for one or multiple times to achieve more flexible packet
+ * processing in the data path, such modifying header fields after MPLS POP
+ * action and selecting bond a slave port for bond ports.
+ *
+ * Data path and user space interface
+ * -----------------------------------
+ *
+ * Two new fields, recirc_id and dp_hash, are added to the current flow data
+ * structure. They are both of type uint32_t. In addition, a new action,
+ * RECIRC, is added.
+ *
+ * The value recirc_id is used to distinguish a packet from multiple
+ * iterations of recirculation. A packet initially received is considered of
+ * having recirc_id of 0. Recirc_id is managed by the user space, opaque to
+ * the data path.
+ *
+ * On the other hand, dp_hash can only be computed by the data path, opaque to
+ * the user space. In fact, user space may not able to recompute the hash
+ * value. The dp_hash value should be wildcarded when for a newly received
+ * packet. HASH action specifies whether the hash is computed, and if
+ * computed, how many fields are to be included in the hash computation. The
+ * computed hash value is stored into the dp_hash field prior to recirculation.
  *
- * Recirculation ID needs to be unique for each datapath. Recirculation
- * ID pool keeps track recirculation ids.
+ * The RECIRC action sets the recirc_id field and then reprocesses the packet
+ * as if it was received on the same input port. RECIRC action works like a
+ * function call; actions listed behind the RECIRC action will be executed
+ * after its execution.  RECIRC action can be nested, data path implementation
+ * limits the number of recirculation executed to prevent unreasonable nesting
+ * depth or infinite loop.
  *
- * Typically, there is one recirculation ID pool for each backer.
+ * Both new flow fields are exposed as OpenFlow fields via Nicira extensions.
  *
- * In theory, Recirculation ID can be any uint32_t value, except 0.
- * The implementation usually limits it to a smaller range to ease
- * debugging.
+ * Post recirculation flow
+ * ------------------------
+ *
+ * Recirculation is hidden from the OpenFlow controllers.  All actions to be
+ * peformed after recirculation are derived from the OpenFlow pipeline and are
+ * stored with the recirculation ID.  When the OpenFlow tables are changed in a
+ * way affecting the recirculation flows, new recirculation ID with new
+ * metadata and actions is allocated and the old one is timed out.
+ *
+ * Recirculation ID pool
+ * ----------------------
+ *
+ * Recirculation ID needs to be unique for all datapaths. Recirculation ID pool
+ * keeps track recirculation ids and stores OpenFlow pipeline translation
+ * context so that flow processing may continue after recirculation.
+ *
+ * A Recirculation ID can be any uint32_t value, except 0.
  *
  * Thread-safety
- * =============
+ * -------------
  *
  * All APIs are thread safe.
  *
  */
-struct recirc_id_pool *recirc_id_pool_create(void);
-void  recirc_id_pool_destroy(struct recirc_id_pool *pool);
-uint32_t recirc_id_alloc(struct recirc_id_pool *pool);
-void recirc_id_free(struct recirc_id_pool *pool, uint32_t recirc_id);
+
+
+/* Pool node fields should NOT be modified after placing the node in the pool.
+ */
+struct recirc_id_node {
+    struct ovs_list exp_node;
+    struct cmap_node id_node;
+    struct cmap_node metadata_node;
+    uint32_t id;
+    uint32_t hash;
+    struct ovs_refcount refcount;
+
+    /* Stored metadata from the recirculation action. */
+    struct ofproto_dpif *ofproto; /* Which bridge recirculated?. */
+
+    /* Actions to be translated on recirculation. */
+    uint32_t action_set_len;      /* How much of 'ofpacts' consists of an
+                                   * action set? */
+    uint32_t ofpacts_len;         /* Size of 'ofpacts', in bytes. */
+    struct ofpact ofpacts[];      /* Sequence of "struct ofpacts". */
+};
+
+void recirc_init(void);
+
+uint32_t recirc_alloc_id(struct ofproto_dpif *);
+uint32_t recirc_alloc_id_ctx(struct ofproto_dpif *, uint32_t action_set_len,
+                             uint32_t ofpacts_len,
+                             const struct ofpact *ofpacts);
+uint32_t recirc_find_id(struct ofproto_dpif *, uint32_t action_set_len,
+                        uint32_t ofpacts_len, const struct ofpact *ofpacts);
+void recirc_free_id(uint32_t recirc_id);
+void recirc_free_ofproto(struct ofproto_dpif *, const char *ofproto_name);
+
+const struct recirc_id_node *recirc_id_node_find(uint32_t recirc_id);
+ofp_port_t recirc_id_node_get_in_port(const struct recirc_id_node *);
+
+static inline bool recirc_id_node_try_ref_rcu(const struct recirc_id_node *n_)
+{
+    struct recirc_id_node *node = CONST_CAST(struct recirc_id_node *, n_);
+
+    return node ? ovs_refcount_try_ref_rcu(&node->refcount) : false;
+}
+
+void recirc_id_node_unref(const struct recirc_id_node *);
+
+void recirc_run(void);
+
 #endif
diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c
index 97e4588..964efa4 100644
--- a/ofproto/ofproto-dpif-upcall.c
+++ b/ofproto/ofproto-dpif-upcall.c
@@ -152,6 +152,8 @@ enum upcall_type {
 
 struct upcall {
     struct ofproto_dpif *ofproto;  /* Parent ofproto. */
+    const struct recirc_id_node *recirc; /* Recirculation context. */
+    bool have_recirc_ref;                /* Reference held on recirc ctx? */
 
     /* The flow and packet are only required to be constant when using
      * dpif-netdev.  If a modification is absolutely necessary, a const cast
@@ -229,6 +231,10 @@ struct udpif_key {
         struct odputil_keybuf buf;
         struct nlattr nla;
     } keybuf, maskbuf;
+
+    /* Recirculation IDs with references held by the ukey. */
+    unsigned n_recircs;
+    uint32_t recircs[];   /* 'n_recircs' id's for which references are held. */
 };
 
 /* Datapath operation with optional ukey attached. */
@@ -271,7 +277,7 @@ static void upcall_unixctl_dump_wait(struct unixctl_conn *conn, int argc,
 static void upcall_unixctl_purge(struct unixctl_conn *conn, int argc,
                                  const char *argv[], void *aux);
 
-static struct udpif_key *ukey_create_from_upcall(const struct upcall *);
+static struct udpif_key *ukey_create_from_upcall(struct upcall *);
 static int ukey_create_from_dpif_flow(const struct udpif *,
                                       const struct dpif_flow *,
                                       struct udpif_key **);
@@ -738,6 +744,8 @@ udpif_revalidator(void *arg)
         if (leader) {
             uint64_t reval_seq;
 
+            recirc_run(); /* Recirculation cleanup. */
+
             reval_seq = seq_read(udpif->reval_seq);
             last_reval_seq = reval_seq;
 
@@ -899,11 +907,24 @@ upcall_receive(struct upcall *upcall, const struct dpif_backer *backer,
     int error;
 
     error = xlate_lookup(backer, flow, &upcall->ofproto, &upcall->ipfix,
-                         &upcall->sflow, NULL, &upcall->in_port);
+                         &upcall->sflow, NULL, &upcall->in_port,
+                         &upcall->recirc);
     if (error) {
         return error;
     }
 
+    /* Try to get a reference only if we may install a flow based on this
+     * upcall. */
+    upcall->have_recirc_ref = false;
+    if (upcall->recirc && type == DPIF_UC_MISS) {
+        /* We may install a datapath flow only if we get a reference to the
+         * recirculation context (otherwise we could have recirculation upcalls
+         * using recirculation ID for which no context can be found).  We may
+         * still execute the flow's actions even if we don't install the
+         * flow. */
+        upcall->have_recirc_ref = recirc_id_node_try_ref_rcu(upcall->recirc);
+    }
+
     upcall->flow = flow;
     upcall->packet = packet;
     upcall->ufid = ufid;
@@ -944,14 +965,16 @@ upcall_xlate(struct udpif *udpif, struct upcall *upcall,
     if (upcall->type == DPIF_UC_MISS) {
         xin.resubmit_stats = &stats;
     } else {
-        /* For non-miss upcalls, there's a flow in the datapath which this
-         * packet was accounted to.  Presumably the revalidators will deal
+        /* For non-miss upcalls, we are either executing actions (one of which
+         * is an userspace action) for an upcall, in which case the stats have
+         * already been taken care of, or there's a flow in the datapath which
+         * this packet was accounted to.  Presumably the revalidators will deal
          * with pushing its stats eventually. */
     }
 
     upcall->dump_seq = seq_read(udpif->dump_seq);
     upcall->reval_seq = seq_read(udpif->reval_seq);
-    xlate_actions(&xin, &upcall->xout);
+    xlate_actions(&xin, &upcall->xout, upcall->recirc);
     upcall->xout_initialized = true;
 
     /* Special case for fail-open mode.
@@ -1003,8 +1026,13 @@ upcall_uninit(struct upcall *upcall)
             xlate_out_uninit(&upcall->xout);
         }
         ofpbuf_uninit(&upcall->put_actions);
-        if (!upcall->ukey_persists) {
-            ukey_delete__(upcall->ukey);
+        if (upcall->ukey) {
+            if (!upcall->ukey_persists) {
+                ukey_delete__(upcall->ukey);
+            }
+        } else if (upcall->have_recirc_ref) {
+            /* The reference was transferred to the ukey if one was created. */
+            recirc_id_node_unref(upcall->recirc);
         }
     }
 }
@@ -1054,10 +1082,16 @@ upcall_cb(const struct ofpbuf *packet, const struct flow *flow, ovs_u128 *ufid,
         goto out;
     }
 
-    if (upcall.ukey && !ukey_install(udpif, upcall.ukey)) {
+    /* Prevent miss flow installation if the key has recirculation ID but we
+     * were not able to get a reference on it. */
+    if (type == DPIF_UC_MISS && upcall.recirc && !upcall.have_recirc_ref) {
         error = ENOSPC;
+        goto out;
     }
 
+    if (upcall.ukey && !ukey_install(udpif, upcall.ukey)) {
+        error = ENOSPC;
+    }
 out:
     if (!error) {
         upcall.ukey_persists = true;
@@ -1187,8 +1221,12 @@ handle_upcalls(struct udpif *udpif, struct upcall *upcalls,
          *    - The datapath already has too many flows.
          *
          *    - We received this packet via some flow installed in the kernel
-         *      already. */
-        if (may_put && upcall->type == DPIF_UC_MISS) {
+         *      already.
+         *
+         *    - Upcall was a recirculation but we do not have a reference to
+         *      to the recirculation ID. */
+        if (may_put && upcall->type == DPIF_UC_MISS &&
+            (!upcall->recirc || upcall->have_recirc_ref)) {
             struct udpif_key *ukey = upcall->ukey;
 
             upcall->ukey_persists = true;
@@ -1275,10 +1313,13 @@ ukey_create__(const struct nlattr *key, size_t key_len,
               const struct nlattr *mask, size_t mask_len,
               bool ufid_present, const ovs_u128 *ufid,
               const int pmd_id, const struct ofpbuf *actions,
-              uint64_t dump_seq, uint64_t reval_seq, long long int used)
+              uint64_t dump_seq, uint64_t reval_seq, long long int used,
+              const struct recirc_id_node *key_recirc, struct xlate_out *xout)
     OVS_NO_THREAD_SAFETY_ANALYSIS
 {
-    struct udpif_key *ukey = xmalloc(sizeof *ukey);
+    unsigned n_recircs = (key_recirc ? 1 : 0) + (xout ? xout->n_recircs : 0);
+    struct udpif_key *ukey = xmalloc(sizeof *ukey +
+                                     n_recircs * sizeof *ukey->recircs);
 
     memcpy(&ukey->keybuf, key, key_len);
     ukey->key = &ukey->keybuf.nla;
@@ -1301,15 +1342,27 @@ ukey_create__(const struct nlattr *key, size_t key_len,
     ukey->stats.used = used;
     ukey->xcache = NULL;
 
+    ukey->n_recircs = n_recircs;
+    if (key_recirc) {
+        ukey->recircs[0] = key_recirc->id;
+    }
+    if (xout && xout->n_recircs) {
+        const uint32_t *act_recircs = xlate_out_get_recircs(xout);
+
+        memcpy(ukey->recircs + (key_recirc ? 1 : 0), act_recircs,
+               xout->n_recircs * sizeof *ukey->recircs);
+        xlate_out_take_recircs(xout);
+    }
     return ukey;
 }
 
 static struct udpif_key *
-ukey_create_from_upcall(const struct upcall *upcall)
+ukey_create_from_upcall(struct upcall *upcall)
 {
     struct odputil_keybuf keystub, maskstub;
     struct ofpbuf keybuf, maskbuf;
     bool recirc, megaflow;
+    const struct recirc_id_node *key_recirc;
 
     if (upcall->key_len) {
         ofpbuf_use_const(&keybuf, upcall->key, upcall->key_len);
@@ -1332,11 +1385,16 @@ ukey_create_from_upcall(const struct upcall *upcall)
                                UINT32_MAX, max_mpls, recirc);
     }
 
+    key_recirc = NULL;
+    if (upcall->have_recirc_ref) {
+        key_recirc = upcall->recirc;
+    }
+
     return ukey_create__(ofpbuf_data(&keybuf), ofpbuf_size(&keybuf),
                          ofpbuf_data(&maskbuf), ofpbuf_size(&maskbuf),
                          true, upcall->ufid, upcall->pmd_id,
                          &upcall->put_actions, upcall->dump_seq,
-                         upcall->reval_seq, 0);
+                         upcall->reval_seq, 0, key_recirc, &upcall->xout);
 }
 
 static int
@@ -1348,12 +1406,15 @@ ukey_create_from_dpif_flow(const struct udpif *udpif,
     struct ofpbuf actions;
     uint64_t dump_seq, reval_seq;
     uint64_t stub[DPIF_FLOW_BUFSIZE / 8];
+    const struct nlattr *a;
+    unsigned int left;
 
-    if (!flow->key_len) {
+    if (!flow->key_len || !flow->actions_len) {
         struct ofpbuf buf;
         int err;
 
-        /* If the key was not provided by the datapath, fetch the full flow. */
+        /* If the key or actions were not provided by the datapath, fetch the
+         * full flow. */
         ofpbuf_use_stack(&buf, &stub, sizeof stub);
         err = dpif_flow_get(udpif->dpif, NULL, 0, &flow->ufid,
                             flow->pmd_id, &buf, &full_flow);
@@ -1362,13 +1423,23 @@ ukey_create_from_dpif_flow(const struct udpif *udpif,
         }
         flow = &full_flow;
     }
+
+    /* Check the flow actions for recirculation action.  As recirculation
+     * relies on OVS userspace internal state, we need to deleate all old
+     * datapath flows with recirculation upon OVS restart. */
+    NL_ATTR_FOR_EACH_UNSAFE (a, left, flow->actions, flow->actions_len) {
+        if (nl_attr_type(a) == OVS_ACTION_ATTR_RECIRC) {
+            return EINVAL;
+        }
+    }
+
     dump_seq = seq_read(udpif->dump_seq);
     reval_seq = seq_read(udpif->reval_seq);
     ofpbuf_use_const(&actions, &flow->actions, flow->actions_len);
     *ukey = ukey_create__(flow->key, flow->key_len,
                           flow->mask, flow->mask_len, flow->ufid_present,
                           &flow->ufid, flow->pmd_id, &actions, dump_seq,
-                          reval_seq, flow->stats.used);
+                          reval_seq, flow->stats.used, NULL, NULL);
 
     return 0;
 }
@@ -1511,6 +1582,9 @@ ukey_delete__(struct udpif_key *ukey)
     OVS_NO_THREAD_SAFETY_ANALYSIS
 {
     if (ukey) {
+        for (int i = 0; i < ukey->n_recircs; i++) {
+            recirc_free_id(ukey->recircs[i]);
+        }
         xlate_cache_delete(ukey->xcache);
         ofpbuf_delete(ukey->actions);
         ovs_mutex_destroy(&ukey->mutex);
@@ -1579,6 +1653,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey,
     size_t i;
     bool ok;
     bool need_revalidate;
+    const struct recirc_id_node *recirc;
 
     ok = false;
     xoutp = NULL;
@@ -1620,7 +1695,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey,
     }
 
     error = xlate_lookup(udpif->backer, &flow, &ofproto, NULL, NULL, &netflow,
-                         &ofp_in_port);
+                         &ofp_in_port, &recirc);
     if (error) {
         goto exit;
     }
@@ -1640,7 +1715,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey,
     }
     xin.xcache = ukey->xcache;
     xin.skip_wildcards = !need_revalidate;
-    xlate_actions(&xin, &xout);
+    xlate_actions(&xin, &xout, recirc);
     xoutp = &xout;
 
     if (!need_revalidate) {
@@ -1757,6 +1832,7 @@ push_ukey_ops__(struct udpif *udpif, struct ukey_op *ops, size_t n_ops)
             ofp_port_t ofp_in_port;
             struct flow flow;
             int error;
+            const struct recirc_id_node *recirc;
 
             if (op->ukey) {
                 ovs_mutex_lock(&op->ukey->mutex);
@@ -1775,8 +1851,8 @@ push_ukey_ops__(struct udpif *udpif, struct ukey_op *ops, size_t n_ops)
                 continue;
             }
 
-            error = xlate_lookup(udpif->backer, &flow, &ofproto,
-                                 NULL, NULL, &netflow, &ofp_in_port);
+            error = xlate_lookup(udpif->backer, &flow, &ofproto, NULL, NULL,
+                                 &netflow, &ofp_in_port, &recirc);
             if (!error) {
                 struct xlate_in xin;
 
@@ -1785,7 +1861,7 @@ push_ukey_ops__(struct udpif *udpif, struct ukey_op *ops, size_t n_ops)
                 xin.resubmit_stats = push->n_packets ? push : NULL;
                 xin.may_learn = push->n_packets > 0;
                 xin.skip_wildcards = true;
-                xlate_actions_for_side_effects(&xin);
+                xlate_actions_for_side_effects(&xin, recirc);
 
                 if (netflow) {
                     netflow_flow_clear(netflow, &flow);
diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
index 718c494..802cc1f 100644
--- a/ofproto/ofproto-dpif-xlate.c
+++ b/ofproto/ofproto-dpif-xlate.c
@@ -70,9 +70,6 @@ VLOG_DEFINE_THIS_MODULE(ofproto_dpif_xlate);
 #define MAX_INTERNAL_RESUBMITS 1   /* Max resbmits allowed using rules in
                                       internal table. */
 
-/* Timeout for internal rules created to handle recirculation */
-#define RECIRC_TIMEOUT 60
-
 /* Maximum number of resubmit actions in a flow translation, whether they are
  * recursive or not. */
 #define MAX_RESUBMITS (MAX_RESUBMIT_RECURSION * MAX_RESUBMIT_RECURSION)
@@ -198,16 +195,28 @@ struct xlate_ctx {
     bool in_group;              /* Currently translating ofgroup, if true. */
     bool in_action_set;         /* Currently translating action_set, if true. */
 
-    uint32_t orig_skb_priority; /* Priority when packet arrived. */
     uint8_t table_id;           /* OpenFlow table ID where flow was found. */
+    ovs_be64 rule_cookie;       /* Cookie of the rule being translated. */
+    uint32_t orig_skb_priority; /* Priority when packet arrived. */
     uint32_t sflow_n_outputs;   /* Number of output ports. */
     odp_port_t sflow_odp_port;  /* Output port for composing sFlow action. */
     uint16_t user_cookie_offset;/* Used for user_action_cookie fixup. */
     bool exit;                  /* No further actions should be processed. */
 
+    /* These are used for non-bond recirculation.  The recirculation IDs are
+     * stored in xout and must be associated with a datapath flow (ukey),
+     * otherwise they will be freed when the xout is uninitialized. */
+    int recirc_action_offset;   /* Offset in 'action_set' to actions to be
+                                 * executed after recirculation, or -1. */
+    int last_unroll_offset;     /* Offset in 'action_set' to the latest unroll
+                                 * action, or -1. */
+
+    /* These are used only for bonds.  The recirculation ID is associated with
+     * an OpenFlow rule in the TBL_INTERNAL, and is deleted when that rule is
+     * deleted. */
     bool use_recirc;            /* Should generate recirc? */
-    struct xlate_recirc recirc; /* Information used for generating
-                                 * recirculation actions */
+    struct xlate_bond_recirc bond_recirc; /* Information used for generating
+                                           * bond recirculation actions */
 
     /* True if a packet was but is no longer MPLS (due to an MPLS pop action).
      * This is a trigger for recirculation in cases where translating an action
@@ -226,6 +235,15 @@ struct xlate_ctx {
     uint64_t action_set_stub[1024 / 8];
 };
 
+static inline struct ofpact_unroll_xlate *
+get_last_unroll(struct xlate_ctx *ctx)
+{
+    ovs_assert(ctx->last_unroll_offset >= 0);
+
+    return (struct ofpact_unroll_xlate *)
+        ((char *)ofpbuf_data(&ctx->action_set) + ctx->last_unroll_offset);
+}
+
 /* A controller may use OFPP_NONE as the ingress port to indicate that
  * it did not arrive on a "real" port.  'ofpp_none_bundle' exists for
  * when an input bundle is needed for validation (e.g., mirroring or
@@ -965,64 +983,54 @@ xlate_ofport_remove(struct ofport_dpif *ofport)
     xlate_xport_remove(new_xcfg, xport);
 }
 
-/* Given a datapath and flow metadata ('backer', and 'flow' respectively)
- * returns the corresponding struct xport, or NULL if none is found. */
-static struct xport *
-xlate_lookup_xport(const struct dpif_backer *backer, const struct flow *flow)
-{
-    struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
-
-    return xport_lookup(xcfg, tnl_port_should_receive(flow)
-                         ? tnl_port_receive(flow)
-                         : odp_port_to_ofport(backer, flow->in_port.odp_port));
-}
-
 static struct ofproto_dpif *
 xlate_lookup_ofproto_(const struct dpif_backer *backer, const struct flow *flow,
-                      ofp_port_t *ofp_in_port, const struct xport **xportp)
+                      ofp_port_t *ofp_in_port, const struct xport **xportp,
+                      const struct recirc_id_node **recirc_node)
 {
-    struct ofproto_dpif *recv_ofproto = NULL;
-    struct ofproto_dpif *recirc_ofproto = NULL;
+    struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
+    struct ofproto_dpif *ofproto;
     const struct xport *xport;
-    ofp_port_t in_port = OFPP_NONE;
-
-    *xportp = xport = xlate_lookup_xport(backer, flow);
+    ofp_port_t in_port;
+    const struct recirc_id_node *rid;
 
-    if (xport) {
-        recv_ofproto = xport->xbridge->ofproto;
-        in_port = xport->ofp_port;
-    }
+    if (flow->recirc_id) {
+        rid = recirc_id_node_find(flow->recirc_id);
+        if (!rid) {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
 
-    /* When recirc_id is set in 'flow', checks whether the ofproto_dpif that
-     * corresponds to the recirc_id is same as the receiving bridge.  If they
-     * are the same, uses the 'recv_ofproto' and keeps the 'ofp_in_port' as
-     * assigned.  Otherwise, uses the 'recirc_ofproto' that owns recirc_id and
-     * assigns OFPP_NONE to 'ofp_in_port'.  Doing this is in that, the
-     * recirculated flow must be processced by the ofproto which originates
-     * the recirculation, and as bridges can only see their own ports, the
-     * in_port of the 'recv_ofproto' should not be passed to the
-     * 'recirc_ofproto'.
-     *
-     * Admittedly, setting the 'ofp_in_port' to OFPP_NONE limits the
-     * 'recirc_ofproto' from meaningfully matching on in_port of recirculated
-     * flow, and should be fixed in the near future.
-     *
-     * TODO: Restore the original patch port.
-     */
-    if (recv_ofproto && flow->recirc_id) {
-        recirc_ofproto = ofproto_dpif_recirc_get_ofproto(backer,
-                                                         flow->recirc_id);
-        if (recv_ofproto != recirc_ofproto) {
-            *xportp = xport = NULL;
+            VLOG_WARN_RL(&rl, "Recirculation context not found for ID %"PRIx32,
+                         flow->recirc_id);
+            return NULL;
+        }
+        if (recirc_node) {
+            *recirc_node = rid;
+        }
+        ofproto = rid->ofproto;
+        in_port = recirc_id_node_get_in_port(rid);
+        xport = xport_lookup(xcfg, ofp_port_to_ofport(ofproto, in_port));
+    } else {
+        if (recirc_node) {
+            *recirc_node = NULL;
+        }
+        xport = xport_lookup(xcfg, tnl_port_should_receive(flow)
+                             ? tnl_port_receive(flow)
+                             : odp_port_to_ofport(backer,
+                                                  flow->in_port.odp_port));
+        if (OVS_LIKELY(xport)) {
+            in_port = xport->ofp_port;
+            ofproto = xport->xbridge->ofproto;
+        } else {
             in_port = OFPP_NONE;
+            ofproto = NULL;
         }
     }
 
+    *xportp = xport;
     if (ofp_in_port) {
         *ofp_in_port = in_port;
     }
-
-    return xport ? recv_ofproto : recirc_ofproto;
+    return ofproto;
 }
 
 /* Given a datapath and flow metadata ('backer', and 'flow' respectively)
@@ -1033,7 +1041,7 @@ xlate_lookup_ofproto(const struct dpif_backer *backer, const struct flow *flow,
 {
     const struct xport *xport;
 
-    return xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport);
+    return xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport, NULL);
 }
 
 /* Given a datapath and flow metadata ('backer', and 'flow' respectively),
@@ -1043,18 +1051,25 @@ xlate_lookup_ofproto(const struct dpif_backer *backer, const struct flow *flow,
  * pointers until quiescing, for longer term use additional references must
  * be taken.
  *
+ * The 'recirc' parameter is used to return a pointer to a recirculation
+ * context if the 'flow' has a non-zero 'recirc_id' and a context for that id
+ * is found, NULL otherwise.  The returned recirculation context is RCU
+ * protected and may not be used past the point in which the calling thread
+ * quiesces, unless the calling thread successfully acquires a reference (see
+ * recirc_id_node_try_ref_rcu()).
+ *
  * Returns 0 if successful, ENODEV if the parsed flow has no associated ofproto.
  */
 int
 xlate_lookup(const struct dpif_backer *backer, const struct flow *flow,
              struct ofproto_dpif **ofprotop, struct dpif_ipfix **ipfix,
              struct dpif_sflow **sflow, struct netflow **netflow,
-             ofp_port_t *ofp_in_port)
+             ofp_port_t *ofp_in_port, const struct recirc_id_node **recirc)
 {
     struct ofproto_dpif *ofproto;
     const struct xport *xport;
 
-    ofproto = xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport);
+    ofproto = xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport, recirc);
 
     if (!ofproto) {
         return ENODEV;
@@ -1670,7 +1685,7 @@ output_normal(struct xlate_ctx *ctx, const struct xbundle *out_xbundle,
     } else {
         struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
         struct flow_wildcards *wc = &ctx->xout->wc;
-        struct xlate_recirc *xr = &ctx->recirc;
+        struct xlate_bond_recirc *xr = &ctx->bond_recirc;
         struct ofport_dpif *ofport;
 
         if (ctx->xbridge->enable_recirc) {
@@ -2723,6 +2738,8 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
         memset(&flow->tunnel, 0, sizeof flow->tunnel);
         memset(flow->regs, 0, sizeof flow->regs);
         flow->actset_output = OFPP_UNSET;
+        /* XXX: Action set should be executed (xlate_action_set()) before
+         * output to a patch port? */
 
         special = process_special(ctx, &ctx->xin->flow, peer,
                                   ctx->xin->packet);
@@ -2764,6 +2781,22 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
             entry->u.dev.rx = netdev_ref(peer->netdev);
             entry->u.dev.bfd = bfd_ref(peer->bfd);
         }
+
+        /* Check if need to unroll metadata for recirculation. */
+        if (ctx->exit && ctx->recirc_action_offset >= 0) {
+            struct ofpact_unroll_xlate *unroll;
+
+            /* Insert the current pipeline context as an unroll action to the
+             * current set of recirculation actions so that any further
+             * recirculation actions have the right context. */
+            ofpact_unroll_metadata_from_flow(
+                ofpact_put_UNROLL_METADATA(&ctx->action_set), flow);
+
+            ctx->last_unroll_offset = ofpbuf_size(&ctx->action_set);
+            unroll = ofpact_put_UNROLL_XLATE(&ctx->action_set);
+            unroll->rule_table_id = ctx->table_id;
+            unroll->rule_cookie = ctx->rule_cookie;
+        }
         return;
     }
 
@@ -2836,7 +2869,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
 
         if (ctx->use_recirc) {
             struct ovs_action_hash *act_hash;
-            struct xlate_recirc *xr = &ctx->recirc;
+            struct xlate_bond_recirc *xr = &ctx->bond_recirc;
 
             /* Hash action. */
             act_hash = nl_msg_put_unspec_uninit(ctx->xout->odp_actions,
@@ -2901,6 +2934,7 @@ static void
 xlate_recursively(struct xlate_ctx *ctx, struct rule_dpif *rule)
 {
     struct rule_dpif *old_rule = ctx->rule;
+    ovs_be64 old_cookie = ctx->rule_cookie;
     const struct rule_actions *actions;
 
     if (ctx->xin->resubmit_stats) {
@@ -2910,8 +2944,10 @@ xlate_recursively(struct xlate_ctx *ctx, struct rule_dpif *rule)
     ctx->resubmits++;
     ctx->recurse++;
     ctx->rule = rule;
+    ctx->rule_cookie = rule_dpif_get_flow_cookie(rule);
     actions = rule_dpif_get_actions(rule);
     do_xlate_actions(actions->ofpacts, actions->ofpacts_len, ctx);
+    ctx->rule_cookie = old_cookie;
     ctx->rule = old_rule;
     ctx->recurse--;
 }
@@ -2971,7 +3007,29 @@ xlate_table_action(struct xlate_ctx *ctx, ofp_port_t in_port, uint8_t table_id,
                 entry = xlate_cache_add_entry(ctx->xin->xcache, XC_RULE);
                 entry->u.rule = rule;
             }
+
             xlate_recursively(ctx, rule);
+
+            /* Check if need to unroll metadata for recirculation. */
+            if (ctx->exit && ctx->recirc_action_offset >= 0) {
+                struct ofpact_unroll_xlate *unroll = get_last_unroll(ctx);
+
+                /* Check if previously unrolled values are now invalid. */
+                if (old_table_id != unroll->rule_table_id
+                    || ctx->rule_cookie != unroll->rule_cookie) {
+
+                    /* Check if there were any actions after the last unroll.
+                     */
+                    if (ofpact_next(&unroll->ofpact) <
+                        (struct ofpact *)ofpbuf_data(&ctx->action_set)) {
+                        /* Push a new one. */
+                        ctx->last_unroll_offset = ofpbuf_size(&ctx->action_set);
+                        unroll = ofpact_put_UNROLL_XLATE(&ctx->action_set);
+                    } /* else reuse the last unroll. */
+                    unroll->rule_table_id = old_table_id;
+                    unroll->rule_cookie = ctx->rule_cookie;
+                }
+            }
         }
 
         ctx->table_id = old_table_id;
@@ -3219,9 +3277,7 @@ execute_controller_action(struct xlate_ctx *ctx, int len,
     pin->up.packet = ofpbuf_steal_data(&packet->ofpbuf);
     pin->up.reason = reason;
     pin->up.table_id = ctx->table_id;
-    pin->up.cookie = (ctx->rule
-                      ? rule_dpif_get_flow_cookie(ctx->rule)
-                      : OVS_BE64_MAX);
+    pin->up.cookie = ctx->rule_cookie;
 
     flow_get_metadata(&ctx->xin->flow, &pin->up.fmd);
 
@@ -3247,78 +3303,44 @@ execute_controller_action(struct xlate_ctx *ctx, int len,
     dpif_packet_delete(packet);
 }
 
+/* Called only when ctx->recirc_action_offset is set. */
 static void
-compose_recirculate_action(struct xlate_ctx *ctx,
-                           const struct ofpact *ofpacts_base,
-                           const struct ofpact *ofpact_current,
-                           size_t ofpacts_base_len)
+compose_recirculate_action(struct xlate_ctx *ctx)
 {
     uint32_t id;
-    int error;
-
-    ctx->exit = true;
 
-    if (ctx->rule) {
-        id = rule_dpif_get_recirc_id(ctx->rule);
-    } else {
-        /* In the case where ctx has no rule then allocate a recirc id.
-         * The life-cycle of this recirc id is managed by associating it
-         * with the internal rule that is created to to handle
-         * recirculation below.
-         *
-         * The known use-case of this is packet_out which
-         * translates actions without a rule */
-        id = ofproto_dpif_alloc_recirc_id(ctx->xbridge->ofproto);
-    }
-    if (!id) {
-        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
-        VLOG_ERR_RL(&rl, "Failed to allocate recirculation id");
-        return;
-    }
+    ctx->xout->slow |= commit_odp_actions(&ctx->xin->flow, &ctx->base_flow,
+                                          ctx->xout->odp_actions,
+                                          &ctx->xout->wc,
+                                          ctx->xbridge->masked_set_action);
 
     /* Only create the recirculation flow if we have a packet. */
     if (ctx->xin->packet) {
-        struct match match;
-        struct rule *rule;
-        unsigned ofpacts_len;
-        struct ofpbuf ofpacts;
-
-        ofpacts_len = ofpacts_base_len -
-            ((uint8_t *)ofpact_current - (uint8_t *)ofpacts_base);
-
-        match_init_catchall(&match);
-        match_set_recirc_id(&match, id);
-        ofpbuf_use_const(&ofpacts, ofpact_current, ofpacts_len);
-        error = ofproto_dpif_add_internal_flow(ctx->xbridge->ofproto, &match,
-                                               RECIRC_RULE_PRIORITY,
-                                               RECIRC_TIMEOUT, &ofpacts, &rule);
-        if (error) {
+        /* Allocate a unique recirc id for the given metadata state in the
+         * flow.  The life-cycle of this recirc id is managed by associating it
+         * with the udpif key ('ukey') created for each new datapath flow. */
+        id = recirc_alloc_id_ctx(ctx->xbridge->ofproto,
+                                 ctx->recirc_action_offset,
+                                 ofpbuf_size(&ctx->action_set),
+                                 ofpbuf_data(&ctx->action_set));
+        if (!id) {
             static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
-            VLOG_ERR_RL(&rl, "Failed to add post recirculation flow %s",
-                        match_to_string(&match, 0));
-            if (!ctx->rule) {
-                ofproto_dpif_free_recirc_id(ctx->xbridge->ofproto, id);
-            }
+            VLOG_ERR_RL(&rl, "Failed to allocate recirculation id");
             return;
         }
-        /* If ctx has no rule then associate the recirc id, which
-         * was allocated above, with the internal rule.  This allows
-         * the recirc id to be released when the internal rule times out. */
-        if (!ctx->rule) {
-            rule_set_recirc_id(rule, id);
-        }
+        xlate_out_add_recirc(ctx->xout, id);
+    } else {
+        /* Look up an existing recirc id for the given metadata state in the
+         * flow.  No new reference is taken, as the ID is RCU protected and is
+         * only required temporarily for verification. */
+        id = recirc_find_id(ctx->xbridge->ofproto, ctx->recirc_action_offset,
+                            ofpbuf_size(&ctx->action_set),
+                            ofpbuf_data(&ctx->action_set));
+        /* We let zero 'id' to be used in the RECIRC action below, which will
+         * fail all revalidations as zero is not a valid recirculation ID. */
     }
 
-    ctx->xout->slow |= commit_odp_actions(&ctx->xin->flow, &ctx->base_flow,
-                                          ctx->xout->odp_actions,
-                                          &ctx->xout->wc,
-                                          ctx->xbridge->masked_set_action);
     nl_msg_put_u32(ctx->xout->odp_actions, OVS_ACTION_ATTR_RECIRC, id);
-
-    /* Free recirc id if it was not associated with any rule. */
-    if (!ctx->rule && !ctx->xin->packet) {
-        ofproto_dpif_free_recirc_id(ctx->xbridge->ofproto, id);
-    }
 }
 
 static void
@@ -3797,6 +3819,8 @@ ofpact_needs_recirculation_after_mpls(const struct ofpact *a, struct xlate_ctx *
     case OFPACT_POP_QUEUE:
     case OFPACT_CONJUNCTION:
     case OFPACT_NOTE:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_OUTPUT_REG:
     case OFPACT_EXIT:
     case OFPACT_METER:
@@ -3868,13 +3892,32 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
         const struct ofpact_set_field *set_field;
         const struct mf_field *mf;
 
-        if (ctx->exit) {
-            break;
+        if (!ctx->exit && ofpact_needs_recirculation_after_mpls(a, ctx)) {
+            struct ofpact_unroll_xlate *unroll;
+
+            ctx->recirc_action_offset = ofpbuf_size(&ctx->action_set);
+            /* Insert the current pipeline context as an unroll action right
+             * after the action set, at the 'recirc_action_offset'. */
+            ofpact_unroll_metadata_from_flow(
+                ofpact_put_UNROLL_METADATA(&ctx->action_set), flow);
+
+            ctx->last_unroll_offset = ofpbuf_size(&ctx->action_set);
+            unroll = ofpact_put_UNROLL_XLATE(&ctx->action_set);
+            unroll->rule_table_id = ctx->table_id;
+            unroll->rule_cookie = ctx->rule_cookie;
+
+            ctx->exit = true;
         }
 
-        if (ofpact_needs_recirculation_after_mpls(a, ctx)) {
-            compose_recirculate_action(ctx, ofpacts, a, ofpacts_len);
-            return;
+        if (ctx->exit) {
+            /* Check if need to store the remaining actions for later
+             * execution. */
+            if (ctx->recirc_action_offset >= 0) {
+                ofpbuf_put(&ctx->action_set, a,
+                           OFPACT_ALIGN(ofpacts_len -
+                                        ((uint8_t *)a - (uint8_t *)ofpacts)));
+            }
+            break;
         }
 
         switch (a->type) {
@@ -4123,6 +4166,20 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
             ctx->exit = true;
             break;
 
+        case OFPACT_UNROLL_METADATA:
+            /* Restore pipeline metadata that was stored earlier. */
+            ofpact_unroll_metadata_to_flow(ofpact_get_UNROLL_METADATA(a),
+                                           flow);
+            break;
+
+        case OFPACT_UNROLL_XLATE: {
+            struct ofpact_unroll_xlate *unroll = ofpact_get_UNROLL_XLATE(a);
+
+            /* Restore translation context data that was stored earlier. */
+            ctx->table_id = unroll->rule_table_id;
+            ctx->rule_cookie = unroll->rule_cookie;
+            break;
+        }
         case OFPACT_FIN_TIMEOUT:
             memset(&wc->masks.nw_proto, 0xff, sizeof wc->masks.nw_proto);
             ctx->xout->has_fin_timeout = true;
@@ -4197,19 +4254,23 @@ xlate_in_init(struct xlate_in *xin, struct ofproto_dpif *ofproto,
 void
 xlate_out_uninit(struct xlate_out *xout)
 {
-    if (xout && xout->odp_actions == &xout->odp_actions_buf) {
-        ofpbuf_uninit(xout->odp_actions);
+    if (xout) {
+        if (xout->odp_actions == &xout->odp_actions_buf) {
+            ofpbuf_uninit(xout->odp_actions);
+        }
+        xlate_out_free_recircs(xout);
     }
 }
 
 /* Translates the 'ofpacts_len' bytes of "struct ofpact"s starting at 'ofpacts'
  * into datapath actions, using 'ctx', and discards the datapath actions. */
 void
-xlate_actions_for_side_effects(struct xlate_in *xin)
+xlate_actions_for_side_effects(struct xlate_in *xin,
+                               const struct recirc_id_node *recirc)
 {
     struct xlate_out xout;
 
-    xlate_actions(xin, &xout);
+    xlate_actions(xin, &xout, recirc);
     xlate_out_uninit(&xout);
 }
 
@@ -4362,13 +4423,17 @@ too_many_output_actions(const struct ofpbuf *odp_actions OVS_UNUSED)
 #endif
 }
 
-/* Translates the 'ofpacts_len' bytes of "struct ofpacts" starting at 'ofpacts'
- * into datapath actions in 'odp_actions', using 'ctx'.
+/* Translates the flow, actions, or rule in 'xin' into datapath actions in
+ * 'xout', using recirculation context 'recirc'.  The recirculation context
+ * should be NULL for packets that were not recirculated.  The metadata and
+ * actions in 'recirc' will be used to continue processing from the pipeline
+ * state that was in effect when the recircualtion action was issued.
  *
  * The caller must take responsibility for eventually freeing 'xout', with
- * xlate_out_uninit(). */
+ * xlate_out_uninit().  'recirc' is RCU protected. */
 void
-xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
+xlate_actions(struct xlate_in *xin, struct xlate_out *xout,
+              const struct recirc_id_node *recirc)
 {
     struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
     struct flow_wildcards *wc = &xout->wc;
@@ -4415,6 +4480,7 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
     ctx.xout->has_fin_timeout = false;
     ctx.xout->nf_output_iface = NF_OUT_DROP;
     ctx.xout->mirrors = 0;
+    ctx.xout->n_recircs = 0;
 
     xout->odp_actions = xin->odp_actions;
     if (!xout->odp_actions) {
@@ -4431,10 +4497,6 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
 
     ctx.rule = xin->rule;
 
-    ctx.base_flow = *flow;
-    memset(&ctx.base_flow.tunnel, 0, sizeof ctx.base_flow.tunnel);
-    ctx.orig_tunnel_ip_dst = flow->tunnel.ip_dst;
-
     flow_wildcards_init_catchall(wc);
     memset(&wc->masks.in_port, 0xff, sizeof wc->masks.in_port);
     memset(&wc->masks.dl_type, 0xff, sizeof wc->masks.dl_type);
@@ -4443,6 +4505,54 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
     }
     is_icmp = is_icmpv4(flow) || is_icmpv6(flow);
 
+    ctx.table_id = 0;
+    ctx.rule_cookie = OVS_BE64_MAX;
+
+    ctx.action_set_has_group = false;
+    ofpbuf_use_stub(&ctx.action_set,
+                    ctx.action_set_stub, sizeof ctx.action_set_stub);
+
+    if (recirc) {
+        wc->masks.recirc_id = UINT32_MAX;
+
+        if (xin->ofpacts_len > 0 || ctx.rule) {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+
+            VLOG_WARN_RL(&rl, "Recirculation conflict (%s)!",
+                         xin->ofpacts_len > 0
+                         ? "actions"
+                         : "rule");
+            return;
+        }
+
+        /* Restore action set, if any. */
+        if (recirc->action_set_len) {
+            const struct ofpact *a;
+
+            ofpbuf_put(&ctx.action_set, recirc->ofpacts,
+                       recirc->action_set_len);
+            ofpact_pad(&ctx.action_set); /* XXX: Should not be needed. */
+
+            OFPACT_FOR_EACH(a, recirc->ofpacts, recirc->action_set_len) {
+                if (a->type == OFPACT_GROUP) {
+                    ctx.action_set_has_group = true;
+                    break;
+                }
+            }
+        }
+        /* Restore recirculation actions. These include unroll actions to
+         * restore the pipeline metadata. */
+        if (recirc->ofpacts_len - recirc->action_set_len) {
+            xin->ofpacts_len = recirc->ofpacts_len - recirc->action_set_len;
+            xin->ofpacts = (const struct ofpact *)
+                ((const char *)recirc->ofpacts + recirc->action_set_len);
+        }
+    }
+
+    ctx.base_flow = *flow;
+    memset(&ctx.base_flow.tunnel, 0, sizeof ctx.base_flow.tunnel);
+    ctx.orig_tunnel_ip_dst = flow->tunnel.ip_dst;
+
     tnl_may_send = tnl_xlate_init(&ctx.base_flow, flow, wc);
     if (ctx.xbridge->netflow) {
         netflow_mask_wc(flow, wc);
@@ -4453,9 +4563,10 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
     ctx.in_group = false;
     ctx.in_action_set = false;
     ctx.orig_skb_priority = flow->skb_priority;
-    ctx.table_id = 0;
     ctx.exit = false;
     ctx.use_recirc = false;
+    ctx.recirc_action_offset = -1;
+    ctx.last_unroll_offset = -1;
     ctx.was_mpls = false;
 
     if (!xin->ofpacts && !ctx.rule) {
@@ -4488,16 +4599,14 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
 
         ofpacts = actions->ofpacts;
         ofpacts_len = actions->ofpacts_len;
+
+        ctx.rule_cookie = rule_dpif_get_flow_cookie(ctx.rule);
     } else {
         OVS_NOT_REACHED();
     }
 
     ofpbuf_use_stub(&ctx.stack, ctx.init_stack, sizeof ctx.init_stack);
 
-    ctx.action_set_has_group = false;
-    ofpbuf_use_stub(&ctx.action_set,
-                    ctx.action_set_stub, sizeof ctx.action_set_stub);
-
     if (mbridge_has_mirrors(ctx.xbridge->mbridge)) {
         /* Do this conditionally because the copy is expensive enough that it
          * shows up in profiles. */
@@ -4548,8 +4657,23 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
                 /* Drop all actions added by do_xlate_actions() above. */
                 ofpbuf_set_size(ctx.xout->odp_actions, sample_actions_len);
             } else if (ofpbuf_size(&ctx.action_set)) {
-                /* Translate action set only if not dropping the packet. */
-                xlate_action_set(&ctx);
+                /* Translate action set only if not dropping the packet and
+                 * not recirculating. */
+                if (ctx.recirc_action_offset < 0) {
+                    xlate_action_set(&ctx);
+                    /* If action set recirculates the action set needs to be
+                     * cleared, as the actions in the set have been sanitized
+                     * to a temporary action list that is being executed. (If
+                     * we keep the set, the actions would be executed twice. */
+                    if (ctx.recirc_action_offset >= 0) {
+                        ofpbuf_pull(&ctx.action_set, ctx.recirc_action_offset);
+                        ctx.recirc_action_offset = 0;
+                    }
+                }
+                /* Check if need to recirculate. */
+                if (ctx.recirc_action_offset >= 0) {
+                    compose_recirculate_action(&ctx);
+                }
             }
         }
 
diff --git a/ofproto/ofproto-dpif-xlate.h b/ofproto/ofproto-dpif-xlate.h
index 9a03782..715c5cd 100644
--- a/ofproto/ofproto-dpif-xlate.h
+++ b/ofproto/ofproto-dpif-xlate.h
@@ -20,6 +20,7 @@
 #include "odp-util.h"
 #include "ofpbuf.h"
 #include "ofproto-dpif-mirror.h"
+#include "ofproto-dpif-rid.h"
 #include "ofproto-dpif.h"
 #include "ofproto.h"
 #include "stp.h"
@@ -34,7 +35,7 @@ struct mac_learning;
 struct mcast_snooping;
 struct xlate_cache;
 
-struct xlate_recirc {
+struct xlate_bond_recirc {
     uint32_t recirc_id;  /* !0 Use recirculation instead of output. */
     uint8_t  hash_alg;   /* !0 Compute hash for recirc before. */
     uint32_t hash_basis;  /* Compute hash for recirc before. */
@@ -58,11 +59,72 @@ struct xlate_out {
     ofp_port_t nf_output_iface; /* Output interface index for NetFlow. */
     mirror_mask_t mirrors;      /* Bitmap of associated mirrors. */
 
+    /* Recirculation IDs on which references are held. */
+    unsigned n_recircs;
+    union {
+        uint32_t recirc[2];   /* When n_recircs == 1 or 2 */
+        uint32_t *recircs;    /* When 'n_recircs' > 2 */
+    };
+
     uint64_t odp_actions_stub[256 / 8];
     struct ofpbuf odp_actions_buf;
     struct ofpbuf *odp_actions;
 };
 
+/* Helpers to abstract the recirculation union away. */
+static inline void
+xlate_out_add_recirc(struct xlate_out *xout, uint32_t id)
+{
+    if (OVS_LIKELY(xout->n_recircs < ARRAY_SIZE(xout->recirc))) {
+        xout->recirc[xout->n_recircs++] = id;
+    } else {
+        if (xout->n_recircs == ARRAY_SIZE(xout->recirc)) {
+            uint32_t *recircs = xmalloc(sizeof xout->recirc + sizeof id);
+
+            memcpy(recircs, xout->recirc, sizeof xout->recirc);
+            xout->recircs = recircs;
+        } else {
+            xout->recircs = xrealloc(xout->recircs,
+                                     (xout->n_recircs + 1) * sizeof id);
+        }
+        xout->recircs[xout->n_recircs++] = id;
+    }
+}
+
+static inline const uint32_t *
+xlate_out_get_recircs(const struct xlate_out *xout)
+{
+    if (OVS_LIKELY(xout->n_recircs <= ARRAY_SIZE(xout->recirc))) {
+        return xout->recirc;
+    } else {
+        return xout->recircs;
+    }
+}
+
+static inline void
+xlate_out_take_recircs(struct xlate_out *xout)
+{
+    if (OVS_UNLIKELY(xout->n_recircs > ARRAY_SIZE(xout->recirc))) {
+        free(xout->recircs);
+    }
+    xout->n_recircs = 0;
+}
+
+static inline void
+xlate_out_free_recircs(struct xlate_out *xout)
+{
+    if (OVS_LIKELY(xout->n_recircs <= ARRAY_SIZE(xout->recirc))) {
+        for (int i = 0; i < xout->n_recircs; i++) {
+            recirc_free_id(xout->recirc[i]);
+        }
+    } else {
+        for (int i = 0; i < xout->n_recircs; i++) {
+            recirc_free_id(xout->recircs[i]);
+        }
+        free(xout->recircs);
+    }
+}
+
 struct xlate_in {
     struct ofproto_dpif *ofproto;
 
@@ -180,14 +242,16 @@ struct ofproto_dpif * xlate_lookup_ofproto(const struct dpif_backer *,
 int xlate_lookup(const struct dpif_backer *, const struct flow *,
                  struct ofproto_dpif **, struct dpif_ipfix **,
                  struct dpif_sflow **, struct netflow **,
-                 ofp_port_t *ofp_in_port);
+                 ofp_port_t *ofp_in_port, const struct recirc_id_node **);
 
-void xlate_actions(struct xlate_in *, struct xlate_out *);
+void xlate_actions(struct xlate_in *, struct xlate_out *,
+                   const struct recirc_id_node *);
 void xlate_in_init(struct xlate_in *, struct ofproto_dpif *,
                    const struct flow *, ofp_port_t in_port, struct rule_dpif *,
                    uint16_t tcp_flags, const struct ofpbuf *packet);
 void xlate_out_uninit(struct xlate_out *);
-void xlate_actions_for_side_effects(struct xlate_in *);
+void xlate_actions_for_side_effects(struct xlate_in *,
+                                    const struct recirc_id_node *);
 void xlate_out_copy(struct xlate_out *dst, const struct xlate_out *src);
 
 int xlate_send_packet(const struct ofport_dpif *, struct ofpbuf *);
diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index ed29c52..8b2b66b 100644
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -252,13 +252,6 @@ COVERAGE_DEFINE(rev_flow_table);
 COVERAGE_DEFINE(rev_mac_learning);
 COVERAGE_DEFINE(rev_mcast_snooping);
 
-/* Stores mapping between 'recirc_id' and 'ofproto-dpif'. */
-struct dpif_backer_recirc_node {
-    struct cmap_node cmap_node;
-    struct ofproto_dpif *ofproto;
-    uint32_t recirc_id;
-};
-
 /* All datapaths of a given type share a single dpif backer instance. */
 struct dpif_backer {
     char *type;
@@ -276,9 +269,6 @@ struct dpif_backer {
     bool recv_set_enable; /* Enables or disables receiving packets. */
 
     /* Recirculation. */
-    struct recirc_id_pool *rid_pool;       /* Recirculation ID pool. */
-    struct cmap recirc_map;         /* Map of 'recirc_id's to 'ofproto's. */
-    struct ovs_mutex recirc_mutex;  /* Protects 'recirc_map'. */
     bool enable_recirc;   /* True if the datapath supports recirculation */
 
     /* True if the datapath supports unique flow identifiers */
@@ -392,8 +382,6 @@ ofproto_dpif_get_enable_ufid(struct dpif_backer *backer)
     return backer->enable_ufid;
 }
 
-static struct ofport_dpif *get_ofp_port(const struct ofproto_dpif *ofproto,
-                                        ofp_port_t ofp_port);
 static void ofproto_trace(struct ofproto_dpif *, struct flow *,
                           const struct ofpbuf *packet,
                           const struct ofpact[], size_t ofpacts_len,
@@ -854,27 +842,6 @@ dealloc(struct ofproto *ofproto_)
     free(ofproto);
 }
 
-/* Called when 'ofproto' is destructed.  Checks for and clears any
- * recirc_id leak. */
-static void
-dpif_backer_recirc_clear_ofproto(struct dpif_backer *backer,
-                                 struct ofproto_dpif *ofproto)
-{
-    struct dpif_backer_recirc_node *node;
-
-    ovs_mutex_lock(&backer->recirc_mutex);
-    CMAP_FOR_EACH (node, cmap_node, &backer->recirc_map) {
-        if (node->ofproto == ofproto) {
-            VLOG_ERR("recirc_id %"PRIu32", not freed when ofproto (%s) "
-                     "is destructed", node->recirc_id, ofproto->up.name);
-            cmap_remove(&backer->recirc_map, &node->cmap_node,
-                        node->recirc_id);
-            ovsrcu_postpone(free, node);
-        }
-    }
-    ovs_mutex_unlock(&backer->recirc_mutex);
-}
-
 static void
 close_dpif_backer(struct dpif_backer *backer)
 {
@@ -890,9 +857,6 @@ close_dpif_backer(struct dpif_backer *backer)
     ovs_rwlock_destroy(&backer->odp_to_ofport_lock);
     hmap_destroy(&backer->odp_to_ofport_map);
     shash_find_and_delete(&all_dpif_backers, backer->type);
-    recirc_id_pool_destroy(backer->rid_pool);
-    cmap_destroy(&backer->recirc_map);
-    ovs_mutex_destroy(&backer->recirc_mutex);
     free(backer->type);
     free(backer->dp_version_string);
     dpif_close(backer->dpif);
@@ -926,6 +890,8 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
     const char *name;
     int error;
 
+    recirc_init();
+
     backer = shash_find_data(&all_dpif_backers, type);
     if (backer) {
         backer->refcount++;
@@ -1007,9 +973,6 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
     backer->max_mpls_depth = check_max_mpls_depth(backer);
     backer->masked_set_action = check_masked_set_action(backer);
     backer->enable_ufid = check_ufid(backer);
-    backer->rid_pool = recirc_id_pool_create();
-    ovs_mutex_init(&backer->recirc_mutex);
-    cmap_init(&backer->recirc_map);
 
     backer->enable_tnl_push_pop = dpif_supports_tnl_push_pop(backer->dpif);
     atomic_count_init(&backer->tnl_count, 0);
@@ -1460,7 +1423,7 @@ destruct(struct ofproto *ofproto_)
     }
     guarded_list_destroy(&ofproto->pins);
 
-    dpif_backer_recirc_clear_ofproto(ofproto->backer, ofproto);
+    recirc_free_ofproto(ofproto, ofproto->up.name);
 
     mbridge_unref(ofproto->mbridge);
 
@@ -2664,7 +2627,7 @@ bundle_add_port(struct ofbundle *bundle, ofp_port_t ofp_port,
 {
     struct ofport_dpif *port;
 
-    port = get_ofp_port(bundle->ofproto, ofp_port);
+    port = ofp_port_to_ofport(bundle->ofproto, ofp_port);
     if (!port) {
         return false;
     }
@@ -3170,8 +3133,8 @@ set_mcast_snooping_port(struct ofproto *ofproto_, void *aux,
 
 /* Ports. */
 
-static struct ofport_dpif *
-get_ofp_port(const struct ofproto_dpif *ofproto, ofp_port_t ofp_port)
+struct ofport_dpif *
+ofp_port_to_ofport(const struct ofproto_dpif *ofproto, ofp_port_t ofp_port)
 {
     struct ofport *ofport = ofproto_get_port(&ofproto->up, ofp_port);
     return ofport ? ofport_dpif_cast(ofport) : NULL;
@@ -3365,7 +3328,7 @@ static int
 port_del(struct ofproto *ofproto_, ofp_port_t ofp_port)
 {
     struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
-    struct ofport_dpif *ofport = get_ofp_port(ofproto, ofp_port);
+    struct ofport_dpif *ofport = ofp_port_to_ofport(ofproto, ofp_port);
     int error = 0;
 
     if (!ofport) {
@@ -3615,7 +3578,7 @@ ofproto_dpif_execute_actions(struct ofproto_dpif *ofproto,
     xin.ofpacts = ofpacts;
     xin.ofpacts_len = ofpacts_len;
     xin.resubmit_stats = &stats;
-    xlate_actions(&xin, &xout);
+    xlate_actions(&xin, &xout, NULL);
 
     execute.actions = ofpbuf_data(xout.odp_actions);
     execute.actions_len = ofpbuf_size(xout.odp_actions);
@@ -3677,21 +3640,13 @@ static void
 rule_dpif_set_recirc_id(struct rule_dpif *rule, uint32_t id)
     OVS_REQUIRES(rule->up.mutex)
 {
-    ovs_assert(!rule->recirc_id);
-    rule->recirc_id = id;
-}
-
-/* Returns 'rule''s recirculation id. */
-uint32_t
-rule_dpif_get_recirc_id(struct rule_dpif *rule)
-    OVS_REQUIRES(rule->up.mutex)
-{
-    if (!rule->recirc_id) {
-        struct ofproto_dpif *ofproto = ofproto_dpif_cast(rule->up.ofproto);
-
-        rule_dpif_set_recirc_id(rule, ofproto_dpif_alloc_recirc_id(ofproto));
+    ovs_assert(!rule->recirc_id || rule->recirc_id == id);
+    if (rule->recirc_id == id) {
+        /* Release the new reference to the same id. */
+        recirc_free_id(id);
+    } else {
+        rule->recirc_id = id;
     }
-    return rule->recirc_id;
 }
 
 /* Sets 'rule''s recirculation id. */
@@ -3726,8 +3681,9 @@ rule_dpif_lookup(struct ofproto_dpif *ofproto, struct flow *flow,
 {
     *table_id = 0;
 
+    /* This is needed to support bond recirculation via the internal table. */
     if (ofproto_dpif_get_enable_recirc(ofproto)) {
-        /* Always exactly match recirc_id since datapath supports
+        /* Always exactly match recirc_id when datapath supports
          * recirculation.  */
         if (wc) {
             wc->masks.recirc_id = UINT32_MAX;
@@ -3873,7 +3829,7 @@ rule_dpif_lookup_from_table(struct ofproto_dpif *ofproto, struct flow *flow,
             || miss_config == OFPUTIL_TABLE_MISS_CONTROLLER) {
             struct ofport_dpif *port;
 
-            port = get_ofp_port(ofproto, old_in_port);
+            port = ofp_port_to_ofport(ofproto, old_in_port);
             if (!port) {
                 VLOG_WARN_RL(&rl, "packet-in on unknown OpenFlow port %"PRIu16,
                              old_in_port);
@@ -3964,9 +3920,7 @@ rule_destruct(struct rule *rule_)
 
     ovs_mutex_destroy(&rule->stats_mutex);
     if (rule->recirc_id) {
-        struct ofproto_dpif *ofproto = ofproto_dpif_cast(rule->up.ofproto);
-
-        ofproto_dpif_free_recirc_id(ofproto, rule->recirc_id);
+        recirc_free_id(rule->recirc_id);
     }
 }
 
@@ -4834,6 +4788,28 @@ ofproto_trace(struct ofproto_dpif *ofproto, struct flow *flow,
               struct ds *ds)
 {
     struct trace_ctx trace;
+    const struct recirc_id_node *recirc;
+    ofp_port_t in_port;
+
+    if (flow->recirc_id) {
+        recirc = recirc_id_node_find(flow->recirc_id);
+
+        if (!recirc) {
+            ds_put_format(ds,
+                          "Recirculation context not found for ID %"PRIu32"\n",
+                          flow->recirc_id);
+            return;
+        } else if (ofproto != recirc->ofproto) {
+            ds_put_format(ds, "Recirculation bridge (%s) does not match %s\n",
+                          recirc->ofproto->up.name, ofproto->up.name);
+            return;
+        } else {
+            in_port = recirc_id_node_get_in_port(recirc);
+        }
+    } else {
+        recirc = NULL;
+        in_port = flow->in_port.ofp_port;
+    }
 
     ds_put_format(ds, "Bridge: %s\n", ofproto->up.name);
     ds_put_cstr(ds, "Flow: ");
@@ -4845,14 +4821,14 @@ ofproto_trace(struct ofproto_dpif *ofproto, struct flow *flow,
     trace.result = ds;
     trace.key = flow; /* Original flow key, used for megaflow. */
     trace.flow = *flow; /* May be modified by actions. */
-    xlate_in_init(&trace.xin, ofproto, flow, flow->in_port.ofp_port, NULL,
+    xlate_in_init(&trace.xin, ofproto, flow, in_port, NULL,
                   ntohs(flow->tcp_flags), packet);
     trace.xin.ofpacts = ofpacts;
     trace.xin.ofpacts_len = ofpacts_len;
     trace.xin.resubmit_hook = trace_resubmit;
     trace.xin.report_hook = trace_report;
 
-    xlate_actions(&trace.xin, &trace.xout);
+    xlate_actions(&trace.xin, &trace.xout, recirc);
 
     ds_put_char(ds, '\n');
     trace_format_flow(ds, 0, "Final flow", &trace);
@@ -5391,7 +5367,7 @@ vsp_add(struct ofport_dpif *port, ofp_port_t realdev_ofp_port, int vid)
 static odp_port_t
 ofp_port_to_odp_port(const struct ofproto_dpif *ofproto, ofp_port_t ofp_port)
 {
-    const struct ofport_dpif *ofport = get_ofp_port(ofproto, ofp_port);
+    const struct ofport_dpif *ofport = ofp_port_to_ofport(ofproto, ofp_port);
     return ofport ? ofport->odp_port : ODPP_NONE;
 }
 
@@ -5426,61 +5402,6 @@ odp_port_to_ofp_port(const struct ofproto_dpif *ofproto, odp_port_t odp_port)
     }
 }
 
-struct ofproto_dpif *
-ofproto_dpif_recirc_get_ofproto(const struct dpif_backer *backer,
-                                uint32_t recirc_id)
-{
-    struct dpif_backer_recirc_node *node;
-
-    node = CONTAINER_OF(cmap_find(&backer->recirc_map, recirc_id),
-                        struct dpif_backer_recirc_node, cmap_node);
-
-    return node ? node->ofproto : NULL;
-}
-
-uint32_t
-ofproto_dpif_alloc_recirc_id(struct ofproto_dpif *ofproto)
-{
-    struct dpif_backer *backer = ofproto->backer;
-    uint32_t recirc_id = recirc_id_alloc(backer->rid_pool);
-
-    if (recirc_id) {
-        struct dpif_backer_recirc_node *node = xmalloc(sizeof *node);
-
-        node->recirc_id = recirc_id;
-        node->ofproto = ofproto;
-
-        ovs_mutex_lock(&backer->recirc_mutex);
-        cmap_insert(&backer->recirc_map, &node->cmap_node, node->recirc_id);
-        ovs_mutex_unlock(&backer->recirc_mutex);
-    }
-
-    return recirc_id;
-}
-
-void
-ofproto_dpif_free_recirc_id(struct ofproto_dpif *ofproto, uint32_t recirc_id)
-{
-    struct dpif_backer *backer = ofproto->backer;
-    struct dpif_backer_recirc_node *node;
-
-    node = CONTAINER_OF(cmap_find(&backer->recirc_map, recirc_id),
-                        struct dpif_backer_recirc_node, cmap_node);
-    if (node) {
-        ovs_mutex_lock(&backer->recirc_mutex);
-        cmap_remove(&backer->recirc_map, &node->cmap_node, node->recirc_id);
-        ovs_mutex_unlock(&backer->recirc_mutex);
-        recirc_id_free(backer->rid_pool, node->recirc_id);
-
-        /* 'recirc_id' should never be freed by non-owning 'ofproto'. */
-        ovs_assert(node->ofproto == ofproto);
-
-        /* RCU postpone the free, since other threads may be referring
-         * to 'node' at same time. */
-        ovsrcu_postpone(free, node);
-    }
-}
-
 int
 ofproto_dpif_add_internal_flow(struct ofproto_dpif *ofproto,
                                const struct match *match, int priority,
diff --git a/ofproto/ofproto-dpif.h b/ofproto/ofproto-dpif.h
index a8570b2..7c4db5a 100644
--- a/ofproto/ofproto-dpif.h
+++ b/ofproto/ofproto-dpif.h
@@ -115,7 +115,6 @@ uint8_t rule_dpif_get_table(const struct rule_dpif *);
 bool table_is_internal(uint8_t table_id);
 
 const struct rule_actions *rule_dpif_get_actions(const struct rule_dpif *);
-uint32_t rule_dpif_get_recirc_id(struct rule_dpif *);
 void rule_set_recirc_id(struct rule *, uint32_t id);
 
 ovs_be64 rule_dpif_get_flow_cookie(const struct rule_dpif *rule);
@@ -156,83 +155,9 @@ void ofproto_dpif_flow_mod(struct ofproto_dpif *, struct ofputil_flow_mod *);
 struct rule_dpif *ofproto_dpif_refresh_rule(struct rule_dpif *);
 
 struct ofport_dpif *odp_port_to_ofport(const struct dpif_backer *, odp_port_t);
+struct ofport_dpif *ofp_port_to_ofport(const struct ofproto_dpif *,
+                                       ofp_port_t);
 
-/*
- * Recirculation
- * =============
- *
- * Recirculation is a technique to allow a frame to re-enter the packet
- * processing path for one or multiple times to achieve more flexible packet
- * processing in the data path. MPLS handling and selecting bond slave port
- * of a bond ports.
- *
- * Data path and user space interface
- * -----------------------------------
- *
- * Two new fields, recirc_id and dp_hash, are added to the current flow data
- * structure. They are both of type uint32_t. In addition, a new action,
- * RECIRC, are added.
- *
- * The value recirc_id is used to distinguish a packet from multiple
- * iterations of recirculation. A packet initially received is considered of
- * having recirc_id of 0. Recirc_id is managed by the user space, opaque to
- * the data path.
- *
- * On the other hand, dp_hash can only be computed by the data path, opaque to
- * the user space. In fact, user space may not able to recompute the hash
- * value. The dp_hash value should be wildcarded when for a newly received
- * packet. RECIRC action specifies whether the hash is computed. If computed,
- * how many fields to be included in the hash computation. The computed hash
- * value is stored into the dp_hash field prior to recirculation.
- *
- * The RECIRC action computes and set the dp_hash field, set the recirc_id
- * field and then reprocess the packet as if it was received on the same input
- * port. RECIRC action works like a function call; actions listed behind the
- * RECIRC action will be executed after its execution.  RECIRC action can be
- * nested, data path implementation limits the number of recirculation executed
- * to prevent unreasonable nesting depth or infinite loop.
- *
- * Both flow fields and the RECIRC action are exposed as OpenFlow fields via
- * Nicira extensions.
- *
- * Post recirculation flow
- * ------------------------
- *
- * At the OpenFlow level, post recirculation rules are always hidden from the
- * controller.  They are installed in table 254 which is set up as a hidden
- * table during boot time. Those rules are managed by the local user space
- * program only.
- *
- * To speed up the classifier look up process, recirc_id is always reflected
- * into the metadata field, since recirc_id is required to be exactly matched.
- *
- * Classifier look up always starts with table 254. A post recirculation flow
- * lookup should find its hidden rule within this table. On the other hand, A
- * newly received packet should miss all post recirculation rules because its
- * recirc_id is zero, then hit a pre-installed lower priority rule to redirect
- * classifier to look up starting from table 0:
- *
- *       * , actions=resubmit(,0)
- *
- * Post recirculation data path flows are managed like other data path flows.
- * They are created on demand. Miss handling, stats collection and revalidation
- * work the same way as regular flows.
- *
- * If the bridge which originates the recirculation is different from the bridge
- * that receives the post recirculation packet (e.g. when patch port is used),
- * the packet will be processed directly by the recirculation bridge with
- * in_port set to OFPP_NONE.  Admittedly, doing this limits the recirculation
- * bridge from matching on in_port of post recirculation packets, and will be
- * fixed in the near future.
- *
- * TODO: Always restore the correct in_port.
- *
- */
-
-struct ofproto_dpif *ofproto_dpif_recirc_get_ofproto(const struct dpif_backer *ofproto,
-                                                     uint32_t recirc_id);
-uint32_t ofproto_dpif_alloc_recirc_id(struct ofproto_dpif *ofproto);
-void ofproto_dpif_free_recirc_id(struct ofproto_dpif *ofproto, uint32_t recirc_id);
 int ofproto_dpif_add_internal_flow(struct ofproto_dpif *,
                                    const struct match *, int priority,
                                    uint16_t idle_timeout,
diff --git a/tests/mpls-xlate.at b/tests/mpls-xlate.at
index 47e6680..c293db7 100644
--- a/tests/mpls-xlate.at
+++ b/tests/mpls-xlate.at
@@ -31,8 +31,12 @@ dnl Setup multiple MPLS tags.
 AT_CHECK([ovs-ofctl del-flows br0])
 
 AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 in_port=local,dl_type=0x0800,action=push_mpls:0x8847,set_field:10-\>mpls_label,push_mpls:0x8847,set_field:20-\>mpls_label,output:1])
-AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 table=0,dl_type=0x8847,in_port=1,mpls_label=60,action=pop_mpls:0x8847,goto_table:1])
-AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 table=1,dl_type=0x8847,in_port=1,mpls_label=50,action=pop_mpls:0x0800,set_field:10-\>nw_ttl,output:LOCAL])
+# The resubmits will be executed after recirculation, which preserves the
+# register values.
+AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 cookie=0xa,table=0,dl_type=0x8847,in_port=1,mpls_label=60,action=set_field:10-\>reg0,pop_mpls:0x8847,goto_table:1])
+# The pop_mpls below recirculates from within a resubmit
+# After recirculation the (restored) register value is moved to IP ttl.
+AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 cookie=0xb,table=1,dl_type=0x8847,in_port=1,mpls_label=50,action=pop_mpls:0x0800,move:NXM_NX_REG0[[0..7]]-\>NXM_NX_IP_TTL[[]],output:LOCAL])
 
 dnl Double MPLS push
 AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(100),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x0800),ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
@@ -43,15 +47,15 @@ AT_CHECK([tail -1 stdout], [0],
 dnl Double MPLS pop
 AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x8847),mpls(label=60,tc=0/0,ttl=64,bos=0)' -generate], [0], [stdout])
 AT_CHECK([tail -1 stdout], [0],
-  [Datapath actions: pop_mpls(eth_type=0x8847),recirc(300)
+  [Datapath actions: pop_mpls(eth_type=0x8847),recirc(1)
 ])
 
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(300),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x8847),mpls(label=50,tc=0/0,ttl=64,bos=0)' -generate], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(1),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x8847),mpls(label=50,tc=0/0,ttl=64,bos=0)' -generate], [0], [stdout])
 AT_CHECK([tail -1 stdout], [0],
-  [Datapath actions: pop_mpls(eth_type=0x800),recirc(301)
+  [Datapath actions: pop_mpls(eth_type=0x800),recirc(2)
 ])
 
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(301),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x0800),ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)' -generate], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(2),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x0800),ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)' -generate], [0], [stdout])
 AT_CHECK([tail -1 stdout], [0],
   [Datapath actions: set(ipv4(ttl=10)),100
 ])
diff --git a/tests/ofproto-dpif.at b/tests/ofproto-dpif.at
index cdbd7fa..2b712d5 100644
--- a/tests/ofproto-dpif.at
+++ b/tests/ofproto-dpif.at
@@ -195,7 +195,7 @@ AT_CHECK([ovs-appctl revalidator/purge], [0])
 # Checks the flow stats in br1, should only be one flow with non-zero
 # 'n_packets' from internal table.
 AT_CHECK([ovs-appctl bridge/dump-flows br1 | ofctl_strip | grep -- "n_packets" | grep -- "table_id" | sed -e 's/dp_hash=0x[[0-9a-f]][[0-9a-f]]*/dp_hash=0x0/' -e 's/output:[[0-9]][[0-9]]*/output/'], [0], [dnl
-table_id=254, n_packets=1, n_bytes=64, priority=20,recirc_id=0x12d,dp_hash=0x0/0xff,actions=output
+table_id=254, n_packets=1, n_bytes=64, priority=20,recirc_id=0x2,dp_hash=0x0/0xff,actions=output
 ])
 
 # Checks the flow stats in br-int, should be only one match.
@@ -1526,7 +1526,8 @@ cookie=0xd dl_src=60:66:66:66:00:07 actions=pop_mpls:0x0800,learn(table=1,hard_t
 cookie=0xd dl_src=60:66:66:66:00:08 actions=pop_mpls:0x0806,resubmit(1,1)
 cookie=0xd table=1 arp actions=controller
 
-cookie=0xd dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48,controller
+cookie=0xdeadbeef table=2 dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48
+cookie=0xd dl_src=60:66:66:66:00:09 actions=resubmit(,2),controller
 cookie=0xd dl_src=60:66:66:66:00:0a actions=pop_mpls:0x0800,mod_nw_dst:10.0.0.1,controller
 cookie=0xd dl_src=60:66:66:66:00:0b actions=pop_mpls:0x0800,mod_nw_src:10.0.0.1,controller
 
@@ -1535,20 +1536,20 @@ cookie=0xd dl_src=60:66:66:66:01:01 actions=pop_mpls:0x8847,dec_mpls_ttl,control
 cookie=0xd dl_src=60:66:66:66:01:02 actions=pop_mpls:0x8848,load:3->OXM_OF_MPLS_TC[[]],controller
 
 cookie=0xd dl_src=60:66:66:66:02:00 actions=pop_mpls:0x8847,pop_mpls:0x0800,controller
-cookie=0xd dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,controller
-cookie=0xd dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,controller
+cookie=0xe dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,controller
+cookie=0xe dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,controller
 
-cookie=0xd dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,controller
-cookie=0xd dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,controller
-cookie=0xd dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0xe dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,controller
+cookie=0xe dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,controller
+cookie=0xe dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,controller
 
-cookie=0xd dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,controller
-cookie=0xd dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,controller
-cookie=0xd dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0xf dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,controller
+cookie=0xf dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0xf dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,controller
 
-cookie=0xd dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,controller
-cookie=0xd dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,controller
-cookie=0xd dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,controller
+cookie=0x5 dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,controller
+cookie=0x5 dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0x5 dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,controller
 ])
 AT_CHECK([ovs-ofctl add-flows br0 flows.txt])
 
@@ -1822,13 +1823,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -1848,13 +1849,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:02,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:02,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:02,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 ])
 
@@ -1874,13 +1875,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:03,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:03,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:03,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 ])
 
@@ -1900,13 +1901,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:04,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:04,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:04,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 ])
 
@@ -1926,13 +1927,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.106,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:76db
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.106,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:76db
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.106,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:76db
 ])
 
@@ -1952,13 +1953,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:06,dl_dst=50:54:00:00:00:07,nw_src=192.168.255.255,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7745
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:06,dl_dst=50:54:00:00:00:07,nw_src=192.168.255.255,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7745
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:06,dl_dst=50:54:00:00:00:07,nw_src=192.168.255.255,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7745
 ])
 
@@ -1978,13 +1979,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:07,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:07,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:07,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2028,13 +2029,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:09,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=48,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:09,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=48,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:09,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=48,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2054,13 +2055,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0a,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0a,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0a,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 ])
 
@@ -2080,13 +2081,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0b,dl_dst=50:54:00:00:00:07,nw_src=10.0.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2ded
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0b,dl_dst=50:54:00:00:00:07,nw_src=10.0.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2ded
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0b,dl_dst=50:54:00:00:00:07,nw_src=10.0.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2ded
 ])
 
@@ -2134,13 +2135,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:01:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:01:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:01:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 ])
 
@@ -2188,13 +2189,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:00,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:00,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:00,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2211,17 +2212,18 @@ AT_CHECK([ovs-ofctl monitor br0 65534 -P nxm --detach --pidfile 2> ofctl_monitor
 for i in 1 2 3; do
     ovs-appctl netdev-dummy/receive p1 '50 54 00 00 00 07 60 66 66 66 02 01 88 48 00 01 40 20 00 01 41 1f 45 00 00 2c 00 00 00 00 ff 06 3b 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45'
 done
+
 OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2229,7 +2231,7 @@ AT_CHECK([ovs-appctl time/warp 5000], [0], [ignore])
 
 dnl Modified MPLS pop action.
 dnl The input is a frame with two MPLS label stack entries which tcpdump -vve shows as:
-dnl 60:66:66:66:02:10 > 50:54:00:00:02:10, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
+dnl 60:66:66:66:02:10 > 50:54:00:00:00:07, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
 dnl		(label 20, exp 0, [S], ttl 31)
 dnl		(tos 0x0, ttl 254, id 0, offset 0, flags [none], proto TCP (6), length 44)
 dnl        192.168.0.1.80 > 192.168.0.2.0: Flags [none], cksum 0x7744 (correct), seq 42:46, win 10000, length 4
@@ -2242,13 +2244,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:10,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:10,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:10,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2256,7 +2258,7 @@ AT_CHECK([ovs-appctl time/warp 5000], [0], [ignore])
 
 dnl Modified MPLS pop action.
 dnl The input is a frame with three MPLS label stack entries which tcpdump -vve shows as:
-dnl 60:66:66:66:03:00 > 50:54:00:00:00:00, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
+dnl 60:66:66:66:03:00 > 50:54:00:00:00:07, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
 dnl		(label 20, exp 0, ttl 31)
 dnl		(label 20, exp 0, [S], ttl 30)
 dnl		(tos 0x0, ttl 254, id 0, offset 0, flags [none], proto TCP (6), length 44)
@@ -2270,13 +2272,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 ])
 
@@ -2298,13 +2300,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 ])
 
@@ -2326,13 +2328,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 ])
 
@@ -2352,13 +2354,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:00,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=255,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:00,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=255,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:00,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=255,mpls_bos=1
 ])
 
@@ -2378,13 +2380,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:01,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=254,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:01,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=254,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:01,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=254,mpls_bos=1
 ])
 
@@ -2404,13 +2406,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA  ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:10,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=253,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:10,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=253,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:10,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=253,mpls_bos=1
 ])
 
@@ -2426,17 +2428,18 @@ AT_CHECK([ovs-ofctl monitor br0 65534 -P nxm --detach --pidfile 2> ofctl_monitor
 for i in 1 2 3; do
     ovs-appctl netdev-dummy/receive p1 '50 54 00 00 00 07 60 66 66 66 05 00 88 47 00 01 41 20 45 00 00 2c 00 00 00 00 ff 06 3a 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45'
 done
+
 OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=32,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=32,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=32,mpls_bos=1
 ])
 
@@ -2456,13 +2459,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 ])
 
@@ -2478,22 +2481,26 @@ AT_CHECK([ovs-ofctl monitor br0 65534 -P nxm --detach --pidfile 2> ofctl_monitor
 for i in 1 2 3; do
     ovs-appctl netdev-dummy/receive p1 '50 54 00 00 00 07 60 66 66 66 05 10 88 47 00 01 41 20 45 00 00 2c 00 00 00 00 ff 06 3a 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45'
 done
+
 OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 ])
 
 AT_CHECK([ovs-appctl revalidator/purge], [0])
 AT_CHECK([ovs-ofctl dump-flows br0 | ofctl_strip | sort], [0], [dnl
+ cookie=0x5, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,CONTROLLER:65535
+ cookie=0x5, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0x5, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xa, n_packets=3, n_bytes=180, dl_src=40:44:44:44:44:42 actions=push_mpls:0x8847,load:0xa->OXM_OF_MPLS_LABEL[[]],load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
  cookie=0xa, n_packets=3, n_bytes=180, dl_src=40:44:44:44:44:43 actions=push_mpls:0x8847,load:0xa->OXM_OF_MPLS_LABEL[[]],load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
  cookie=0xa, n_packets=3, n_bytes=180, dl_src=40:44:44:44:44:44 actions=push_mpls:0x8847,load:0xa->OXM_OF_MPLS_LABEL[[]],load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
@@ -2513,26 +2520,24 @@ AT_CHECK([ovs-ofctl dump-flows br0 | ofctl_strip | sort], [0], [dnl
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:05 actions=pop_mpls:0x0800,multipath(eth_src,50,modulo_n,256,0,NXM_OF_IP_SRC[[0..7]]),CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:06 actions=pop_mpls:0x0800,bundle_load(eth_src,50,hrw,ofport,NXM_OF_IP_SRC[[0..15]],slaves:1,2),CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:07 actions=pop_mpls:0x0800,learn(table=1,hard_timeout=60,eth_type=0x800,nw_proto=6,NXM_OF_IP_SRC[[]]=NXM_OF_IP_DST[[]]),CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48,CONTROLLER:65535
+ cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:09 actions=resubmit(,2),CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:0a actions=pop_mpls:0x0800,mod_nw_dst:10.0.0.1,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:0b actions=pop_mpls:0x0800,mod_nw_src:10.0.0.1,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:66:66 actions=pop_mpls:0x0800,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:01:00 actions=pop_mpls:0x8848,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:01:01 actions=pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:01:02 actions=pop_mpls:0x8848,load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:00 actions=pop_mpls:0x8847,pop_mpls:0x0800,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xd, table=1, n_packets=3, n_bytes=168, arp actions=CONTROLLER:65535
+ cookie=0xdeadbeef, table=2, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48
+ cookie=0xe, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0xf, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,CONTROLLER:65535
+ cookie=0xf, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0xf, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
 NXST_FLOW reply:
 ])
 
@@ -2616,13 +2621,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-OFPT_PACKET_IN (OF1.2) (xid=0x0): table_id=254 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+OFPT_PACKET_IN (OF1.2) (xid=0x0): total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:08,dl_dst=50:54:00:00:00:01,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=32,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-OFPT_PACKET_IN (OF1.2) (xid=0x0): table_id=254 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+OFPT_PACKET_IN (OF1.2) (xid=0x0): total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:08,dl_dst=50:54:00:00:00:01,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=32,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-OFPT_PACKET_IN (OF1.2) (xid=0x0): table_id=254 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+OFPT_PACKET_IN (OF1.2) (xid=0x0): total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:08,dl_dst=50:54:00:00:00:01,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=32,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
-- 
1.7.10.4




More information about the dev mailing list