[ovs-dev] [PATCH v2 5/6] ofproto-dpif: Restore metadata and registers on recirculation.

Jarno Rajahalme jrajahalme at nicira.com
Tue Mar 10 22:43:25 UTC 2015


xlate_actions() now considers an optional recirculation context (via
'xin') and restores OpenFlow pipeline metadata based on it.  The
recirculation context is returned by the xlate_lookup() and may
contain an action set to be restored, and further actions to be
executed upon recirculation.

The pipeline metadata is restored using two new internal actions.  The
UNROLL_METADATA action stores the reference to the executing bridge
and all non-packet metadata in struct flow.  This is used to restore
metadata right after recirculation, and also after returning from a
patch port, if the patch port recirculated.  This is needed as
pipeline metadata restored for recirculation in this case belongs to a
different bridge than the bridge that initially received the packet.

The UNROLL_XLATE action stores the translation context data visible to
OpenFlow controllers via PACKET_IN messages.  This includes the
current table number and the current rule cookie.  UNROLL_XLATE
actions are inserted only when the remaining actions may generate
PACKET_IN messages.

These changes allow the post-MPLS recirculation to properly continue
with the pipeline metadata that existed at the time of recirculation.

The internal table is still consulted for bonds via explicit goto
table actions in the post-recirculation actions.  Changing bonds to
use this new mechanism is work in progress.

Signed-off-by: Jarno Rajahalme <jrajahalme at nicira.com>
---
 lib/ofp-actions.c             |   64 +++++
 lib/ofp-actions.h             |   60 +++++
 ofproto/bond.c                |    7 +-
 ofproto/ofproto-dpif-rid.c    |  331 +++++++++++++++++++++---
 ofproto/ofproto-dpif-rid.h    |  130 ++++++++--
 ofproto/ofproto-dpif-upcall.c |  116 +++++++--
 ofproto/ofproto-dpif-xlate.c  |  568 +++++++++++++++++++++++++++++++----------
 ofproto/ofproto-dpif-xlate.h  |   73 +++++-
 ofproto/ofproto-dpif.c        |  186 ++++----------
 ofproto/ofproto-dpif.h        |   87 +------
 tests/mpls-xlate.at           |   22 +-
 tests/ofproto-dpif.at         |  203 ++++++++-------
 12 files changed, 1321 insertions(+), 526 deletions(-)

diff --git a/lib/ofp-actions.c b/lib/ofp-actions.c
index 260d854..3740b6d 100644
--- a/lib/ofp-actions.c
+++ b/lib/ofp-actions.c
@@ -4237,6 +4237,56 @@ format_EXIT(const struct ofpact_null *a OVS_UNUSED, struct ds *s)
     ds_put_cstr(s, "exit");
 }
 
+/* Unroll metadata action. */
+
+static void
+encode_UNROLL_METADATA(const struct ofpact_unroll_metadata *unroll OVS_UNUSED,
+                       enum ofp_version ofp_version OVS_UNUSED,
+                       struct ofpbuf *out OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+}
+
+static char * OVS_WARN_UNUSED_RESULT
+parse_UNROLL_METADATA(char *arg OVS_UNUSED, struct ofpbuf *ofpacts OVS_UNUSED,
+                      enum ofputil_protocol *usable_protocols OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+    return NULL;
+}
+
+static void
+format_UNROLL_METADATA(const struct ofpact_unroll_metadata *a OVS_UNUSED,
+                       struct ds *s)
+{
+    ds_put_cstr(s, "unroll_metadata");
+}
+
+/* Unroll xlate action. */
+
+static void
+encode_UNROLL_XLATE(const struct ofpact_unroll_xlate *unroll OVS_UNUSED,
+                    enum ofp_version ofp_version OVS_UNUSED,
+                    struct ofpbuf *out OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+}
+
+static char * OVS_WARN_UNUSED_RESULT
+parse_UNROLL_XLATE(char *arg OVS_UNUSED, struct ofpbuf *ofpacts OVS_UNUSED,
+                   enum ofputil_protocol *usable_protocols OVS_UNUSED)
+{
+    OVS_NOT_REACHED();
+    return NULL;
+}
+
+static void
+format_UNROLL_XLATE(const struct ofpact_unroll_xlate *a OVS_UNUSED,
+                    struct ds *s)
+{
+    ds_put_cstr(s, "unroll_xlate");
+}
+
 /* Action structure for NXAST_SAMPLE.
  *
  * Samples matching packets with the given probability and sends them
@@ -4726,6 +4776,8 @@ ofpact_is_set_or_move_action(const struct ofpact *a)
     case OFPACT_DEC_TTL:
     case OFPACT_ENQUEUE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_FIN_TIMEOUT:
     case OFPACT_GOTO_TABLE:
     case OFPACT_GROUP:
@@ -4795,6 +4847,8 @@ ofpact_is_allowed_in_actions_set(const struct ofpact *a)
     case OFPACT_CONTROLLER:
     case OFPACT_ENQUEUE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_FIN_TIMEOUT:
     case OFPACT_LEARN:
     case OFPACT_CONJUNCTION:
@@ -5017,6 +5071,8 @@ ovs_instruction_type_from_ofpact_type(enum ofpact_type type)
     case OFPACT_MULTIPATH:
     case OFPACT_NOTE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_SAMPLE:
     default:
         return OVSINST_OFPIT11_APPLY_ACTIONS;
@@ -5607,6 +5663,12 @@ ofpact_check__(enum ofputil_protocol *usable_protocols, struct ofpact *a,
     case OFPACT_GROUP:
         return 0;
 
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
+        /* UNROLL is an internal action that should never be seen via
+         * OpenFlow. */
+        return OFPERR_OFPBAC_BAD_TYPE;
+
     default:
         OVS_NOT_REACHED();
     }
@@ -5998,6 +6060,8 @@ ofpact_outputs_to_port(const struct ofpact *ofpact, ofp_port_t port)
     case OFPACT_MULTIPATH:
     case OFPACT_NOTE:
     case OFPACT_EXIT:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_PUSH_MPLS:
     case OFPACT_POP_MPLS:
     case OFPACT_SAMPLE:
diff --git a/lib/ofp-actions.h b/lib/ofp-actions.h
index c232513..cc88d4a 100644
--- a/lib/ofp-actions.h
+++ b/lib/ofp-actions.h
@@ -105,6 +105,8 @@
     OFPACT(NOTE,            ofpact_note,        data, "note")           \
     OFPACT(EXIT,            ofpact_null,        ofpact, "exit")         \
     OFPACT(SAMPLE,          ofpact_sample,      ofpact, "sample")       \
+    OFPACT(UNROLL_METADATA, ofpact_unroll_metadata, ofpact, "unroll_metadata") \
+    OFPACT(UNROLL_XLATE,    ofpact_unroll_xlate, ofpact, "unroll_xlate") \
                                                                         \
     /* Instructions. */                                                 \
     OFPACT(METER,           ofpact_meter,       ofpact, "meter")        \
@@ -715,6 +717,64 @@ struct ofpact_group {
     uint32_t group_id;
 };
 
+/* OFPACT_UNROLL_METADATA.
+ *
+ * Used only internally.  Update when new metadata is added.  Helpers
+ * are inlined below to keep them together with the definition for easier
+ * updates. */
+BUILD_ASSERT_DECL(FLOW_WC_SEQ == 31);
+
+struct ofpact_unroll_metadata {
+    struct ofpact ofpact;
+
+    struct ofproto_dpif *ofproto; /* Bridge that tranlates any following
+                                   * actions. */
+
+    /* Metadata in struct flow. */
+    struct flow_tnl tunnel;       /* Encapsulating tunnel parameters. */
+    ovs_be64 metadata;            /* OpenFlow Metadata. */
+    uint64_t regs[FLOW_N_XREGS];  /* Registers. */
+    ofp_port_t in_port;           /* Incoming port. */
+    ofp_port_t actset_output;     /* Output port in action set. */
+};
+
+static inline void
+ofpact_unroll_metadata_from_flow(struct ofpact_unroll_metadata *unroll,
+                                 struct ofproto_dpif *ofproto,
+                                 const struct flow *flow)
+{
+    unroll->ofproto = ofproto;
+    unroll->tunnel = flow->tunnel;
+    unroll->metadata = flow->metadata;
+    memcpy(unroll->regs, flow->regs, sizeof unroll->regs);
+    unroll->in_port = flow->in_port.ofp_port;
+    unroll->actset_output = flow->actset_output;
+}
+
+static inline void
+ofpact_unroll_metadata_to_flow(const struct ofpact_unroll_metadata *unroll,
+                               struct ofproto_dpif **ofprotop,
+                               struct flow *flow)
+{
+    *ofprotop = unroll->ofproto;
+    flow->tunnel = unroll->tunnel;
+    flow->metadata = unroll->metadata;
+    memcpy(flow->regs, unroll->regs, sizeof flow->regs);
+    flow->in_port.ofp_port = unroll->in_port;
+    flow->actset_output = unroll->actset_output;
+}
+
+/* OFPACT_UNROLL_XLATE.
+ *
+ * Used only internally. */
+struct ofpact_unroll_xlate {
+    struct ofpact ofpact;
+
+    /* Metadata in xlate context, visible to controller via PACKET_INs. */
+    uint8_t  rule_table_id;       /* 0xFF if none. */
+    ovs_be64 rule_cookie;         /* OVS_BE64_MAX if none. */
+};
+
 /* Converting OpenFlow to ofpacts. */
 enum ofperr ofpacts_pull_openflow_actions(struct ofpbuf *openflow,
                                           unsigned int actions_len,
diff --git a/ofproto/bond.c b/ofproto/bond.c
index 7831fa4..2e3ad29 100644
--- a/ofproto/bond.c
+++ b/ofproto/bond.c
@@ -28,6 +28,7 @@
 #include "ofpbuf.h"
 #include "ofproto/ofproto-provider.h"
 #include "ofproto/ofproto-dpif.h"
+#include "ofproto/ofproto-dpif-rid.h"
 #include "connectivity.h"
 #include "coverage.h"
 #include "dynamic-string.h"
@@ -289,7 +290,7 @@ bond_unref(struct bond *bond)
     hmap_destroy(&bond->pr_rule_ops);
 
     if (bond->recirc_id) {
-        ofproto_dpif_free_recirc_id(bond->ofproto, bond->recirc_id);
+        recirc_free_id(bond->recirc_id);
     }
 
     free(bond);
@@ -446,10 +447,10 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s)
 
     if (bond->balance != BM_AB) {
         if (!bond->recirc_id) {
-            bond->recirc_id = ofproto_dpif_alloc_recirc_id(bond->ofproto);
+            bond->recirc_id = recirc_alloc_id(bond->ofproto);
         }
     } else if (bond->recirc_id) {
-        ofproto_dpif_free_recirc_id(bond->ofproto, bond->recirc_id);
+        recirc_free_id(bond->recirc_id);
         bond->recirc_id = 0;
     }
 
diff --git a/ofproto/ofproto-dpif-rid.c b/ofproto/ofproto-dpif-rid.c
index afad3ce..b9adf88 100644
--- a/ofproto/ofproto-dpif-rid.c
+++ b/ofproto/ofproto-dpif-rid.c
@@ -16,59 +16,326 @@
 
 #include <config.h>
 
-#include "id-pool.h"
-#include "ovs-thread.h"
+#include "ofproto-dpif.h"
 #include "ofproto-dpif-rid.h"
+#include "ofproto-provider.h"
+#include "openvswitch/vlog.h"
 
-struct recirc_id_pool {
-    struct ovs_mutex lock;
-    struct id_pool *rids;
-};
+VLOG_DEFINE_THIS_MODULE(ofproto_dpif_rid);
 
-#define RECIRC_ID_BASE  300
-#define RECIRC_ID_N_IDS  1024
+static struct ovs_mutex mutex;
 
-struct recirc_id_pool *
-recirc_id_pool_create(void)
+static struct cmap id_map;
+static struct cmap metadata_map;
+
+static struct ovs_list expiring OVS_GUARDED_BY(mutex);
+static struct ovs_list expired OVS_GUARDED_BY(mutex);
+
+static uint32_t next_id OVS_GUARDED_BY(mutex); /* Possible next free id. */
+
+#define RECIRC_POOL_STATIC_IDS 1024
+
+void
+recirc_init(void)
 {
-    struct recirc_id_pool *pool;
+    static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
 
-    pool = xmalloc(sizeof *pool);
-    pool->rids = id_pool_create(RECIRC_ID_BASE, RECIRC_ID_N_IDS);
-    ovs_mutex_init(&pool->lock);
+    if (ovsthread_once_start(&once)) {
+        ovs_mutex_init(&mutex);
+        ovs_mutex_lock(&mutex);
+        next_id = 1; /* 0 is not a valid ID. */
+        cmap_init(&id_map);
+        cmap_init(&metadata_map);
+        list_init(&expiring);
+        list_init(&expired);
+        ovs_mutex_unlock(&mutex);
+
+        ovsthread_once_done(&once);
+    }
 
-    return pool;
 }
 
+/* This should be called by the revalidator once at each round (every 500ms or
+ * more). */
 void
-recirc_id_pool_destroy(struct recirc_id_pool *pool)
+recirc_run(void)
+{
+    static long long int last = 0;
+    long long int now = time_msec();
+
+    /* Do maintenance at most 4 times / sec. */
+    ovs_mutex_lock(&mutex);
+    if (now - last > 250) {
+        struct recirc_id_node *node, *next;
+
+        last = now;
+
+        /* Nodes in 'expiring' and 'expired' lists have the refcount of zero,
+         * which means that while they can still be found (by id), no new
+         * references can be taken on them.  We have removed the entry from the
+         * 'metadata_map', at the time when refcount reached zero, causing any
+         * new translations to allocate a new ID.  This allows the expiring
+         * entry to be safely deleted while any sudden new use of the similar
+         * recirculation will safely start using a new recirculation ID.  When
+         * the refcount gets to zero, the node is also added to the 'expiring'
+         * list.  At any time after that the nodes in the 'expiring' list can
+         * be moved to the 'expired' list, from which they are deleted at least
+         * 250ms afterwards. */
+
+        /* Delete the expired.  These have been lingering for at least 250 ms,
+         * which should be enough for any ongoing recirculations to be
+         * finished. */
+        LIST_FOR_EACH_SAFE (node, next, exp_node, &expired) {
+            list_remove(&node->exp_node);
+            cmap_remove(&id_map, &node->id_node, node->id);
+            ovsrcu_postpone(free, node);
+        }
+
+        if (!list_is_empty(&expiring)) {
+            /* 'expired' is now empty, move nodes in 'expiring' to it. */
+            list_splice(&expired, list_front(&expiring), &expiring);
+        }
+    }
+    ovs_mutex_unlock(&mutex);
+}
+
+/* We use the id as the hash value, which works due to cmap internal rehashing.
+ * We also only insert nodes with unique IDs, so all possible hash collisions
+ * remain internal to the cmap. */
+static struct recirc_id_node *
+recirc_find__(uint32_t id)
+    OVS_REQUIRES(mutex)
 {
-    id_pool_destroy(pool->rids);
-    ovs_mutex_destroy(&pool->lock);
-    free(pool);
+    struct cmap_node *node = cmap_find_protected(&id_map, id);
+
+    return node ? CONTAINER_OF(node, struct recirc_id_node, id_node) : NULL;
 }
 
+/* Lockless RCU protected lookup.  If node is needed accross RCU quiescent
+ * state, caller should copy the contents. */
+const struct recirc_id_node *
+recirc_id_node_find(uint32_t id)
+{
+    const struct cmap_node *node = cmap_find(&id_map, id);
+
+    return node
+        ? CONTAINER_OF(node, const struct recirc_id_node, id_node)
+        : NULL;
+}
+
+static uint32_t
+recirc_metadata_hash(uint32_t action_set_len, uint32_t ofpacts_len,
+                     const struct ofpact *ofpacts)
+{
+    uint32_t hash;
+
+    BUILD_ASSERT(OFPACT_ALIGNTO == sizeof(uint64_t));
+
+    hash = hash_int(action_set_len, 0);
+    hash = hash_words64((const uint64_t *)ofpacts,
+                        OFPACT_ALIGN(ofpacts_len) / OFPACT_ALIGNTO, hash);
+    return hash;
+}
+
+static bool
+recirc_metadata_equal(const struct recirc_id_node *node,
+                      uint32_t action_set_len, uint32_t ofpacts_len,
+                      const struct ofpact *ofpacts)
+{
+    return node->action_set_len == action_set_len
+        && node->ofpacts_len == ofpacts_len
+        && !memcmp(node->ofpacts, ofpacts, ofpacts_len);
+}
+
+/* Lockless RCU protected lookup.  If node is needed accross RCU quiescent
+ * state, caller should take a reference. */
+static struct recirc_id_node *
+recirc_find_equal(uint32_t action_set_len, uint32_t ofpacts_len,
+                  const struct ofpact *ofpacts, uint32_t hash)
+{
+    struct recirc_id_node *node;
+
+    CMAP_FOR_EACH_WITH_HASH(node, metadata_node, hash, &metadata_map) {
+        if (recirc_metadata_equal(node, action_set_len, ofpacts_len,
+                                  ofpacts)) {
+            return node;
+        }
+    }
+    return NULL;
+}
+
+static struct recirc_id_node *
+recirc_ref_equal(uint32_t action_set_len, uint32_t ofpacts_len,
+                 const struct ofpact *ofpacts, uint32_t hash)
+{
+    struct recirc_id_node *node;
+
+    do {
+        node = recirc_find_equal(action_set_len, ofpacts_len, ofpacts, hash);
+
+        /* Try again if the node was released before we get the reference. */
+    } while (node && !ovs_refcount_try_ref_rcu(&node->refcount));
+
+    return node;
+}
+
+/* Allocate a unique recirculation id for the given set of flow metadata.
+ * The ID space is 2^^32, so there should never be a situation in which all
+ * the IDs are used up.  We loop until we find a free one.
+ * hash is recomputed if it is passed in as 0. */
+static struct recirc_id_node *
+recirc_alloc_id__(uint32_t action_set_len, uint32_t ofpacts_len,
+                  const struct ofpact *ofpacts, uint32_t hash)
+{
+    struct recirc_id_node *node = xzalloc(sizeof *node +
+                                          OFPACT_ALIGN(ofpacts_len));
+
+    /* All the data must be initialized before the node is made visible to
+     * lookups. */
+    ovs_refcount_init(&node->refcount);
+
+    node->action_set_len = action_set_len;
+    node->ofpacts_len = ofpacts_len;
+    memcpy(node->ofpacts, ofpacts, ofpacts_len);
+
+    node->hash = hash;
+    ovs_mutex_lock(&mutex);
+    for (;;) {
+        /* Claim the next ID.  The ID space should be sparse enough for the
+           allocation to succeed at the first try.  We do skip the first
+           RECIRC_POOL_STATIC_IDS IDs on the later rounds, though, as some of
+           the initial allocations may be for long term uses (like bonds). */
+        node->id = next_id++;
+        if (OVS_UNLIKELY(!node->id)) {
+            next_id = RECIRC_POOL_STATIC_IDS + 1;
+            node->id = next_id++;
+        }
+        /* Find if the id is free. */
+        if (OVS_LIKELY(!recirc_find__(node->id))) {
+            break;
+        }
+    }
+    cmap_insert(&id_map, &node->id_node, node->id);
+    cmap_insert(&metadata_map, &node->metadata_node, node->hash);
+    ovs_mutex_unlock(&mutex);
+    return node;
+}
+
+/* Look up an existing ID for the given flow's metadata and optional actions.
+ */
+uint32_t
+recirc_find_id(uint32_t action_set_len, uint32_t ofpacts_len,
+               const struct ofpact *ofpacts)
+{
+    /* Check if an ID with the given metadata already exists. */
+    struct recirc_id_node *node;
+    uint32_t hash;
+
+    hash = recirc_metadata_hash(action_set_len, ofpacts_len, ofpacts);
+    node = recirc_find_equal(action_set_len, ofpacts_len, ofpacts, hash);
+
+    return node ? node->id : 0;
+}
+
+/* Allocate a unique recirculation id for the given set of flow metadata and
+   optional actions. */
+uint32_t
+recirc_alloc_id_ctx(uint32_t action_set_len, uint32_t ofpacts_len,
+                    const struct ofpact *ofpacts)
+{
+    struct recirc_id_node *node;
+    uint32_t hash;
+
+    /* Look up an existing ID. */
+    hash = recirc_metadata_hash(action_set_len, ofpacts_len, ofpacts);
+    node = recirc_ref_equal(action_set_len, ofpacts_len, ofpacts, hash);
+
+    /* Allocate a new recirc ID if needed. */
+    if (!node) {
+        ovs_assert(action_set_len <= ofpacts_len);
+
+        node = recirc_alloc_id__(action_set_len, ofpacts_len, ofpacts, hash);
+    }
+
+    return node->id;
+}
+
+/* Allocate a unique recirculation id. */
 uint32_t
-recirc_id_alloc(struct recirc_id_pool *pool)
+recirc_alloc_id(struct ofproto_dpif *ofproto)
 {
-    uint32_t id;
-    bool ret;
+    struct {
+        struct ofpact_unroll_metadata unroll;
+        struct ofpact_goto_table goto_table;
+    } ofpacts;
+    struct recirc_id_node *node;
+    uint32_t hash;
 
-    ovs_mutex_lock(&pool->lock);
-    ret = id_pool_alloc_id(pool->rids, &id);
-    ovs_mutex_unlock(&pool->lock);
+    memset(&ofpacts, 0, sizeof ofpacts);
+    ofpact_init(&ofpacts.unroll.ofpact, OFPACT_UNROLL_METADATA,
+                sizeof ofpacts.unroll);
+    ofpacts.unroll.ofproto = ofproto;
+    ofpact_init(&ofpacts.goto_table.ofpact, OFPACT_GOTO_TABLE,
+                sizeof ofpacts.goto_table);
+    ofpacts.goto_table.table_id = TBL_INTERNAL;
 
-    if (!ret) {
-        return 0;
+    hash = recirc_metadata_hash(0, sizeof ofpacts, &ofpacts.unroll.ofpact);
+    node = recirc_alloc_id__(0, sizeof ofpacts, &ofpacts.unroll.ofpact, hash);
+
+    return node->id;
+}
+
+void
+recirc_id_node_unref(const struct recirc_id_node *node_)
+    OVS_EXCLUDED(mutex)
+{
+    struct recirc_id_node *node = CONST_CAST(struct recirc_id_node *, node_);
+
+    if (node && ovs_refcount_unref(&node->refcount) == 1) {
+        ovs_mutex_lock(&mutex);
+        /* Prevent re-use of this node by removing the node from 'metadata_map'
+         */
+        cmap_remove(&metadata_map, &node->metadata_node, node->hash);
+        /* We keep the node in the 'id_map' so that it can be found as long
+         * as it lingers, and add it to the 'expiring' list. */
+        list_insert(&expiring, &node->exp_node);
+        ovs_mutex_unlock(&mutex);
     }
+}
+
+void
+recirc_free_id(uint32_t id)
+{
+    const struct recirc_id_node *node;
 
-    return id;
+    node = recirc_id_node_find(id);
+    if (node) {
+        recirc_id_node_unref(node);
+    } else {
+        VLOG_ERR("Freeing non-exiting recirculation ID: %"PRIu32, id);
+    }
 }
 
+/* Called when 'ofproto' is destructed.  Checks for and clears any
+ * recirc_id leak.
+ * No other thread may have access to the 'ofproto' being destructed.
+ * All related datapath flows must be deleted before calling this. */
 void
-recirc_id_free(struct recirc_id_pool *pool, uint32_t id)
+recirc_free_ofproto(struct ofproto_dpif *ofproto, const char *ofproto_name)
 {
-    ovs_mutex_lock(&pool->lock);
-    id_pool_free_id(pool->rids, id);
-    ovs_mutex_unlock(&pool->lock);
+    struct recirc_id_node *n;
+
+    CMAP_FOR_EACH (n, id_node, &id_map) {
+        const struct ofpact *a = &n->ofpacts[n->action_set_len / sizeof *a];
+        size_t left = n->ofpacts_len - n->action_set_len;
+
+        OFPACT_FOR_EACH (a, a, left) {
+            if (a->type == OFPACT_UNROLL_METADATA
+                && ofpact_get_UNROLL_METADATA(a)->ofproto == ofproto) {
+                VLOG_ERR("recirc_id %"PRIu32" left allocated when ofproto (%s)"
+                         " is destructed", n->id, ofproto_name);
+                break;
+            }
+        }
+    }
 }
diff --git a/ofproto/ofproto-dpif-rid.h b/ofproto/ofproto-dpif-rid.h
index 3344e2a..3c90b8a 100644
--- a/ofproto/ofproto-dpif-rid.h
+++ b/ofproto/ofproto-dpif-rid.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Nicira, Inc.
+ * Copyright (c) 2014, 2015 Nicira, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,29 +20,127 @@
 #include <stddef.h>
 #include <stdint.h>
 
-struct recirc_id_pool;
+#include "cmap.h"
+#include "list.h"
+#include "ofp-actions.h"
+#include "ovs-thread.h"
+
+struct ofproto_dpif;
+struct rule;
 
 /*
- * Recirculation ID pool.
- * ======================
+ * Recirculation
+ * =============
+ *
+ * Recirculation is a technique to allow a frame to re-enter the datapath
+ * packet processing path for one or multiple times to achieve more flexible
+ * packet processing, such modifying header fields after MPLS POP action and
+ * selecting bond a slave port for bond ports.
+ *
+ * Data path and user space interface
+ * -----------------------------------
+ *
+ * Recirculation uses two uint32_t fields, recirc_id and dp_hash, and a RECIRC
+ * action.  The value recirc_id is used to select the next packet processing
+ * steps among multiple instances of recirculation.  When a packet initially
+ * enters the data path it is assigned with recirc_id 0, which indicates no
+ * recirculation.  Recirc_ids are managed by the user space, opaque to the
+ * data path.
+ *
+ * On the other hand, dp_hash can only be computed by the data path, opaque to
+ * the user space.  In fact, user space may not able to recompute the hash
+ * value.  The dp_hash value should be wildcarded for a newly received
+ * packet.  HASH action specifies whether the hash is computed, and if
+ * computed, how many fields are to be included in the hash computation.  The
+ * computed hash value is stored into the dp_hash field prior to recirculation.
+ *
+ * The RECIRC action sets the recirc_id field and then reprocesses the packet
+ * as if it was received on the same input port.  RECIRC action works like a
+ * function call; actions listed behind the RECIRC action will be executed
+ * after its execution.  RECIRC action can be nested, data path implementation
+ * limits the number of recirculation executed to prevent unreasonable nesting
+ * depth or infinite loop.
  *
- * Recirculation ID needs to be unique for each datapath. Recirculation
- * ID pool keeps track recirculation ids.
+ * User space recirculation context
+ * ---------------------------------
  *
- * Typically, there is one recirculation ID pool for each backer.
+ * Recirculation is hidden from the OpenFlow controllers.  Action translation
+ * code deduces when recirculation is necessary and issues a data path
+ * recirculation action.  All OpenFlow actions to be performed after
+ * recirculation are derived from the OpenFlow pipeline and are stored with the
+ * recirculation ID.  When the OpenFlow tables are changed in a way affecting
+ * the recirculation flows, new recirculation ID with new metadata and actions
+ * is allocated and the old one is timed out.
  *
- * In theory, Recirculation ID can be any uint32_t value, except 0.
- * The implementation usually limits it to a smaller range to ease
- * debugging.
+ * Recirculation ID pool
+ * ----------------------
+ *
+ * Recirculation ID needs to be unique for all data paths.  Recirculation ID
+ * pool keeps track recirculation ids and stores OpenFlow pipeline translation
+ * context so that flow processing may continue after recirculation.
+ *
+ * A Recirculation ID can be any uint32_t value, except for that the value 0 is
+ * reserved for 'no recirculation' case.
  *
  * Thread-safety
- * =============
+ * --------------
  *
  * All APIs are thread safe.
- *
  */
-struct recirc_id_pool *recirc_id_pool_create(void);
-void  recirc_id_pool_destroy(struct recirc_id_pool *pool);
-uint32_t recirc_id_alloc(struct recirc_id_pool *pool);
-void recirc_id_free(struct recirc_id_pool *pool, uint32_t recirc_id);
+
+
+/* Pool node fields should NOT be modified after placing the node in the pool.
+ */
+struct recirc_id_node {
+    struct ovs_list exp_node OVS_GUARDED;
+    struct cmap_node id_node;
+    struct cmap_node metadata_node;
+    uint32_t id;
+    uint32_t hash;
+    struct ovs_refcount refcount;
+
+    /* Actions to be translated on recirculation. */
+    uint32_t action_set_len;      /* How much of 'ofpacts' consists of an
+                                   * action set? */
+    uint32_t ofpacts_len;         /* Size of 'ofpacts', in bytes. */
+    struct ofpact ofpacts[];      /* Sequence of "struct ofpacts". */
+};
+
+void recirc_init(void);
+
+/* This is only used for bonds and will go away when bonds implementation is
+ * updated to use this mechanism instead of internal rules. */
+uint32_t recirc_alloc_id(struct ofproto_dpif *);
+
+uint32_t recirc_alloc_id_ctx(uint32_t action_set_len, uint32_t ofpacts_len,
+                             const struct ofpact *ofpacts);
+uint32_t recirc_find_id(uint32_t action_set_len, uint32_t ofpacts_len,
+                        const struct ofpact *ofpacts);
+void recirc_free_id(uint32_t recirc_id);
+void recirc_free_ofproto(struct ofproto_dpif *, const char *ofproto_name);
+
+const struct recirc_id_node *recirc_id_node_find(uint32_t recirc_id);
+
+static inline bool recirc_id_node_try_ref_rcu(const struct recirc_id_node *n_)
+{
+    struct recirc_id_node *node = CONST_CAST(struct recirc_id_node *, n_);
+
+    return node ? ovs_refcount_try_ref_rcu(&node->refcount) : false;
+}
+
+void recirc_id_node_unref(const struct recirc_id_node *);
+
+void recirc_run(void);
+
+static inline const struct ofpact_unroll_metadata *
+recirc_id_node_get_unroll_metadata(const struct recirc_id_node *n)
+{
+    const struct ofpact *a = &n->ofpacts[n->action_set_len / sizeof *a];
+
+    return (n->ofpacts_len
+            >= n->action_set_len + sizeof(struct ofpact_unroll_metadata))
+        && a->type == OFPACT_UNROLL_METADATA
+        ? ofpact_get_UNROLL_METADATA(a) : NULL;
+}
+
 #endif
diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c
index 3c80a0c..3ee438f 100644
--- a/ofproto/ofproto-dpif-upcall.c
+++ b/ofproto/ofproto-dpif-upcall.c
@@ -152,6 +152,8 @@ enum upcall_type {
 
 struct upcall {
     struct ofproto_dpif *ofproto;  /* Parent ofproto. */
+    const struct recirc_id_node *recirc; /* Recirculation context. */
+    bool have_recirc_ref;                /* Reference held on recirc ctx? */
 
     /* The flow and packet are only required to be constant when using
      * dpif-netdev.  If a modification is absolutely necessary, a const cast
@@ -229,6 +231,10 @@ struct udpif_key {
         struct odputil_keybuf buf;
         struct nlattr nla;
     } keybuf, maskbuf;
+
+    /* Recirculation IDs with references held by the ukey. */
+    unsigned n_recircs;
+    uint32_t recircs[];   /* 'n_recircs' id's for which references are held. */
 };
 
 /* Datapath operation with optional ukey attached. */
@@ -271,7 +277,7 @@ static void upcall_unixctl_dump_wait(struct unixctl_conn *conn, int argc,
 static void upcall_unixctl_purge(struct unixctl_conn *conn, int argc,
                                  const char *argv[], void *aux);
 
-static struct udpif_key *ukey_create_from_upcall(const struct upcall *);
+static struct udpif_key *ukey_create_from_upcall(struct upcall *);
 static int ukey_create_from_dpif_flow(const struct udpif *,
                                       const struct dpif_flow *,
                                       struct udpif_key **);
@@ -737,6 +743,8 @@ udpif_revalidator(void *arg)
         if (leader) {
             uint64_t reval_seq;
 
+            recirc_run(); /* Recirculation cleanup. */
+
             reval_seq = seq_read(udpif->reval_seq);
             last_reval_seq = reval_seq;
 
@@ -898,11 +906,24 @@ upcall_receive(struct upcall *upcall, const struct dpif_backer *backer,
     int error;
 
     error = xlate_lookup(backer, flow, &upcall->ofproto, &upcall->ipfix,
-                         &upcall->sflow, NULL, &upcall->in_port);
+                         &upcall->sflow, NULL, &upcall->in_port,
+                         &upcall->recirc);
     if (error) {
         return error;
     }
 
+    /* Try to get a reference only if we may install a flow based on this
+     * upcall. */
+    upcall->have_recirc_ref = false;
+    if (upcall->recirc && type == DPIF_UC_MISS) {
+        /* We may install a datapath flow only if we get a reference to the
+         * recirculation context (otherwise we could have recirculation upcalls
+         * using recirculation ID for which no context can be found).  We may
+         * still execute the flow's actions even if we don't install the
+         * flow. */
+        upcall->have_recirc_ref = recirc_id_node_try_ref_rcu(upcall->recirc);
+    }
+
     upcall->flow = flow;
     upcall->packet = packet;
     upcall->ufid = ufid;
@@ -937,14 +958,16 @@ upcall_xlate(struct udpif *udpif, struct upcall *upcall,
     stats.tcp_flags = ntohs(upcall->flow->tcp_flags);
 
     xlate_in_init(&xin, upcall->ofproto, upcall->flow, upcall->in_port, NULL,
-                  stats.tcp_flags, upcall->packet);
+                  stats.tcp_flags, upcall->packet, upcall->recirc);
     xin.odp_actions = odp_actions;
 
     if (upcall->type == DPIF_UC_MISS) {
         xin.resubmit_stats = &stats;
     } else {
-        /* For non-miss upcalls, there's a flow in the datapath which this
-         * packet was accounted to.  Presumably the revalidators will deal
+        /* For non-miss upcalls, we are either executing actions (one of which
+         * is an userspace action) for an upcall, in which case the stats have
+         * already been taken care of, or there's a flow in the datapath which
+         * this packet was accounted to.  Presumably the revalidators will deal
          * with pushing its stats eventually. */
     }
 
@@ -1005,8 +1028,13 @@ upcall_uninit(struct upcall *upcall)
             xlate_out_uninit(&upcall->xout);
         }
         ofpbuf_uninit(&upcall->put_actions);
-        if (!upcall->ukey_persists) {
-            ukey_delete__(upcall->ukey);
+        if (upcall->ukey) {
+            if (!upcall->ukey_persists) {
+                ukey_delete__(upcall->ukey);
+            }
+        } else if (upcall->have_recirc_ref) {
+            /* The reference was transferred to the ukey if one was created. */
+            recirc_id_node_unref(upcall->recirc);
         }
     }
 }
@@ -1056,10 +1084,16 @@ upcall_cb(const struct dp_packet *packet, const struct flow *flow, ovs_u128 *ufi
         goto out;
     }
 
-    if (upcall.ukey && !ukey_install(udpif, upcall.ukey)) {
+    /* Prevent miss flow installation if the key has recirculation ID but we
+     * were not able to get a reference on it. */
+    if (type == DPIF_UC_MISS && upcall.recirc && !upcall.have_recirc_ref) {
         error = ENOSPC;
+        goto out;
     }
 
+    if (upcall.ukey && !ukey_install(udpif, upcall.ukey)) {
+        error = ENOSPC;
+    }
 out:
     if (!error) {
         upcall.ukey_persists = true;
@@ -1189,8 +1223,12 @@ handle_upcalls(struct udpif *udpif, struct upcall *upcalls,
          *    - The datapath already has too many flows.
          *
          *    - We received this packet via some flow installed in the kernel
-         *      already. */
-        if (may_put && upcall->type == DPIF_UC_MISS) {
+         *      already.
+         *
+         *    - Upcall was a recirculation but we do not have a reference to
+         *      to the recirculation ID. */
+        if (may_put && upcall->type == DPIF_UC_MISS &&
+            (!upcall->recirc || upcall->have_recirc_ref)) {
             struct udpif_key *ukey = upcall->ukey;
 
             upcall->ukey_persists = true;
@@ -1277,10 +1315,13 @@ ukey_create__(const struct nlattr *key, size_t key_len,
               const struct nlattr *mask, size_t mask_len,
               bool ufid_present, const ovs_u128 *ufid,
               const int pmd_id, const struct ofpbuf *actions,
-              uint64_t dump_seq, uint64_t reval_seq, long long int used)
+              uint64_t dump_seq, uint64_t reval_seq, long long int used,
+              const struct recirc_id_node *key_recirc, struct xlate_out *xout)
     OVS_NO_THREAD_SAFETY_ANALYSIS
 {
-    struct udpif_key *ukey = xmalloc(sizeof *ukey);
+    unsigned n_recircs = (key_recirc ? 1 : 0) + (xout ? xout->n_recircs : 0);
+    struct udpif_key *ukey = xmalloc(sizeof *ukey +
+                                     n_recircs * sizeof *ukey->recircs);
 
     memcpy(&ukey->keybuf, key, key_len);
     ukey->key = &ukey->keybuf.nla;
@@ -1303,11 +1344,22 @@ ukey_create__(const struct nlattr *key, size_t key_len,
     ukey->stats.used = used;
     ukey->xcache = NULL;
 
+    ukey->n_recircs = n_recircs;
+    if (key_recirc) {
+        ukey->recircs[0] = key_recirc->id;
+    }
+    if (xout && xout->n_recircs) {
+        const uint32_t *act_recircs = xlate_out_get_recircs(xout);
+
+        memcpy(ukey->recircs + (key_recirc ? 1 : 0), act_recircs,
+               xout->n_recircs * sizeof *ukey->recircs);
+        xlate_out_take_recircs(xout);
+    }
     return ukey;
 }
 
 static struct udpif_key *
-ukey_create_from_upcall(const struct upcall *upcall)
+ukey_create_from_upcall(struct upcall *upcall)
 {
     struct odputil_keybuf keystub, maskstub;
     struct ofpbuf keybuf, maskbuf;
@@ -1337,7 +1389,9 @@ ukey_create_from_upcall(const struct upcall *upcall)
     return ukey_create__(keybuf.data, keybuf.size, maskbuf.data, maskbuf.size,
                          true, upcall->ufid, upcall->pmd_id,
                          &upcall->put_actions, upcall->dump_seq,
-                         upcall->reval_seq, 0);
+                         upcall->reval_seq, 0,
+                         upcall->have_recirc_ref ? upcall->recirc : NULL,
+                         &upcall->xout);
 }
 
 static int
@@ -1349,12 +1403,15 @@ ukey_create_from_dpif_flow(const struct udpif *udpif,
     struct ofpbuf actions;
     uint64_t dump_seq, reval_seq;
     uint64_t stub[DPIF_FLOW_BUFSIZE / 8];
+    const struct nlattr *a;
+    unsigned int left;
 
-    if (!flow->key_len) {
+    if (!flow->key_len || !flow->actions_len) {
         struct ofpbuf buf;
         int err;
 
-        /* If the key was not provided by the datapath, fetch the full flow. */
+        /* If the key or actions were not provided by the datapath, fetch the
+         * full flow. */
         ofpbuf_use_stack(&buf, &stub, sizeof stub);
         err = dpif_flow_get(udpif->dpif, NULL, 0, &flow->ufid,
                             flow->pmd_id, &buf, &full_flow);
@@ -1363,13 +1420,23 @@ ukey_create_from_dpif_flow(const struct udpif *udpif,
         }
         flow = &full_flow;
     }
+
+    /* Check the flow actions for recirculation action.  As recirculation
+     * relies on OVS userspace internal state, we need to delete all old
+     * datapath flows with recirculation upon OVS restart. */
+    NL_ATTR_FOR_EACH_UNSAFE (a, left, flow->actions, flow->actions_len) {
+        if (nl_attr_type(a) == OVS_ACTION_ATTR_RECIRC) {
+            return EINVAL;
+        }
+    }
+
     dump_seq = seq_read(udpif->dump_seq);
     reval_seq = seq_read(udpif->reval_seq);
     ofpbuf_use_const(&actions, &flow->actions, flow->actions_len);
     *ukey = ukey_create__(flow->key, flow->key_len,
                           flow->mask, flow->mask_len, flow->ufid_present,
                           &flow->ufid, flow->pmd_id, &actions, dump_seq,
-                          reval_seq, flow->stats.used);
+                          reval_seq, flow->stats.used, NULL, NULL);
 
     return 0;
 }
@@ -1512,6 +1579,9 @@ ukey_delete__(struct udpif_key *ukey)
     OVS_NO_THREAD_SAFETY_ANALYSIS
 {
     if (ukey) {
+        for (int i = 0; i < ukey->n_recircs; i++) {
+            recirc_free_id(ukey->recircs[i]);
+        }
         xlate_cache_delete(ukey->xcache);
         ofpbuf_delete(ukey->actions);
         ovs_mutex_destroy(&ukey->mutex);
@@ -1580,6 +1650,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey,
     size_t i;
     bool ok;
     bool need_revalidate;
+    const struct recirc_id_node *recirc;
 
     ok = false;
     xoutp = NULL;
@@ -1621,7 +1692,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey,
     }
 
     error = xlate_lookup(udpif->backer, &flow, &ofproto, NULL, NULL, &netflow,
-                         &ofp_in_port);
+                         &ofp_in_port, &recirc);
     if (error) {
         goto exit;
     }
@@ -1634,7 +1705,7 @@ revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey,
     }
 
     xlate_in_init(&xin, ofproto, &flow, ofp_in_port, NULL, push.tcp_flags,
-                  NULL);
+                  NULL, recirc);
     if (push.n_packets) {
         xin.resubmit_stats = &push;
         xin.may_learn = true;
@@ -1758,6 +1829,7 @@ push_ukey_ops__(struct udpif *udpif, struct ukey_op *ops, size_t n_ops)
             ofp_port_t ofp_in_port;
             struct flow flow;
             int error;
+            const struct recirc_id_node *recirc;
 
             if (op->ukey) {
                 ovs_mutex_lock(&op->ukey->mutex);
@@ -1776,13 +1848,13 @@ push_ukey_ops__(struct udpif *udpif, struct ukey_op *ops, size_t n_ops)
                 continue;
             }
 
-            error = xlate_lookup(udpif->backer, &flow, &ofproto,
-                                 NULL, NULL, &netflow, &ofp_in_port);
+            error = xlate_lookup(udpif->backer, &flow, &ofproto, NULL, NULL,
+                                 &netflow, &ofp_in_port, &recirc);
             if (!error) {
                 struct xlate_in xin;
 
                 xlate_in_init(&xin, ofproto, &flow, ofp_in_port, NULL,
-                              push->tcp_flags, NULL);
+                              push->tcp_flags, NULL, recirc);
                 xin.resubmit_stats = push->n_packets ? push : NULL;
                 xin.may_learn = push->n_packets > 0;
                 xin.skip_wildcards = true;
diff --git a/ofproto/ofproto-dpif-xlate.c b/ofproto/ofproto-dpif-xlate.c
index a1928cd..eb2669b 100644
--- a/ofproto/ofproto-dpif-xlate.c
+++ b/ofproto/ofproto-dpif-xlate.c
@@ -71,9 +71,6 @@ VLOG_DEFINE_THIS_MODULE(ofproto_dpif_xlate);
 #define MAX_INTERNAL_RESUBMITS 1   /* Max resbmits allowed using rules in
                                       internal table. */
 
-/* Timeout for internal rules created to handle recirculation */
-#define RECIRC_TIMEOUT 60
-
 /* Maximum number of resubmit actions in a flow translation, whether they are
  * recursive or not. */
 #define MAX_RESUBMITS (MAX_RESUBMIT_RECURSION * MAX_RESUBMIT_RECURSION)
@@ -208,9 +205,102 @@ struct xlate_ctx {
     uint16_t user_cookie_offset;/* Used for user_action_cookie fixup. */
     bool exit;                  /* No further actions should be processed. */
 
+   /* These are used for non-bond recirculation.  The recirculation IDs are
+    * stored in xout and must be associated with a datapath flow (ukey),
+    * otherwise they will be freed when the xout is uninitialized.
+    *
+    *
+    * Steps in Recirculation Translation
+    * ==================================
+    *
+    * At some point during translation, the code recognizes the need for
+    * recirculation.  For example, recirculation is necessary when, after
+    * popping the last MPLS label, an action or a match tries to examine or
+    * modify a field that has been newly revealed following the MPLS label.
+    *
+    * The simplest part of the work to be done is to commit existing changes to
+    * the packet, which produces datapath actions corresponding to the changes,
+    * and after this, add an OVS_ACTION_ATTR_RECIRC datapath action.
+    *
+    * The main problem here is preserving state.  When the datapath executes
+    * OVS_ACTION_ATTR_RECIRC, it will upcall to userspace to get a translation
+    * for the post-recirculation actions.  At this point userspace has to
+    * resume the translation where it left off, which means that it has to
+    * execute the following:
+    *
+    *     - The action that prompted recirculation, and any actions following
+    *       it within the same flow.
+    *
+    *     - If the action that prompted recirculation was invoked within a
+    *       NXAST_RESUBMIT, then any actions following the resubmit.  These
+    *       "resubmit"s can be nested, so this has to go all the way up the
+    *       control stack.
+    *
+    *     - The OpenFlow 1.1+ action set.
+    *
+    * State that actions and flow table lookups can depend on, such as the
+    * following, must also be preserved:
+    *
+    *     - Metadata fields (input port, registers, OF1.1+ metadata, ...).
+    *
+    *     - The table ID and cookie of the flow being translated at each level
+    *       of the control stack (since OFPAT_CONTROLLER actions send these to
+    *       the controller).
+    *
+    * Translation allows for this state preservation through these members and
+    * 'action_set'.  When a need for recirculation is identified, the
+    * translation process:
+    *
+    * 1. Sets 'recirc_action_offset' to the current size of 'action_set'.  The
+    *    action set is part of what needs to be preserved, so this allows the
+    *    action set and the additional state to share the 'action_set' buffer.
+    *    Later steps can tell that setup for recirculation is in progress from
+    *    the nonnegative value of 'recirc_action_offset'.
+    *
+    * 2. Sets 'exit' to true to tell later steps that we're exiting from the
+    *    translation process.
+    *
+    * 3. Adds an OFPACT_UNROLL_METADATA action to 'action_set'.  This action
+    *    holds the current values of metadata fields so that they can be
+    *    restored during a post-recirculation upcall translation.
+    *
+    * 4. Adds an OFPACT_UNROLL_XLATE action to 'action_set'.  This action
+    *    holds the current table ID and cookie so that they can be restored
+    *    during a post-recirculation upcall translation.
+    *
+    * 5. Adds the action that prompted recirculation and any actions following
+    *    it within the same flow to 'action_set', so that they can be executed
+    *    during a post-recirculation upcall translation.
+    *
+    * 6. Returns.
+    *
+    * 7. The action that prompted recirculation might be nested in a stack of
+    *    nested "resubmit"s that have actions remaining.  Each of these
+    *    notices that we're exiting (from 'exit') and that recirculation setup
+    *    is in progress (from 'recirc_action_offset') and responds by adding
+    *    more OFPACT_UNROLL_METADATA and OFPACT_UNROLL_XLATE actions to
+    *    'action_set', as necessary, and any actions that were yet
+    *    unprocessed.
+    *
+    * The caller stores all the state produced by this process associated with
+    * the recirculation ID.  For post-recirculation upcall translation, the
+    * caller passes it back in for the new translation to execute.  The
+    * process yielded a set of ofpacts that can be translated directly, so it
+    * is not much of a special case at that point.
+    */
+    int recirc_action_offset;   /* Offset in 'action_set' to actions to be
+                                 * executed after recirculation, or -1. */
+    int last_unroll_offset;     /* Offset in 'action_set' to the latest unroll
+                                 * action, or -1. */
+    bool metadata_unrolled;     /* Metadata may be unrolled at most once per
+                                 * bridge.  This is true when it is done. */
+
+    /* These are used only for bonds.  The recirculation ID is associated with
+     * an OpenFlow rule in the TBL_INTERNAL, and is deleted when that rule is
+     * deleted. */
     bool use_recirc;            /* Should generate recirc? */
-    struct xlate_recirc recirc; /* Information used for generating
-                                 * recirculation actions */
+    struct xlate_bond_recirc bond_recirc; /* Information used for generating
+                                           * bond recirculation actions */
 
     /* True if a packet was but is no longer MPLS (due to an MPLS pop action).
      * This is a trigger for recirculation in cases where translating an action
@@ -229,6 +319,14 @@ struct xlate_ctx {
     uint64_t action_set_stub[1024 / 8];
 };
 
+static inline bool
+exit_recirculates(const struct xlate_ctx *ctx)
+{
+    /* When recirculating the 'recirc_action_offset' has a non-negative value.
+     */
+    return ctx->recirc_action_offset >= 0;
+}
+
 /* A controller may use OFPP_NONE as the ingress port to indicate that
  * it did not arrive on a "real" port.  'ofpp_none_bundle' exists for
  * when an input bundle is needed for validation (e.g., mirroring or
@@ -975,64 +1073,56 @@ xlate_ofport_remove(struct ofport_dpif *ofport)
     xlate_xport_remove(new_xcfg, xport);
 }
 
-/* Given a datapath and flow metadata ('backer', and 'flow' respectively)
- * returns the corresponding struct xport, or NULL if none is found. */
-static struct xport *
-xlate_lookup_xport(const struct dpif_backer *backer, const struct flow *flow)
-{
-    struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
-
-    return xport_lookup(xcfg, tnl_port_should_receive(flow)
-                         ? tnl_port_receive(flow)
-                         : odp_port_to_ofport(backer, flow->in_port.odp_port));
-}
-
 static struct ofproto_dpif *
 xlate_lookup_ofproto_(const struct dpif_backer *backer, const struct flow *flow,
-                      ofp_port_t *ofp_in_port, const struct xport **xportp)
+                      ofp_port_t *ofp_in_port, const struct xport **xportp,
+                      const struct recirc_id_node **recirc_node)
 {
-    struct ofproto_dpif *recv_ofproto = NULL;
-    struct ofproto_dpif *recirc_ofproto = NULL;
+    struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
+    struct ofproto_dpif *ofproto;
     const struct xport *xport;
-    ofp_port_t in_port = OFPP_NONE;
+    ofp_port_t in_port;
+    const struct recirc_id_node *rid;
 
-    *xportp = xport = xlate_lookup_xport(backer, flow);
+    if (flow->recirc_id) {
+        const struct ofpact_unroll_metadata *unroll;
 
-    if (xport) {
-        recv_ofproto = xport->xbridge->ofproto;
-        in_port = xport->ofp_port;
-    }
+        rid = recirc_id_node_find(flow->recirc_id);
+        if (OVS_UNLIKELY(!rid)) {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
 
-    /* When recirc_id is set in 'flow', checks whether the ofproto_dpif that
-     * corresponds to the recirc_id is same as the receiving bridge.  If they
-     * are the same, uses the 'recv_ofproto' and keeps the 'ofp_in_port' as
-     * assigned.  Otherwise, uses the 'recirc_ofproto' that owns recirc_id and
-     * assigns OFPP_NONE to 'ofp_in_port'.  Doing this is in that, the
-     * recirculated flow must be processced by the ofproto which originates
-     * the recirculation, and as bridges can only see their own ports, the
-     * in_port of the 'recv_ofproto' should not be passed to the
-     * 'recirc_ofproto'.
-     *
-     * Admittedly, setting the 'ofp_in_port' to OFPP_NONE limits the
-     * 'recirc_ofproto' from meaningfully matching on in_port of recirculated
-     * flow, and should be fixed in the near future.
-     *
-     * TODO: Restore the original patch port.
-     */
-    if (recv_ofproto && flow->recirc_id) {
-        recirc_ofproto = ofproto_dpif_recirc_get_ofproto(backer,
-                                                         flow->recirc_id);
-        if (recv_ofproto != recirc_ofproto) {
-            *xportp = xport = NULL;
-            in_port = OFPP_NONE;
+            VLOG_WARN_RL(&rl, "Recirculation context not found for ID %"PRIx32,
+                         flow->recirc_id);
+            return NULL;
         }
+        if (recirc_node) {
+            *recirc_node = rid;
+        }
+        unroll = recirc_id_node_get_unroll_metadata(rid);
+
+        ofproto = unroll->ofproto;
+        in_port = unroll->in_port;
+        xport = xport_lookup(xcfg, ofp_port_to_ofport(ofproto, in_port));
+    } else {
+        if (recirc_node) {
+            *recirc_node = NULL;
+        }
+        xport = xport_lookup(xcfg, tnl_port_should_receive(flow)
+                             ? tnl_port_receive(flow)
+                             : odp_port_to_ofport(backer,
+                                                  flow->in_port.odp_port));
+        if (OVS_UNLIKELY(!xport)) {
+            return NULL;
+        }
+        in_port = xport->ofp_port;
+        ofproto = xport->xbridge->ofproto;
     }
 
+    *xportp = xport;
     if (ofp_in_port) {
         *ofp_in_port = in_port;
     }
-
-    return xport ? recv_ofproto : recirc_ofproto;
+    return ofproto;
 }
 
 /* Given a datapath and flow metadata ('backer', and 'flow' respectively)
@@ -1043,7 +1133,7 @@ xlate_lookup_ofproto(const struct dpif_backer *backer, const struct flow *flow,
 {
     const struct xport *xport;
 
-    return xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport);
+    return xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport, NULL);
 }
 
 /* Given a datapath and flow metadata ('backer', and 'flow' respectively),
@@ -1053,18 +1143,25 @@ xlate_lookup_ofproto(const struct dpif_backer *backer, const struct flow *flow,
  * pointers until quiescing, for longer term use additional references must
  * be taken.
  *
+ * The 'recirc' parameter is used to return a pointer to a recirculation
+ * context if the 'flow' has a non-zero 'recirc_id' and a context for that id
+ * is found, NULL otherwise.  The returned recirculation context is RCU
+ * protected and may not be used past the point in which the calling thread
+ * quiesces, unless the calling thread successfully acquires a reference (see
+ * recirc_id_node_try_ref_rcu()).
+ *
  * Returns 0 if successful, ENODEV if the parsed flow has no associated ofproto.
  */
 int
 xlate_lookup(const struct dpif_backer *backer, const struct flow *flow,
              struct ofproto_dpif **ofprotop, struct dpif_ipfix **ipfix,
              struct dpif_sflow **sflow, struct netflow **netflow,
-             ofp_port_t *ofp_in_port)
+             ofp_port_t *ofp_in_port, const struct recirc_id_node **recirc)
 {
     struct ofproto_dpif *ofproto;
     const struct xport *xport;
 
-    ofproto = xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport);
+    ofproto = xlate_lookup_ofproto_(backer, flow, ofp_in_port, &xport, recirc);
 
     if (!ofproto) {
         return ENODEV;
@@ -1680,7 +1777,7 @@ output_normal(struct xlate_ctx *ctx, const struct xbundle *out_xbundle,
     } else {
         struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
         struct flow_wildcards *wc = &ctx->xout->wc;
-        struct xlate_recirc *xr = &ctx->recirc;
+        struct xlate_bond_recirc *xr = &ctx->bond_recirc;
         struct ofport_dpif *ofport;
 
         if (ctx->xbridge->enable_recirc) {
@@ -2730,7 +2827,6 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
         const struct xport *peer = xport->peer;
         struct flow old_flow = ctx->xin->flow;
         enum slow_path_reason special;
-        uint8_t table_id = rule_dpif_lookup_get_init_table_id(&ctx->xin->flow);
 
         ctx->xbridge = peer->xbridge;
         flow->in_port.ofp_port = peer->ofp_port;
@@ -2745,16 +2841,15 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
             ctx->xout->slow |= special;
         } else if (may_receive(peer, ctx)) {
             if (xport_stp_forward_state(peer) && xport_rstp_forward_state(peer)) {
-                xlate_table_action(ctx, flow->in_port.ofp_port, table_id,
-                                   true, true);
+                xlate_table_action(ctx, flow->in_port.ofp_port, 0, true, true);
             } else {
                 /* Forwarding is disabled by STP and RSTP.  Let OFPP_NORMAL and
                  * the learning action look at the packet, then drop it. */
                 struct flow old_base_flow = ctx->base_flow;
                 size_t old_size = ctx->xout->odp_actions->size;
                 mirror_mask_t old_mirrors = ctx->xout->mirrors;
-                xlate_table_action(ctx, flow->in_port.ofp_port, table_id,
-                                   true, true);
+
+                xlate_table_action(ctx, flow->in_port.ofp_port, 0, true, true);
                 ctx->xout->mirrors = old_mirrors;
                 ctx->base_flow = old_base_flow;
                 ctx->xout->odp_actions->size = old_size;
@@ -2779,6 +2874,10 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
             entry->u.dev.rx = netdev_ref(peer->netdev);
             entry->u.dev.bfd = bfd_ref(peer->bfd);
         }
+
+        /* Trigger new metadata unroll in the bridge we returned to. */
+        ctx->metadata_unrolled = false;
+
         return;
     }
 
@@ -2851,7 +2950,7 @@ compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
 
         if (ctx->use_recirc) {
             struct ovs_action_hash *act_hash;
-            struct xlate_recirc *xr = &ctx->recirc;
+            struct xlate_bond_recirc *xr = &ctx->bond_recirc;
 
             /* Hash action. */
             act_hash = nl_msg_put_unspec_uninit(ctx->xout->odp_actions,
@@ -3263,68 +3362,40 @@ execute_controller_action(struct xlate_ctx *ctx, int len,
     dp_packet_delete(packet);
 }
 
+/* Called only when ctx->recirc_action_offset is set. */
 static void
-compose_recirculate_action(struct xlate_ctx *ctx,
-                           const struct ofpact *ofpacts_base,
-                           const struct ofpact *ofpact_current,
-                           size_t ofpacts_base_len)
+compose_recirculate_action(struct xlate_ctx *ctx)
 {
     uint32_t id;
-    int error;
-    unsigned ofpacts_len;
-    struct match match;
-    struct rule *rule;
-    struct ofpbuf ofpacts;
-
-    ctx->exit = true;
-
-    ofpacts_len = ofpacts_base_len -
-        ((uint8_t *)ofpact_current - (uint8_t *)ofpacts_base);
-
-    if (ctx->rule) {
-        id = rule_dpif_get_recirc_id(ctx->rule);
-    } else {
-        /* In the case where ctx has no rule then allocate a recirc id.
-         * The life-cycle of this recirc id is managed by associating it
-         * with the internal rule that is created to to handle
-         * recirculation below.
-         *
-         * The known use-case of this is packet_out which
-         * translates actions without a rule */
-        id = ofproto_dpif_alloc_recirc_id(ctx->xbridge->ofproto);
-    }
-    if (!id) {
-        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
-        VLOG_ERR_RL(&rl, "Failed to allocate recirculation id");
-        return;
-    }
-
-    match_init_catchall(&match);
-    match_set_recirc_id(&match, id);
-    ofpbuf_use_const(&ofpacts, ofpact_current, ofpacts_len);
-    error = ofproto_dpif_add_internal_flow(ctx->xbridge->ofproto, &match,
-                                           RECIRC_RULE_PRIORITY,
-                                           RECIRC_TIMEOUT, &ofpacts, &rule);
-    if (error) {
-        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
-        VLOG_ERR_RL(&rl, "Failed to add post recirculation flow %s",
-                    match_to_string(&match, 0));
-        if (!ctx->rule) {
-            ofproto_dpif_free_recirc_id(ctx->xbridge->ofproto, id);
-        }
-        return;
-    }
-    /* If ctx has no rule then associate the recirc id, which
-     * was allocated above, with the internal rule. This allows
-     * the recirc id to be released when the internal rule times out. */
-    if (!ctx->rule) {
-        rule_set_recirc_id(rule, id);
-    }
 
     ctx->xout->slow |= commit_odp_actions(&ctx->xin->flow, &ctx->base_flow,
                                           ctx->xout->odp_actions,
                                           &ctx->xout->wc,
                                           ctx->xbridge->masked_set_action);
+
+    /* Only allocate recirculation ID if we have a packet. */
+    if (ctx->xin->packet) {
+        /* Allocate a unique recirc id for the given metadata state in the
+         * flow.  The life-cycle of this recirc id is managed by associating it
+         * with the udpif key ('ukey') created for each new datapath flow. */
+        id = recirc_alloc_id_ctx(ctx->recirc_action_offset,
+                                 ctx->action_set.size, ctx->action_set.data);
+        if (!id) {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
+            VLOG_ERR_RL(&rl, "Failed to allocate recirculation id");
+            return;
+        }
+        xlate_out_add_recirc(ctx->xout, id);
+    } else {
+        /* Look up an existing recirc id for the given metadata state in the
+         * flow.  No new reference is taken, as the ID is RCU protected and is
+         * only required temporarily for verification. */
+        id = recirc_find_id(ctx->recirc_action_offset,
+                            ctx->action_set.size, ctx->action_set.data);
+        /* We let zero 'id' to be used in the RECIRC action below, which will
+         * fail all revalidations as zero is not a valid recirculation ID. */
+    }
+
     nl_msg_put_u32(ctx->xout->odp_actions, OVS_ACTION_ATTR_RECIRC, id);
 }
 
@@ -3804,6 +3875,8 @@ ofpact_needs_recirculation_after_mpls(const struct ofpact *a, struct xlate_ctx *
     case OFPACT_POP_QUEUE:
     case OFPACT_CONJUNCTION:
     case OFPACT_NOTE:
+    case OFPACT_UNROLL_METADATA:
+    case OFPACT_UNROLL_XLATE:
     case OFPACT_OUTPUT_REG:
     case OFPACT_EXIT:
     case OFPACT_METER:
@@ -3857,6 +3930,132 @@ ofpact_needs_recirculation_after_mpls(const struct ofpact *a, struct xlate_ctx *
 }
 
 static void
+recirc_put_unroll_metadata(struct xlate_ctx *ctx)
+{
+
+    /* Returning from a patch port requires restoring of the metadata
+     * of the bridge we return to.  We do it here only when needed. */
+    if (!ctx->metadata_unrolled) {
+        /* Insert the pipeline context of the current bridge before any
+         * actions that depend on or may modify any of the metedata. */
+        ofpact_unroll_metadata_from_flow(
+            ofpact_put_UNROLL_METADATA(&ctx->action_set),
+            ctx->xbridge->ofproto, &ctx->xin->flow);
+        ctx->metadata_unrolled = true;
+    }
+}
+
+static void
+recirc_put_unroll_xlate(struct xlate_ctx *ctx)
+{
+    struct ofpact_unroll_xlate *unroll;
+
+    unroll = ctx->last_unroll_offset < 0
+        ? NULL
+        : ALIGNED_CAST(struct ofpact_unroll_xlate *,
+                       (char *)ctx->action_set.data + ctx->last_unroll_offset);
+
+    /* Restore the table_id and rule cookie for a potential PACKET
+     * IN if needed. */
+    if (!unroll ||
+        (ctx->table_id != unroll->rule_table_id
+         || ctx->rule_cookie != unroll->rule_cookie)) {
+
+        ctx->last_unroll_offset = ctx->action_set.size;
+        unroll = ofpact_put_UNROLL_XLATE(&ctx->action_set);
+        unroll->rule_table_id = ctx->table_id;
+        unroll->rule_cookie = ctx->rule_cookie;
+    }
+}
+
+
+/* Copy remaining actions to the action_set to be executed after recirculation.
+ * UNROLL_METADATA action is inserted, if not already done, before any action
+ * that depends or may modify any flow metadata.  UNROLL_XLATE action is
+ * inserted, if not already done so, before actions that may generate
+ * PACKET_INs from the current table and without matching another rule. */
+static void
+recirc_unroll_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
+                      struct xlate_ctx *ctx)
+{
+    const struct ofpact *a;
+
+    OFPACT_FOR_EACH (a, ofpacts, ofpacts_len) {
+        switch (a->type) {
+            /* May depend on or modify packet metadata. */
+        case OFPACT_SET_TUNNEL:
+        case OFPACT_REG_MOVE:
+        case OFPACT_SET_FIELD:
+        case OFPACT_STACK_PUSH:
+        case OFPACT_STACK_POP:
+        case OFPACT_LEARN:
+        case OFPACT_WRITE_METADATA:
+        case OFPACT_RESUBMIT:        /* May indirectly generate PACKET INs, */
+        case OFPACT_GOTO_TABLE:      /* but from a different table and rule. */
+            recirc_put_unroll_metadata(ctx);
+            break;
+
+            /* May generate PACKET IN and may depend on metadata. */
+        case OFPACT_OUTPUT_REG:
+        case OFPACT_GROUP:
+            recirc_put_unroll_metadata(ctx);
+            recirc_put_unroll_xlate(ctx);
+            break;
+
+            /* May generate PACKET IN, but may not modify metadata. */
+        case OFPACT_OUTPUT:
+        case OFPACT_CONTROLLER:
+        case OFPACT_DEC_MPLS_TTL:
+        case OFPACT_DEC_TTL:
+            recirc_put_unroll_xlate(ctx);
+            break;
+
+            /* These do not depend on or modify metadata nor may they generate
+             * PACKET INs. */
+        case OFPACT_ENQUEUE:
+        case OFPACT_SET_VLAN_VID:
+        case OFPACT_SET_VLAN_PCP:
+        case OFPACT_STRIP_VLAN:
+        case OFPACT_PUSH_VLAN:
+        case OFPACT_SET_ETH_SRC:
+        case OFPACT_SET_ETH_DST:
+        case OFPACT_SET_IPV4_SRC:
+        case OFPACT_SET_IPV4_DST:
+        case OFPACT_SET_IP_DSCP:
+        case OFPACT_SET_IP_ECN:
+        case OFPACT_SET_IP_TTL:
+        case OFPACT_SET_L4_SRC_PORT:
+        case OFPACT_SET_L4_DST_PORT:
+        case OFPACT_SET_QUEUE:
+        case OFPACT_POP_QUEUE:
+        case OFPACT_PUSH_MPLS:
+        case OFPACT_POP_MPLS:
+        case OFPACT_SET_MPLS_LABEL:
+        case OFPACT_SET_MPLS_TC:
+        case OFPACT_SET_MPLS_TTL:
+        case OFPACT_MULTIPATH:
+        case OFPACT_BUNDLE:
+        case OFPACT_EXIT:
+        case OFPACT_UNROLL_METADATA:
+        case OFPACT_UNROLL_XLATE:
+        case OFPACT_FIN_TIMEOUT:
+        case OFPACT_CLEAR_ACTIONS:
+        case OFPACT_WRITE_ACTIONS:
+        case OFPACT_METER:
+        case OFPACT_SAMPLE:
+            break;
+
+            /* These need not be copied for restoration. */
+        case OFPACT_NOTE:
+        case OFPACT_CONJUNCTION:
+            continue;
+        }
+        /* Copy the action over. */
+        ofpbuf_put(&ctx->action_set, a, OFPACT_ALIGN(a->len));
+    }
+}
+
+static void
 do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
                  struct xlate_ctx *ctx)
 {
@@ -3875,13 +4074,27 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
         const struct ofpact_set_field *set_field;
         const struct mf_field *mf;
 
-        if (ctx->exit) {
-            break;
+        if (!ctx->exit && ofpact_needs_recirculation_after_mpls(a, ctx)) {
+            ctx->recirc_action_offset = ctx->action_set.size;
+
+            /* Insert the current pipeline context as an unroll action right
+             * after the action set, at the 'recirc_action_offset'.
+             * Post-recirculation processing depends on this being always
+             * inserted, and as the first post-recirculation action. */
+            recirc_put_unroll_metadata(ctx);
+            ctx->exit = true;
         }
 
-        if (ofpact_needs_recirculation_after_mpls(a, ctx)) {
-            compose_recirculate_action(ctx, ofpacts, a, ofpacts_len);
-            return;
+        if (ctx->exit) {
+            /* Check if need to store the remaining actions for later
+             * execution. */
+            if (exit_recirculates(ctx)) {
+                recirc_unroll_actions(a, OFPACT_ALIGN(ofpacts_len -
+                                                      ((uint8_t *)a -
+                                                       (uint8_t *)ofpacts)),
+                                      ctx);
+            }
+            break;
         }
 
         switch (a->type) {
@@ -4130,6 +4343,36 @@ do_xlate_actions(const struct ofpact *ofpacts, size_t ofpacts_len,
             ctx->exit = true;
             break;
 
+        case OFPACT_UNROLL_METADATA: {
+            struct xlate_cfg *xcfg = ovsrcu_get(struct xlate_cfg *, &xcfgp);
+            struct ofproto_dpif *ofproto;
+            const struct xbridge *xbridge;
+
+            /* Restore pipeline metadata that was stored earlier. */
+            ofpact_unroll_metadata_to_flow(ofpact_get_UNROLL_METADATA(a),
+                                           &ofproto, flow);
+            xbridge = xbridge_lookup(xcfg, ofproto);
+            if (OVS_UNLIKELY(!xbridge)) {
+                /* Drop the packet if the bridge cannot be found. */
+                static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+
+                VLOG_WARN_RL(&rl,
+                             "Recirculation after bridge has been deleted.");
+                ctx->exit = true;
+                ctx->xout->odp_actions->size = 0;
+            } else {
+                ctx->xbridge = xbridge;
+            }
+            break;
+        }
+        case OFPACT_UNROLL_XLATE: {
+            struct ofpact_unroll_xlate *unroll = ofpact_get_UNROLL_XLATE(a);
+
+            /* Restore translation context data that was stored earlier. */
+            ctx->table_id = unroll->rule_table_id;
+            ctx->rule_cookie = unroll->rule_cookie;
+            break;
+        }
         case OFPACT_FIN_TIMEOUT:
             memset(&wc->masks.nw_proto, 0xff, sizeof wc->masks.nw_proto);
             ctx->xout->has_fin_timeout = true;
@@ -4181,7 +4424,8 @@ void
 xlate_in_init(struct xlate_in *xin, struct ofproto_dpif *ofproto,
               const struct flow *flow, ofp_port_t in_port,
               struct rule_dpif *rule, uint16_t tcp_flags,
-              const struct dp_packet *packet)
+              const struct dp_packet *packet,
+              const struct recirc_id_node *recirc)
 {
     xin->ofproto = ofproto;
     xin->flow = *flow;
@@ -4199,13 +4443,17 @@ xlate_in_init(struct xlate_in *xin, struct ofproto_dpif *ofproto,
     xin->resubmit_stats = NULL;
     xin->skip_wildcards = false;
     xin->odp_actions = NULL;
+    xin->recirc = recirc;
 }
 
 void
 xlate_out_uninit(struct xlate_out *xout)
 {
-    if (xout && xout->odp_actions == &xout->odp_actions_buf) {
-        ofpbuf_uninit(xout->odp_actions);
+    if (xout) {
+        if (xout->odp_actions == &xout->odp_actions_buf) {
+            ofpbuf_uninit(xout->odp_actions);
+        }
+        xlate_out_free_recircs(xout);
     }
 }
 
@@ -4367,11 +4615,14 @@ too_many_output_actions(const struct ofpbuf *odp_actions OVS_UNUSED)
 #endif
 }
 
-/* Translates the 'ofpacts_len' bytes of "struct ofpacts" starting at 'ofpacts'
- * into datapath actions in 'odp_actions', using 'ctx'.
+/* Translates the flow, actions, or rule in 'xin' into datapath actions in
+ * 'xout', using recirculation context 'recirc'.  The recirculation context
+ * should be NULL for packets that were not recirculated.  The metadata and
+ * actions in 'recirc' will be used to continue processing from the pipeline
+ * state that was in effect when the recircualtion action was issued.
  *
  * The caller must take responsibility for eventually freeing 'xout', with
- * xlate_out_uninit(). */
+ * xlate_out_uninit().  'recirc' is RCU protected. */
 void
 xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
 {
@@ -4420,6 +4671,7 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
     ctx.xout->has_fin_timeout = false;
     ctx.xout->nf_output_iface = NF_OUT_DROP;
     ctx.xout->mirrors = 0;
+    ctx.xout->n_recircs = 0;
 
     xout->odp_actions = xin->odp_actions;
     if (!xout->odp_actions) {
@@ -4470,6 +4722,49 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
     ctx.exit = false;
     ctx.use_recirc = false;
     ctx.was_mpls = false;
+    ctx.recirc_action_offset = -1;
+    ctx.last_unroll_offset = -1;
+    ctx.metadata_unrolled = false;
+
+    ctx.action_set_has_group = false;
+    ofpbuf_use_stub(&ctx.action_set,
+                    ctx.action_set_stub, sizeof ctx.action_set_stub);
+
+    if (xin->recirc) {
+        const struct recirc_id_node *recirc = xin->recirc;
+
+        if (xin->ofpacts_len > 0 || ctx.rule) {
+            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+
+            VLOG_WARN_RL(&rl, "Recirculation conflict (%s)!",
+                         xin->ofpacts_len > 0
+                         ? "actions"
+                         : "rule");
+            return;
+        }
+
+        /* Restore action set, if any. */
+        if (recirc->action_set_len) {
+            const struct ofpact *a;
+
+            ofpbuf_put(&ctx.action_set, recirc->ofpacts,
+                       recirc->action_set_len);
+
+            OFPACT_FOR_EACH(a, recirc->ofpacts, recirc->action_set_len) {
+                if (a->type == OFPACT_GROUP) {
+                    ctx.action_set_has_group = true;
+                    break;
+                }
+            }
+        }
+        /* Restore recirculation actions.  These include unroll actions to
+         * restore the pipeline metadata. */
+        if (recirc->ofpacts_len - recirc->action_set_len) {
+            xin->ofpacts_len = recirc->ofpacts_len - recirc->action_set_len;
+            xin->ofpacts = recirc->ofpacts +
+                recirc->action_set_len / sizeof *recirc->ofpacts;
+        }
+    }
 
     if (!xin->ofpacts && !ctx.rule) {
         rule = rule_dpif_lookup(ctx.xbridge->ofproto, flow, wc,
@@ -4508,10 +4803,6 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
 
     ofpbuf_use_stub(&ctx.stack, ctx.init_stack, sizeof ctx.init_stack);
 
-    ctx.action_set_has_group = false;
-    ofpbuf_use_stub(&ctx.action_set,
-                    ctx.action_set_stub, sizeof ctx.action_set_stub);
-
     if (mbridge_has_mirrors(ctx.xbridge->mbridge)) {
         /* Do this conditionally because the copy is expensive enough that it
          * shows up in profiles. */
@@ -4562,8 +4853,23 @@ xlate_actions(struct xlate_in *xin, struct xlate_out *xout)
                 /* Drop all actions added by do_xlate_actions() above. */
                 ctx.xout->odp_actions->size = sample_actions_len;
             } else if (ctx.action_set.size) {
-                /* Translate action set only if not dropping the packet. */
-                xlate_action_set(&ctx);
+                /* Translate action set only if not dropping the packet and
+                 * not recirculating. */
+                if (!exit_recirculates(&ctx)) {
+                    xlate_action_set(&ctx);
+                    /* If action set recirculates the action set needs to be
+                     * cleared, as the actions in the set have been sanitized
+                     * to a temporary action list that is being executed. (If
+                     * we keep the set, the actions would be executed twice. */
+                    if (exit_recirculates(&ctx)) {
+                        ofpbuf_pull(&ctx.action_set, ctx.recirc_action_offset);
+                        ctx.recirc_action_offset = 0;
+                    }
+                }
+                /* Check if need to recirculate. */
+                if (exit_recirculates(&ctx)) {
+                    compose_recirculate_action(&ctx);
+                }
             }
         }
 
diff --git a/ofproto/ofproto-dpif-xlate.h b/ofproto/ofproto-dpif-xlate.h
index a53fa8e..8d22b6c 100644
--- a/ofproto/ofproto-dpif-xlate.h
+++ b/ofproto/ofproto-dpif-xlate.h
@@ -21,6 +21,7 @@
 #include "odp-util.h"
 #include "ofpbuf.h"
 #include "ofproto-dpif-mirror.h"
+#include "ofproto-dpif-rid.h"
 #include "ofproto-dpif.h"
 #include "ofproto.h"
 #include "stp.h"
@@ -36,7 +37,7 @@ struct mac_learning;
 struct mcast_snooping;
 struct xlate_cache;
 
-struct xlate_recirc {
+struct xlate_bond_recirc {
     uint32_t recirc_id;  /* !0 Use recirculation instead of output. */
     uint8_t  hash_alg;   /* !0 Compute hash for recirc before. */
     uint32_t hash_basis;  /* Compute hash for recirc before. */
@@ -60,11 +61,72 @@ struct xlate_out {
     ofp_port_t nf_output_iface; /* Output interface index for NetFlow. */
     mirror_mask_t mirrors;      /* Bitmap of associated mirrors. */
 
+    /* Recirculation IDs on which references are held. */
+    unsigned n_recircs;
+    union {
+        uint32_t recirc[2];   /* When n_recircs == 1 or 2 */
+        uint32_t *recircs;    /* When 'n_recircs' > 2 */
+    };
+
     uint64_t odp_actions_stub[256 / 8];
     struct ofpbuf odp_actions_buf;
     struct ofpbuf *odp_actions;
 };
 
+/* Helpers to abstract the recirculation union away. */
+static inline void
+xlate_out_add_recirc(struct xlate_out *xout, uint32_t id)
+{
+    if (OVS_LIKELY(xout->n_recircs < ARRAY_SIZE(xout->recirc))) {
+        xout->recirc[xout->n_recircs++] = id;
+    } else {
+        if (xout->n_recircs == ARRAY_SIZE(xout->recirc)) {
+            uint32_t *recircs = xmalloc(sizeof xout->recirc + sizeof id);
+
+            memcpy(recircs, xout->recirc, sizeof xout->recirc);
+            xout->recircs = recircs;
+        } else {
+            xout->recircs = xrealloc(xout->recircs,
+                                     (xout->n_recircs + 1) * sizeof id);
+        }
+        xout->recircs[xout->n_recircs++] = id;
+    }
+}
+
+static inline const uint32_t *
+xlate_out_get_recircs(const struct xlate_out *xout)
+{
+    if (OVS_LIKELY(xout->n_recircs <= ARRAY_SIZE(xout->recirc))) {
+        return xout->recirc;
+    } else {
+        return xout->recircs;
+    }
+}
+
+static inline void
+xlate_out_take_recircs(struct xlate_out *xout)
+{
+    if (OVS_UNLIKELY(xout->n_recircs > ARRAY_SIZE(xout->recirc))) {
+        free(xout->recircs);
+    }
+    xout->n_recircs = 0;
+}
+
+static inline void
+xlate_out_free_recircs(struct xlate_out *xout)
+{
+    if (OVS_LIKELY(xout->n_recircs <= ARRAY_SIZE(xout->recirc))) {
+        for (int i = 0; i < xout->n_recircs; i++) {
+            recirc_free_id(xout->recirc[i]);
+        }
+    } else {
+        for (int i = 0; i < xout->n_recircs; i++) {
+            recirc_free_id(xout->recircs[i]);
+        }
+        free(xout->recircs);
+    }
+}
+
 struct xlate_in {
     struct ofproto_dpif *ofproto;
 
@@ -143,6 +205,10 @@ struct xlate_in {
      * odp_actions stored in xlate_out.  If NULL, the default buffer will be
      * used. */
     struct ofpbuf *odp_actions;
+
+    /* The recirculation context related to this translation, as returned by
+     * xlate_lookup. */
+    const struct recirc_id_node *recirc;
 };
 
 void xlate_ofproto_set(struct ofproto_dpif *, const char *name, struct dpif *,
@@ -181,12 +247,13 @@ struct ofproto_dpif * xlate_lookup_ofproto(const struct dpif_backer *,
 int xlate_lookup(const struct dpif_backer *, const struct flow *,
                  struct ofproto_dpif **, struct dpif_ipfix **,
                  struct dpif_sflow **, struct netflow **,
-                 ofp_port_t *ofp_in_port);
+                 ofp_port_t *ofp_in_port, const struct recirc_id_node **);
 
 void xlate_actions(struct xlate_in *, struct xlate_out *);
 void xlate_in_init(struct xlate_in *, struct ofproto_dpif *,
                    const struct flow *, ofp_port_t in_port, struct rule_dpif *,
-                   uint16_t tcp_flags, const struct dp_packet *packet);
+                   uint16_t tcp_flags, const struct dp_packet *packet,
+                   const struct recirc_id_node *);
 void xlate_out_uninit(struct xlate_out *);
 void xlate_actions_for_side_effects(struct xlate_in *);
 void xlate_out_copy(struct xlate_out *dst, const struct xlate_out *src);
diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index 38ad6e2..4f178dd 100644
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -255,13 +255,6 @@ COVERAGE_DEFINE(rev_flow_table);
 COVERAGE_DEFINE(rev_mac_learning);
 COVERAGE_DEFINE(rev_mcast_snooping);
 
-/* Stores mapping between 'recirc_id' and 'ofproto-dpif'. */
-struct dpif_backer_recirc_node {
-    struct cmap_node cmap_node;
-    struct ofproto_dpif *ofproto;
-    uint32_t recirc_id;
-};
-
 /* All datapaths of a given type share a single dpif backer instance. */
 struct dpif_backer {
     char *type;
@@ -279,9 +272,6 @@ struct dpif_backer {
     bool recv_set_enable; /* Enables or disables receiving packets. */
 
     /* Recirculation. */
-    struct recirc_id_pool *rid_pool;       /* Recirculation ID pool. */
-    struct cmap recirc_map;         /* Map of 'recirc_id's to 'ofproto's. */
-    struct ovs_mutex recirc_mutex;  /* Protects 'recirc_map'. */
     bool enable_recirc;   /* True if the datapath supports recirculation */
 
     /* True if the datapath supports unique flow identifiers */
@@ -395,8 +385,6 @@ ofproto_dpif_get_enable_ufid(struct dpif_backer *backer)
     return backer->enable_ufid;
 }
 
-static struct ofport_dpif *get_ofp_port(const struct ofproto_dpif *ofproto,
-                                        ofp_port_t ofp_port);
 static void ofproto_trace(struct ofproto_dpif *, struct flow *,
                           const struct dp_packet *packet,
                           const struct ofpact[], size_t ofpacts_len,
@@ -857,27 +845,6 @@ dealloc(struct ofproto *ofproto_)
     free(ofproto);
 }
 
-/* Called when 'ofproto' is destructed.  Checks for and clears any
- * recirc_id leak. */
-static void
-dpif_backer_recirc_clear_ofproto(struct dpif_backer *backer,
-                                 struct ofproto_dpif *ofproto)
-{
-    struct dpif_backer_recirc_node *node;
-
-    ovs_mutex_lock(&backer->recirc_mutex);
-    CMAP_FOR_EACH (node, cmap_node, &backer->recirc_map) {
-        if (node->ofproto == ofproto) {
-            VLOG_ERR("recirc_id %"PRIu32", not freed when ofproto (%s) "
-                     "is destructed", node->recirc_id, ofproto->up.name);
-            cmap_remove(&backer->recirc_map, &node->cmap_node,
-                        node->recirc_id);
-            ovsrcu_postpone(free, node);
-        }
-    }
-    ovs_mutex_unlock(&backer->recirc_mutex);
-}
-
 static void
 close_dpif_backer(struct dpif_backer *backer)
 {
@@ -893,9 +860,6 @@ close_dpif_backer(struct dpif_backer *backer)
     ovs_rwlock_destroy(&backer->odp_to_ofport_lock);
     hmap_destroy(&backer->odp_to_ofport_map);
     shash_find_and_delete(&all_dpif_backers, backer->type);
-    recirc_id_pool_destroy(backer->rid_pool);
-    cmap_destroy(&backer->recirc_map);
-    ovs_mutex_destroy(&backer->recirc_mutex);
     free(backer->type);
     free(backer->dp_version_string);
     dpif_close(backer->dpif);
@@ -929,6 +893,8 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
     const char *name;
     int error;
 
+    recirc_init();
+
     backer = shash_find_data(&all_dpif_backers, type);
     if (backer) {
         backer->refcount++;
@@ -1010,9 +976,6 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
     backer->max_mpls_depth = check_max_mpls_depth(backer);
     backer->masked_set_action = check_masked_set_action(backer);
     backer->enable_ufid = check_ufid(backer);
-    backer->rid_pool = recirc_id_pool_create();
-    ovs_mutex_init(&backer->recirc_mutex);
-    cmap_init(&backer->recirc_map);
 
     backer->enable_tnl_push_pop = dpif_supports_tnl_push_pop(backer->dpif);
     atomic_count_init(&backer->tnl_count, 0);
@@ -1461,7 +1424,7 @@ destruct(struct ofproto *ofproto_)
     }
     guarded_list_destroy(&ofproto->pins);
 
-    dpif_backer_recirc_clear_ofproto(ofproto->backer, ofproto);
+    recirc_free_ofproto(ofproto, ofproto->up.name);
 
     mbridge_unref(ofproto->mbridge);
 
@@ -2743,7 +2706,7 @@ bundle_add_port(struct ofbundle *bundle, ofp_port_t ofp_port,
 {
     struct ofport_dpif *port;
 
-    port = get_ofp_port(bundle->ofproto, ofp_port);
+    port = ofp_port_to_ofport(bundle->ofproto, ofp_port);
     if (!port) {
         return false;
     }
@@ -3249,8 +3212,8 @@ set_mcast_snooping_port(struct ofproto *ofproto_, void *aux,
 
 /* Ports. */
 
-static struct ofport_dpif *
-get_ofp_port(const struct ofproto_dpif *ofproto, ofp_port_t ofp_port)
+struct ofport_dpif *
+ofp_port_to_ofport(const struct ofproto_dpif *ofproto, ofp_port_t ofp_port)
 {
     struct ofport *ofport = ofproto_get_port(&ofproto->up, ofp_port);
     return ofport ? ofport_dpif_cast(ofport) : NULL;
@@ -3444,7 +3407,7 @@ static int
 port_del(struct ofproto *ofproto_, ofp_port_t ofp_port)
 {
     struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
-    struct ofport_dpif *ofport = get_ofp_port(ofproto, ofp_port);
+    struct ofport_dpif *ofport = ofp_port_to_ofport(ofproto, ofp_port);
     int error = 0;
 
     if (!ofport) {
@@ -3690,7 +3653,7 @@ ofproto_dpif_execute_actions(struct ofproto_dpif *ofproto,
     }
 
     xlate_in_init(&xin, ofproto, flow, flow->in_port.ofp_port, rule,
-                  stats.tcp_flags, packet);
+                  stats.tcp_flags, packet, NULL);
     xin.ofpacts = ofpacts;
     xin.ofpacts_len = ofpacts_len;
     xin.resubmit_stats = &stats;
@@ -3756,21 +3719,13 @@ static void
 rule_dpif_set_recirc_id(struct rule_dpif *rule, uint32_t id)
     OVS_REQUIRES(rule->up.mutex)
 {
-    ovs_assert(!rule->recirc_id);
-    rule->recirc_id = id;
-}
-
-/* Returns 'rule''s recirculation id. */
-uint32_t
-rule_dpif_get_recirc_id(struct rule_dpif *rule)
-    OVS_REQUIRES(rule->up.mutex)
-{
-    if (!rule->recirc_id) {
-        struct ofproto_dpif *ofproto = ofproto_dpif_cast(rule->up.ofproto);
-
-        rule_dpif_set_recirc_id(rule, ofproto_dpif_alloc_recirc_id(ofproto));
+    ovs_assert(!rule->recirc_id || rule->recirc_id == id);
+    if (rule->recirc_id == id) {
+        /* Release the new reference to the same id. */
+        recirc_free_id(id);
+    } else {
+        rule->recirc_id = id;
     }
-    return rule->recirc_id;
 }
 
 /* Sets 'rule''s recirculation id. */
@@ -3784,15 +3739,13 @@ rule_set_recirc_id(struct rule *rule_, uint32_t id)
     ovs_mutex_unlock(&rule->up.mutex);
 }
 
-/* Lookup 'flow' in table 0 of 'ofproto''s classifier.
- * If 'wc' is non-null, sets the fields that were relevant as part of
- * the lookup. Returns the table id where a match or miss occurred via
- * 'table_id'.  This will be zero unless there was a miss and
- * OFPTC11_TABLE_MISS_CONTINUE is in effect for the sequence of tables
- * where misses occur, or TBL_INTERNAL if the rule has a non-zero
- * recirculation ID, and a match was found in the internal table, or if
- * there was no match and one of the special rules (drop_frags_rule,
- * miss_rule, or no_packet_in_rule) was returned.
+/* Lookup 'flow' in table 0 of 'ofproto''s classifier.  If 'wc' is non-null,
+ * sets the fields that were relevant as part of the lookup. Returns the table
+ * id where a match or miss occurred via 'table_id'.  This will be zero unless
+ * there was a miss and OFPTC11_TABLE_MISS_CONTINUE is in effect for the
+ * sequence of tables where misses occur, or TBL_INTERNAL if there was no match
+ * and one of the special rules (drop_frags_rule, miss_rule, or
+ * no_packet_in_rule) was returned.
  *
  * The return value is the found rule, which is valid at least until the next
  * RCU quiescent period.  If the rule needs to stay around longer,
@@ -3803,7 +3756,7 @@ rule_dpif_lookup(struct ofproto_dpif *ofproto, struct flow *flow,
                  struct flow_wildcards *wc, bool take_ref,
                  const struct dpif_flow_stats *stats, uint8_t *table_id)
 {
-    *table_id = rule_dpif_lookup_get_init_table_id(flow);
+    *table_id = 0;
 
     return rule_dpif_lookup_from_table(ofproto, flow, wc, take_ref, stats,
                                        table_id, flow->in_port.ofp_port, true,
@@ -3943,7 +3896,7 @@ rule_dpif_lookup_from_table(struct ofproto_dpif *ofproto, struct flow *flow,
             || miss_config == OFPUTIL_TABLE_MISS_CONTROLLER) {
             struct ofport_dpif *port;
 
-            port = get_ofp_port(ofproto, old_in_port);
+            port = ofp_port_to_ofport(ofproto, old_in_port);
             if (!port) {
                 VLOG_WARN_RL(&rl, "packet-in on unknown OpenFlow port %"PRIu16,
                              old_in_port);
@@ -4034,9 +3987,7 @@ rule_destruct(struct rule *rule_)
 
     ovs_mutex_destroy(&rule->stats_mutex);
     if (rule->recirc_id) {
-        struct ofproto_dpif *ofproto = ofproto_dpif_cast(rule->up.ofproto);
-
-        ofproto_dpif_free_recirc_id(ofproto, rule->recirc_id);
+        recirc_free_id(rule->recirc_id);
     }
 }
 
@@ -4901,6 +4852,34 @@ ofproto_trace(struct ofproto_dpif *ofproto, struct flow *flow,
               struct ds *ds)
 {
     struct trace_ctx trace;
+    const struct recirc_id_node *recirc;
+    ofp_port_t in_port;
+
+    if (flow->recirc_id) {
+        const struct ofpact_unroll_metadata *unroll;
+
+        recirc = recirc_id_node_find(flow->recirc_id);
+
+        if (!recirc) {
+            ds_put_format(ds,
+                          "Recirculation context not found for ID %"PRIu32"\n",
+                          flow->recirc_id);
+            return;
+        }
+        unroll = recirc_id_node_get_unroll_metadata(recirc);
+
+        if (ofproto != unroll->ofproto) {
+            /* 'unroll->ofproto' might be stale, do not use it. */
+            ds_put_format(ds, "Recirculation bridge for recirc ID %"PRIx32
+                          " does not match %s\n",
+                          flow->recirc_id, ofproto->up.name);
+            return;
+        }
+        in_port = unroll->in_port;
+    } else {
+        recirc = NULL;
+        in_port = flow->in_port.ofp_port;
+    }
 
     ds_put_format(ds, "Bridge: %s\n", ofproto->up.name);
     ds_put_cstr(ds, "Flow: ");
@@ -4912,8 +4891,8 @@ ofproto_trace(struct ofproto_dpif *ofproto, struct flow *flow,
     trace.result = ds;
     trace.key = flow; /* Original flow key, used for megaflow. */
     trace.flow = *flow; /* May be modified by actions. */
-    xlate_in_init(&trace.xin, ofproto, flow, flow->in_port.ofp_port, NULL,
-                  ntohs(flow->tcp_flags), packet);
+    xlate_in_init(&trace.xin, ofproto, flow, in_port, NULL,
+                  ntohs(flow->tcp_flags), packet, recirc);
     trace.xin.ofpacts = ofpacts;
     trace.xin.ofpacts_len = ofpacts_len;
     trace.xin.resubmit_hook = trace_resubmit;
@@ -5458,7 +5437,7 @@ vsp_add(struct ofport_dpif *port, ofp_port_t realdev_ofp_port, int vid)
 static odp_port_t
 ofp_port_to_odp_port(const struct ofproto_dpif *ofproto, ofp_port_t ofp_port)
 {
-    const struct ofport_dpif *ofport = get_ofp_port(ofproto, ofp_port);
+    const struct ofport_dpif *ofport = ofp_port_to_ofport(ofproto, ofp_port);
     return ofport ? ofport->odp_port : ODPP_NONE;
 }
 
@@ -5493,61 +5472,6 @@ odp_port_to_ofp_port(const struct ofproto_dpif *ofproto, odp_port_t odp_port)
     }
 }
 
-struct ofproto_dpif *
-ofproto_dpif_recirc_get_ofproto(const struct dpif_backer *backer,
-                                uint32_t recirc_id)
-{
-    struct dpif_backer_recirc_node *node;
-
-    node = CONTAINER_OF(cmap_find(&backer->recirc_map, recirc_id),
-                        struct dpif_backer_recirc_node, cmap_node);
-
-    return node ? node->ofproto : NULL;
-}
-
-uint32_t
-ofproto_dpif_alloc_recirc_id(struct ofproto_dpif *ofproto)
-{
-    struct dpif_backer *backer = ofproto->backer;
-    uint32_t recirc_id = recirc_id_alloc(backer->rid_pool);
-
-    if (recirc_id) {
-        struct dpif_backer_recirc_node *node = xmalloc(sizeof *node);
-
-        node->recirc_id = recirc_id;
-        node->ofproto = ofproto;
-
-        ovs_mutex_lock(&backer->recirc_mutex);
-        cmap_insert(&backer->recirc_map, &node->cmap_node, node->recirc_id);
-        ovs_mutex_unlock(&backer->recirc_mutex);
-    }
-
-    return recirc_id;
-}
-
-void
-ofproto_dpif_free_recirc_id(struct ofproto_dpif *ofproto, uint32_t recirc_id)
-{
-    struct dpif_backer *backer = ofproto->backer;
-    struct dpif_backer_recirc_node *node;
-
-    node = CONTAINER_OF(cmap_find(&backer->recirc_map, recirc_id),
-                        struct dpif_backer_recirc_node, cmap_node);
-    if (node) {
-        ovs_mutex_lock(&backer->recirc_mutex);
-        cmap_remove(&backer->recirc_map, &node->cmap_node, node->recirc_id);
-        ovs_mutex_unlock(&backer->recirc_mutex);
-        recirc_id_free(backer->rid_pool, node->recirc_id);
-
-        /* 'recirc_id' should never be freed by non-owning 'ofproto'. */
-        ovs_assert(node->ofproto == ofproto);
-
-        /* RCU postpone the free, since other threads may be referring
-         * to 'node' at same time. */
-        ovsrcu_postpone(free, node);
-    }
-}
-
 int
 ofproto_dpif_add_internal_flow(struct ofproto_dpif *ofproto,
                                const struct match *match, int priority,
diff --git a/ofproto/ofproto-dpif.h b/ofproto/ofproto-dpif.h
index e2359cd..14d2c3b 100644
--- a/ofproto/ofproto-dpif.h
+++ b/ofproto/ofproto-dpif.h
@@ -92,14 +92,6 @@ struct rule_dpif *rule_dpif_lookup_from_table(struct ofproto_dpif *,
                                               bool may_packet_in,
                                               bool honor_table_miss);
 
-/* If 'recirc_id' is set, starts looking up from internal table for
- * post recirculation flows or packets.  Otherwise, starts from table 0. */
-static inline uint8_t
-rule_dpif_lookup_get_init_table_id(const struct flow *flow)
-{
-    return flow->recirc_id ? TBL_INTERNAL : 0;
-}
-
 static inline void rule_dpif_ref(struct rule_dpif *);
 static inline void rule_dpif_unref(struct rule_dpif *);
 
@@ -115,7 +107,6 @@ uint8_t rule_dpif_get_table(const struct rule_dpif *);
 bool table_is_internal(uint8_t table_id);
 
 const struct rule_actions *rule_dpif_get_actions(const struct rule_dpif *);
-uint32_t rule_dpif_get_recirc_id(struct rule_dpif *);
 void rule_set_recirc_id(struct rule *, uint32_t id);
 
 ovs_be64 rule_dpif_get_flow_cookie(const struct rule_dpif *rule);
@@ -156,83 +147,9 @@ void ofproto_dpif_flow_mod(struct ofproto_dpif *, struct ofputil_flow_mod *);
 struct rule_dpif *ofproto_dpif_refresh_rule(struct rule_dpif *);
 
 struct ofport_dpif *odp_port_to_ofport(const struct dpif_backer *, odp_port_t);
+struct ofport_dpif *ofp_port_to_ofport(const struct ofproto_dpif *,
+                                       ofp_port_t);
 
-/*
- * Recirculation
- * =============
- *
- * Recirculation is a technique to allow a frame to re-enter the packet
- * processing path for one or multiple times to achieve more flexible packet
- * processing in the data path. MPLS handling and selecting bond slave port
- * of a bond ports.
- *
- * Data path and user space interface
- * -----------------------------------
- *
- * Two new fields, recirc_id and dp_hash, are added to the current flow data
- * structure. They are both of type uint32_t. In addition, a new action,
- * RECIRC, are added.
- *
- * The value recirc_id is used to distinguish a packet from multiple
- * iterations of recirculation. A packet initially received is considered of
- * having recirc_id of 0. Recirc_id is managed by the user space, opaque to
- * the data path.
- *
- * On the other hand, dp_hash can only be computed by the data path, opaque to
- * the user space. In fact, user space may not able to recompute the hash
- * value. The dp_hash value should be wildcarded when for a newly received
- * packet. RECIRC action specifies whether the hash is computed. If computed,
- * how many fields to be included in the hash computation. The computed hash
- * value is stored into the dp_hash field prior to recirculation.
- *
- * The RECIRC action computes and set the dp_hash field, set the recirc_id
- * field and then reprocess the packet as if it was received on the same input
- * port. RECIRC action works like a function call; actions listed behind the
- * RECIRC action will be executed after its execution.  RECIRC action can be
- * nested, data path implementation limits the number of recirculation executed
- * to prevent unreasonable nesting depth or infinite loop.
- *
- * Both flow fields and the RECIRC action are exposed as OpenFlow fields via
- * Nicira extensions.
- *
- * Post recirculation flow
- * ------------------------
- *
- * At the OpenFlow level, post recirculation rules are always hidden from the
- * controller.  They are installed in table 254 which is set up as a hidden
- * table during boot time. Those rules are managed by the local user space
- * program only.
- *
- * To speed up the classifier look up process, recirc_id is always reflected
- * into the metadata field, since recirc_id is required to be exactly matched.
- *
- * Classifier look up always starts with table 254. A post recirculation flow
- * lookup should find its hidden rule within this table. On the other hand, A
- * newly received packet should miss all post recirculation rules because its
- * recirc_id is zero, then hit a pre-installed lower priority rule to redirect
- * classifier to look up starting from table 0:
- *
- *       * , actions=resubmit(,0)
- *
- * Post recirculation data path flows are managed like other data path flows.
- * They are created on demand. Miss handling, stats collection and revalidation
- * work the same way as regular flows.
- *
- * If the bridge which originates the recirculation is different from the bridge
- * that receives the post recirculation packet (e.g. when patch port is used),
- * the packet will be processed directly by the recirculation bridge with
- * in_port set to OFPP_NONE.  Admittedly, doing this limits the recirculation
- * bridge from matching on in_port of post recirculation packets, and will be
- * fixed in the near future.
- *
- * TODO: Always restore the correct in_port.
- *
- */
-
-struct ofproto_dpif *ofproto_dpif_recirc_get_ofproto(const struct dpif_backer *ofproto,
-                                                     uint32_t recirc_id);
-uint32_t ofproto_dpif_alloc_recirc_id(struct ofproto_dpif *ofproto);
-void ofproto_dpif_free_recirc_id(struct ofproto_dpif *ofproto, uint32_t recirc_id);
 int ofproto_dpif_add_internal_flow(struct ofproto_dpif *,
                                    const struct match *, int priority,
                                    uint16_t idle_timeout,
diff --git a/tests/mpls-xlate.at b/tests/mpls-xlate.at
index e2ef2e7..c293db7 100644
--- a/tests/mpls-xlate.at
+++ b/tests/mpls-xlate.at
@@ -31,8 +31,12 @@ dnl Setup multiple MPLS tags.
 AT_CHECK([ovs-ofctl del-flows br0])
 
 AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 in_port=local,dl_type=0x0800,action=push_mpls:0x8847,set_field:10-\>mpls_label,push_mpls:0x8847,set_field:20-\>mpls_label,output:1])
-AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 table=0,dl_type=0x8847,in_port=1,mpls_label=60,action=pop_mpls:0x8847,goto_table:1])
-AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 table=1,dl_type=0x8847,in_port=1,mpls_label=50,action=pop_mpls:0x0800,output:LOCAL])
+# The resubmits will be executed after recirculation, which preserves the
+# register values.
+AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 cookie=0xa,table=0,dl_type=0x8847,in_port=1,mpls_label=60,action=set_field:10-\>reg0,pop_mpls:0x8847,goto_table:1])
+# The pop_mpls below recirculates from within a resubmit
+# After recirculation the (restored) register value is moved to IP ttl.
+AT_CHECK([ovs-ofctl -O OpenFlow13 add-flow br0 cookie=0xb,table=1,dl_type=0x8847,in_port=1,mpls_label=50,action=pop_mpls:0x0800,move:NXM_NX_REG0[[0..7]]-\>NXM_NX_IP_TTL[[]],output:LOCAL])
 
 dnl Double MPLS push
 AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(100),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x0800),ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)'], [0], [stdout])
@@ -41,9 +45,19 @@ AT_CHECK([tail -1 stdout], [0],
 ])
 
 dnl Double MPLS pop
-AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x8847),mpls(label=60,tc=0/0,ttl=64,bos=0)'], [0], [stdout])
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x8847),mpls(label=60,tc=0/0,ttl=64,bos=0)' -generate], [0], [stdout])
 AT_CHECK([tail -1 stdout], [0],
-  [Datapath actions: pop_mpls(eth_type=0x8847),recirc(300)
+  [Datapath actions: pop_mpls(eth_type=0x8847),recirc(1)
+])
+
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(1),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x8847),mpls(label=50,tc=0/0,ttl=64,bos=0)' -generate], [0], [stdout])
+AT_CHECK([tail -1 stdout], [0],
+  [Datapath actions: pop_mpls(eth_type=0x800),recirc(2)
+])
+
+AT_CHECK([ovs-appctl ofproto/trace ovs-dummy 'recirc_id(2),in_port(1),eth(src=f8:bc:12:44:34:b6,dst=f8:bc:12:46:58:e0),eth_type(0x0800),ipv4(src=1.1.2.92,dst=1.1.2.88,proto=47,tos=0,ttl=64,frag=no)' -generate], [0], [stdout])
+AT_CHECK([tail -1 stdout], [0],
+  [Datapath actions: set(ipv4(ttl=10)),100
 ])
 
 OVS_VSWITCHD_STOP
diff --git a/tests/ofproto-dpif.at b/tests/ofproto-dpif.at
index 4932fcc..9b454c9 100644
--- a/tests/ofproto-dpif.at
+++ b/tests/ofproto-dpif.at
@@ -195,7 +195,7 @@ AT_CHECK([ovs-appctl revalidator/purge], [0])
 # Checks the flow stats in br1, should only be one flow with non-zero
 # 'n_packets' from internal table.
 AT_CHECK([ovs-appctl bridge/dump-flows br1 | ofctl_strip | grep -- "n_packets" | grep -- "table_id" | sed -e 's/dp_hash=0x[[0-9a-f]][[0-9a-f]]*/dp_hash=0x0/' -e 's/output:[[0-9]][[0-9]]*/output/'], [0], [dnl
-table_id=254, n_packets=1, n_bytes=64, priority=20,recirc_id=0x12d,dp_hash=0x0/0xff,actions=output
+table_id=254, n_packets=1, n_bytes=64, priority=20,recirc_id=0x2,dp_hash=0x0/0xff,actions=output
 ])
 
 # Checks the flow stats in br-int, should be only one match.
@@ -1526,7 +1526,8 @@ cookie=0xd dl_src=60:66:66:66:00:07 actions=pop_mpls:0x0800,learn(table=1,hard_t
 cookie=0xd dl_src=60:66:66:66:00:08 actions=pop_mpls:0x0806,resubmit(1,1)
 cookie=0xd table=1 arp actions=controller
 
-cookie=0xd dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48,controller
+cookie=0xdeadbeef table=2 dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48
+cookie=0xd dl_src=60:66:66:66:00:09 actions=resubmit(,2),controller
 cookie=0xd dl_src=60:66:66:66:00:0a actions=pop_mpls:0x0800,mod_nw_dst:10.0.0.1,controller
 cookie=0xd dl_src=60:66:66:66:00:0b actions=pop_mpls:0x0800,mod_nw_src:10.0.0.1,controller
 
@@ -1535,20 +1536,20 @@ cookie=0xd dl_src=60:66:66:66:01:01 actions=pop_mpls:0x8847,dec_mpls_ttl,control
 cookie=0xd dl_src=60:66:66:66:01:02 actions=pop_mpls:0x8848,load:3->OXM_OF_MPLS_TC[[]],controller
 
 cookie=0xd dl_src=60:66:66:66:02:00 actions=pop_mpls:0x8847,pop_mpls:0x0800,controller
-cookie=0xd dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,controller
-cookie=0xd dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,controller
+cookie=0xe dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,controller
+cookie=0xe dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,controller
 
-cookie=0xd dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,controller
-cookie=0xd dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,controller
-cookie=0xd dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0xe dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,controller
+cookie=0xe dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,controller
+cookie=0xe dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,controller
 
-cookie=0xd dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,controller
-cookie=0xd dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,controller
-cookie=0xd dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0xf dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,controller
+cookie=0xf dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0xf dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,controller
 
-cookie=0xd dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,controller
-cookie=0xd dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,controller
-cookie=0xd dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,controller
+cookie=0x5 dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,controller
+cookie=0x5 dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,controller
+cookie=0x5 dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,controller
 ])
 AT_CHECK([ovs-ofctl add-flows br0 flows.txt])
 
@@ -1822,13 +1823,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -1848,13 +1849,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:02,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:02,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:02,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 ])
 
@@ -1874,13 +1875,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:03,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:03,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:03,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 ])
 
@@ -1900,13 +1901,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:04,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:04,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:04,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.2,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7743
 ])
 
@@ -1926,13 +1927,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.106,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:76db
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.106,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:76db
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:05,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.106,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:76db
 ])
 
@@ -1952,13 +1953,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:06,dl_dst=50:54:00:00:00:07,nw_src=192.168.255.255,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7745
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:06,dl_dst=50:54:00:00:00:07,nw_src=192.168.255.255,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7745
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:06,dl_dst=50:54:00:00:00:07,nw_src=192.168.255.255,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7745
 ])
 
@@ -1978,13 +1979,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:07,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:07,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:07,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2028,13 +2029,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:09,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=48,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:09,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=48,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:09,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=48,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2054,13 +2055,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0a,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0a,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0a,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=10.0.0.1,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2dee
 ])
 
@@ -2080,13 +2081,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0b,dl_dst=50:54:00:00:00:07,nw_src=10.0.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2ded
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0b,dl_dst=50:54:00:00:00:07,nw_src=10.0.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2ded
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:0b,dl_dst=50:54:00:00:00:07,nw_src=10.0.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:2ded
 ])
 
@@ -2134,13 +2135,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:01:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:01:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:01:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 ])
 
@@ -2188,13 +2189,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:00,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:00,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:00,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=255,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2211,17 +2212,18 @@ AT_CHECK([ovs-ofctl monitor br0 65534 -P nxm --detach --pidfile 2> ofctl_monitor
 for i in 1 2 3; do
     ovs-appctl netdev-dummy/receive p1 '50 54 00 00 00 07 60 66 66 66 02 01 88 48 00 01 40 20 00 01 41 1f 45 00 00 2c 00 00 00 00 ff 06 3b 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45'
 done
+
 OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:01,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2229,7 +2231,7 @@ AT_CHECK([ovs-appctl time/warp 5000], [0], [ignore])
 
 dnl Modified MPLS pop action.
 dnl The input is a frame with two MPLS label stack entries which tcpdump -vve shows as:
-dnl 60:66:66:66:02:10 > 50:54:00:00:02:10, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
+dnl 60:66:66:66:02:10 > 50:54:00:00:00:07, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
 dnl		(label 20, exp 0, [S], ttl 31)
 dnl		(tos 0x0, ttl 254, id 0, offset 0, flags [none], proto TCP (6), length 44)
 dnl        192.168.0.1.80 > 192.168.0.2.0: Flags [none], cksum 0x7744 (correct), seq 42:46, win 10000, length 4
@@ -2242,13 +2244,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:10,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:10,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:02:10,dl_dst=50:54:00:00:00:07,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=0,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
@@ -2256,7 +2258,7 @@ AT_CHECK([ovs-appctl time/warp 5000], [0], [ignore])
 
 dnl Modified MPLS pop action.
 dnl The input is a frame with three MPLS label stack entries which tcpdump -vve shows as:
-dnl 60:66:66:66:03:00 > 50:54:00:00:00:00, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
+dnl 60:66:66:66:03:00 > 50:54:00:00:00:07, ethertype MPLS unicast (0x8847), length 66: MPLS (label 20, exp 0, ttl 32)
 dnl		(label 20, exp 0, ttl 31)
 dnl		(label 20, exp 0, [S], ttl 30)
 dnl		(tos 0x0, ttl 254, id 0, offset 0, flags [none], proto TCP (6), length 44)
@@ -2270,13 +2272,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=30,mpls_bos=1
 ])
 
@@ -2298,13 +2300,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 ])
 
@@ -2326,13 +2328,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xe total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:03:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=29,mpls_bos=1
 ])
 
@@ -2352,13 +2354,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:00,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=255,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:00,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=255,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:00,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=255,mpls_bos=1
 ])
 
@@ -2378,13 +2380,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:01,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=254,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:01,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=254,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:01,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=254,mpls_bos=1
 ])
 
@@ -2404,13 +2406,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA  ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:10,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=253,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:10,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=253,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0xf total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:04:10,dl_dst=50:54:00:00:00:07,mpls_label=0,mpls_tc=0,mpls_ttl=253,mpls_bos=1
 ])
 
@@ -2426,17 +2428,18 @@ AT_CHECK([ovs-ofctl monitor br0 65534 -P nxm --detach --pidfile 2> ofctl_monitor
 for i in 1 2 3; do
     ovs-appctl netdev-dummy/receive p1 '50 54 00 00 00 07 60 66 66 66 05 00 88 47 00 01 41 20 45 00 00 2c 00 00 00 00 ff 06 3a 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45'
 done
+
 OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=32,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=32,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): cookie=0xd total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:00,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=32,mpls_bos=1
 ])
 
@@ -2456,13 +2459,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mplsm,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:01,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 ])
 
@@ -2478,22 +2481,26 @@ AT_CHECK([ovs-ofctl monitor br0 65534 -P nxm --detach --pidfile 2> ofctl_monitor
 for i in 1 2 3; do
     ovs-appctl netdev-dummy/receive p1 '50 54 00 00 00 07 60 66 66 66 05 10 88 47 00 01 41 20 45 00 00 2c 00 00 00 00 ff 06 3a 78 c0 a8 00 01 c0 a8 00 02 00 50 00 00 00 00 00 2a 00 00 00 2a 50 00 27 10 77 44 00 00 48 4f 47 45'
 done
+
 OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([cat ofctl_monitor.log], [0], [dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 dnl
-NXT_PACKET_IN (xid=0x0): table_id=254 cookie=0x0 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
+NXT_PACKET_IN (xid=0x0): cookie=0x5 total_len=62 in_port=1 (via action) data_len=62 (unbuffered)
 mpls,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:05:10,dl_dst=50:54:00:00:00:07,mpls_label=20,mpls_tc=0,mpls_ttl=31,mpls_bos=1
 ])
 
 AT_CHECK([ovs-appctl revalidator/purge], [0])
 AT_CHECK([ovs-ofctl dump-flows br0 | ofctl_strip | sort], [0], [dnl
+ cookie=0x5, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,CONTROLLER:65535
+ cookie=0x5, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0x5, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xa, n_packets=3, n_bytes=180, dl_src=40:44:44:44:44:42 actions=push_mpls:0x8847,load:0xa->OXM_OF_MPLS_LABEL[[]],load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
  cookie=0xa, n_packets=3, n_bytes=180, dl_src=40:44:44:44:44:43 actions=push_mpls:0x8847,load:0xa->OXM_OF_MPLS_LABEL[[]],load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
  cookie=0xa, n_packets=3, n_bytes=180, dl_src=40:44:44:44:44:44 actions=push_mpls:0x8847,load:0xa->OXM_OF_MPLS_LABEL[[]],load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
@@ -2513,26 +2520,24 @@ AT_CHECK([ovs-ofctl dump-flows br0 | ofctl_strip | sort], [0], [dnl
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:05 actions=pop_mpls:0x0800,multipath(eth_src,50,modulo_n,256,0,NXM_OF_IP_SRC[[0..7]]),CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:06 actions=pop_mpls:0x0800,bundle_load(eth_src,50,hrw,ofport,NXM_OF_IP_SRC[[0..15]],slaves:1,2),CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:07 actions=pop_mpls:0x0800,learn(table=1,hard_timeout=60,eth_type=0x800,nw_proto=6,NXM_OF_IP_SRC[[]]=NXM_OF_IP_DST[[]]),CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48,CONTROLLER:65535
+ cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:09 actions=resubmit(,2),CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:0a actions=pop_mpls:0x0800,mod_nw_dst:10.0.0.1,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:0b actions=pop_mpls:0x0800,mod_nw_src:10.0.0.1,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:00 actions=push_mpls:0x8848,pop_mpls:0x8847,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:01 actions=push_mpls:0x8847,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:05:10 actions=push_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=186, dl_src=60:66:66:66:66:66 actions=pop_mpls:0x0800,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:01:00 actions=pop_mpls:0x8848,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:01:01 actions=pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:01:02 actions=pop_mpls:0x8848,load:0x3->OXM_OF_MPLS_TC[[]],CONTROLLER:65535
  cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:00 actions=pop_mpls:0x8847,pop_mpls:0x0800,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
- cookie=0xd, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
  cookie=0xd, table=1, n_packets=3, n_bytes=168, arp actions=CONTROLLER:65535
+ cookie=0xdeadbeef, table=2, n_packets=3, n_bytes=186, dl_src=60:66:66:66:00:09 actions=pop_mpls:0x0800,mod_nw_tos:48
+ cookie=0xe, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:01 actions=pop_mpls:0x8848,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=198, dl_src=60:66:66:66:02:10 actions=pop_mpls:0x8847,dec_mpls_ttl,pop_mpls:0x0800,dec_ttl,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:00 actions=pop_mpls:0x8848,pop_mpls:0x8848,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:01 actions=pop_mpls:0x8847,pop_mpls:0x8847,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0xe, n_packets=3, n_bytes=210, dl_src=60:66:66:66:03:10 actions=pop_mpls:0x8848,dec_mpls_ttl,pop_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0xf, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:00 actions=pop_mpls:0x0800,push_mpls:0x8847,CONTROLLER:65535
+ cookie=0xf, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:01 actions=pop_mpls:0x0800,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
+ cookie=0xf, n_packets=3, n_bytes=186, dl_src=60:66:66:66:04:10 actions=pop_mpls:0x0800,dec_ttl,push_mpls:0x8848,dec_mpls_ttl,CONTROLLER:65535
 NXST_FLOW reply:
 ])
 
@@ -2616,13 +2621,13 @@ OVS_WAIT_UNTIL([test `wc -l < ofctl_monitor.log` -ge 6])
 OVS_APP_EXIT_AND_WAIT(ovs-ofctl)
 
 AT_CHECK([STRIP_METADATA ofctl_monitor.log], [0], [dnl
-OFPT_PACKET_IN (OF1.2) (xid=0x0): table_id=254 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+OFPT_PACKET_IN (OF1.2) (xid=0x0): total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:08,dl_dst=50:54:00:00:00:01,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=32,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-OFPT_PACKET_IN (OF1.2) (xid=0x0): table_id=254 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+OFPT_PACKET_IN (OF1.2) (xid=0x0): total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:08,dl_dst=50:54:00:00:00:01,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=32,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 dnl
-OFPT_PACKET_IN (OF1.2) (xid=0x0): table_id=254 total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
+OFPT_PACKET_IN (OF1.2) (xid=0x0): total_len=58 in_port=1 (via action) data_len=58 (unbuffered)
 tcp,in_port=0,vlan_tci=0x0000,dl_src=60:66:66:66:00:08,dl_dst=50:54:00:00:00:01,nw_src=192.168.0.1,nw_dst=192.168.0.2,nw_tos=32,nw_ecn=0,nw_ttl=254,tp_src=80,tp_dst=0,tcp_flags=0 tcp_csum:7744
 ])
 
-- 
1.7.10.4




More information about the dev mailing list