[ovs-dev] [PATCH ovn v2 3/3] lflow: Consistent conjunction id generation.

Han Zhou hzhou at ovn.org
Wed Nov 3 17:10:36 UTC 2021


When a logical flow translation uses conjunction ids, it is better to
keep the conjuntion ids consistent between iterations, so that the
generated OVS flows don't change unnecessarily and avoid reinstalling
unchanged flows to OVS. The problem was partially solved when
lflow-cache was enabled but there was a bug for a corner case so the
lflow-cache based approach was reverted in a previous commit.

This patch takes an approach that generates conjunction ids based on the
logical flow's uuid, which ensures the same ids being used when
processing the same lflow in most cases. There is a very little chance
that different logial flows happen to have the same 32-bit prefix in
their uuids, but the probability is really low. For the very unlikely
situation, the conflicts are handled properly and correctness is
ensured.

A new module lflow-conj-ids is created to manage the id allocations and
maintaining the usage of each ids and their owner lflows.

Unlike the previously lflow-cache based approach, this approach works
equally well when lflow-cache is not in use.

Signed-off-by: Han Zhou <hzhou at ovn.org>
---
 controller/automake.mk        |   2 +
 controller/lflow-cache.c      |   6 +-
 controller/lflow-cache.h      |   9 +-
 controller/lflow-conj-ids.c   | 217 ++++++++++++++++++++++++++++++++++
 controller/lflow-conj-ids.h   |  53 +++++++++
 controller/lflow.c            | 140 +++++++++-------------
 controller/lflow.h            |   4 +-
 controller/ovn-controller.c   |  32 ++---
 controller/test-lflow-cache.c |  36 ++++--
 tests/ovn-lflow-cache.at      | 123 ++++++++++++-------
 tests/ovn.at                  |  49 +++++---
 11 files changed, 491 insertions(+), 180 deletions(-)
 create mode 100644 controller/lflow-conj-ids.c
 create mode 100644 controller/lflow-conj-ids.h

diff --git a/controller/automake.mk b/controller/automake.mk
index ad2d68af2..a6ced858a 100644
--- a/controller/automake.mk
+++ b/controller/automake.mk
@@ -18,6 +18,8 @@ controller_ovn_controller_SOURCES = \
 	controller/lflow.h \
 	controller/lflow-cache.c \
 	controller/lflow-cache.h \
+	controller/lflow-conj-ids.c \
+	controller/lflow-conj-ids.h \
 	controller/lport.c \
 	controller/lport.h \
 	controller/ofctrl.c \
diff --git a/controller/lflow-cache.c b/controller/lflow-cache.c
index 26228c960..6db85fc12 100644
--- a/controller/lflow-cache.c
+++ b/controller/lflow-cache.c
@@ -203,7 +203,7 @@ lflow_cache_get_stats(const struct lflow_cache *lc, struct ds *output)
 
 void
 lflow_cache_add_expr(struct lflow_cache *lc, const struct uuid *lflow_uuid,
-                     uint32_t conj_id_ofs, struct expr *expr, size_t expr_sz)
+                     struct expr *expr, size_t expr_sz)
 {
     struct lflow_cache_value *lcv =
         lflow_cache_add__(lc, lflow_uuid, LCACHE_T_EXPR, expr_sz);
@@ -213,12 +213,12 @@ lflow_cache_add_expr(struct lflow_cache *lc, const struct uuid *lflow_uuid,
         return;
     }
     COVERAGE_INC(lflow_cache_add_expr);
-    lcv->conj_id_ofs = conj_id_ofs;
     lcv->expr = expr;
 }
 
 void
 lflow_cache_add_matches(struct lflow_cache *lc, const struct uuid *lflow_uuid,
+                        uint32_t conj_id_ofs, uint32_t n_conjs,
                         struct hmap *matches, size_t matches_sz)
 {
     struct lflow_cache_value *lcv =
@@ -231,6 +231,8 @@ lflow_cache_add_matches(struct lflow_cache *lc, const struct uuid *lflow_uuid,
     }
     COVERAGE_INC(lflow_cache_add_matches);
     lcv->expr_matches = matches;
+    lcv->n_conjs = n_conjs;
+    lcv->conj_id_ofs = conj_id_ofs;
 }
 
 struct lflow_cache_value *
diff --git a/controller/lflow-cache.h b/controller/lflow-cache.h
index d42bdeaa3..faad32bb6 100644
--- a/controller/lflow-cache.h
+++ b/controller/lflow-cache.h
@@ -43,6 +43,11 @@ enum lflow_cache_type {
 
 struct lflow_cache_value {
     enum lflow_cache_type type;
+
+    /* n_conjs and conj_id_ofs are used only for LCACHE_T_MATCHES.
+     * They are saved together with the match, so that we can re-allocate the
+     * same ids when reusing the cached match for a given lflow. */
+    uint32_t n_conjs;
     uint32_t conj_id_ofs;
 
     union {
@@ -61,10 +66,10 @@ bool lflow_cache_is_enabled(const struct lflow_cache *);
 void lflow_cache_get_stats(const struct lflow_cache *, struct ds *output);
 
 void lflow_cache_add_expr(struct lflow_cache *, const struct uuid *lflow_uuid,
-                          uint32_t conj_id_ofs, struct expr *expr,
-                          size_t expr_sz);
+                          struct expr *expr, size_t expr_sz);
 void lflow_cache_add_matches(struct lflow_cache *,
                              const struct uuid *lflow_uuid,
+                             uint32_t conj_id_ofs, uint32_t n_conjs,
                              struct hmap *matches, size_t matches_sz);
 
 struct lflow_cache_value *lflow_cache_get(struct lflow_cache *,
diff --git a/controller/lflow-conj-ids.c b/controller/lflow-conj-ids.c
new file mode 100644
index 000000000..e8d129dd3
--- /dev/null
+++ b/controller/lflow-conj-ids.c
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <config.h>
+#include "coverage.h"
+#include "lflow-conj-ids.h"
+#include "util.h"
+
+COVERAGE_DEFINE(lflow_conj_conflict);
+
+/* Insert n_conjs conjuntion ids starting from start_conj_id into the conj_ids,
+ * assuming the ids are confirmed to be available. */
+static void
+lflow_conj_ids_insert_(struct conj_ids *conj_ids,
+                       const struct uuid *lflow_uuid,
+                       uint32_t start_conj_id, uint32_t n_conjs)
+{
+    ovs_assert(n_conjs);
+    uint32_t conj_id = start_conj_id;
+    for (uint32_t i = 0; i < n_conjs; i++) {
+        ovs_assert(conj_id);
+        struct conj_id_node *node = xzalloc(sizeof *node);
+        node->conj_id = conj_id;
+        hmap_insert(&conj_ids->conj_id_allocations, &node->hmap_node, conj_id);
+        conj_id++;
+    }
+
+    struct lflow_conj_node *lflow_conj = xzalloc(sizeof *lflow_conj);
+    lflow_conj->lflow_uuid = *lflow_uuid;
+    lflow_conj->start_conj_id = start_conj_id;
+    lflow_conj->n_conjs = n_conjs;
+    hmap_insert(&conj_ids->lflow_conj_ids, &lflow_conj->hmap_node,
+                uuid_hash(lflow_uuid));
+}
+
+/* Allocate n_conjs continuous conjuction ids from the conj_ids for the given
+ * lflow_uuid. (0 is never included in an allocated range)
+ *
+ * The first conjunction id is returned. If no conjunction ids available, or if
+ * the input is invalid (n_conjs == 0), then 0 is returned.
+ *
+ * The algorithm tries to allocate the parts[0] of the input uuid as the first
+ * conjunction id. If it is unavailable, or any of the subsequent n_conjs - 1
+ * ids are unavailable, iterate until the next available n_conjs ids are found.
+ * Given that n_conjs is very small (in most cases will be 1), the algorithm
+ * should be efficient enough and in most cases just return the lflow_uuid's
+ * part[0], which ensures conjunction ids are consistent for the same logical
+ * flow in most cases.
+ *
+ * The performance will degrade if most of the uint32_t are allocated because
+ * conflicts will happen a lot. In practice this is not expected to happen in
+ * reasonalbe scales. Otherwise, if the amount of logical flows is close to
+ * this (4G logical flows that need conjunction ids) there are other parts of
+ * the system expected to be suffered even before reaching to a scale much
+ * smaller than this. */
+uint32_t
+lflow_conj_ids_alloc(struct conj_ids *conj_ids, const struct uuid *lflow_uuid,
+                     uint32_t n_conjs)
+{
+    if (!n_conjs) {
+        return 0;
+    }
+    uint32_t start_conj_id = lflow_uuid->parts[0];
+    uint32_t initial_id = start_conj_id;
+    bool initial = true;
+    while (true) {
+        if (start_conj_id == 0) {
+            start_conj_id++;
+        }
+        bool available = true;
+        uint32_t conj_id = start_conj_id;
+        for (uint32_t i = 0; i < n_conjs; i++) {
+            if (conj_id == 0) {
+                /* Overflow. Consider the current range as unavailable because
+                 * we need a continuous range. Start over from 1 (0 is
+                 * skipped). */
+                available = false;
+                break;
+            }
+            if (!initial && conj_id == initial_id) {
+                /* It has checked all ids (extreme situation, not expected in
+                 * real environment). */
+                return 0;
+            }
+            initial = false;
+            struct conj_id_node *conj_id_node;
+            /* conj_id is both the key and the hash */
+            HMAP_FOR_EACH_WITH_HASH (conj_id_node, hmap_node, conj_id,
+                                     &conj_ids->conj_id_allocations) {
+                if (conj_id_node->conj_id == conj_id) {
+                    available = false;
+                    COVERAGE_INC(lflow_conj_conflict);
+                    break;
+                }
+            }
+            if (!available) {
+                break;
+            }
+            conj_id++;
+        }
+        if (available) {
+            break;
+        }
+        start_conj_id = conj_id + 1;
+    }
+    lflow_conj_ids_insert_(conj_ids, lflow_uuid, start_conj_id, n_conjs);
+    return start_conj_id;
+}
+
+/* Similar to lflow_conj_ids_alloc, except that it takes an extra parameter
+ * start_conj_id, which specifies the desired conjunction ids to be allocated,
+ * and if they are unavailable, return false directly without trying to find
+ * the next available ones. It returns true if the specified range is
+ * allocated successfully. */
+bool
+lflow_conj_ids_alloc_specified(struct conj_ids *conj_ids,
+                               const struct uuid *lflow_uuid,
+                               uint32_t start_conj_id, uint32_t n_conjs)
+{
+    if (!n_conjs) {
+        return false;
+    }
+    uint32_t conj_id = start_conj_id;
+    for (uint32_t i = 0; i < n_conjs; i++) {
+        ovs_assert(conj_id);
+        struct conj_id_node *conj_id_node;
+        HMAP_FOR_EACH_WITH_HASH (conj_id_node, hmap_node, conj_id,
+                                 &conj_ids->conj_id_allocations) {
+            if (conj_id_node->conj_id == conj_id) {
+                return false;
+            }
+        }
+        conj_id++;
+    }
+    lflow_conj_ids_insert_(conj_ids, lflow_uuid, start_conj_id, n_conjs);
+    return true;
+}
+
+/* Frees the conjunction IDs used by lflow_uuid. */
+void
+lflow_conj_ids_free(struct conj_ids *conj_ids, const struct uuid *lflow_uuid)
+{
+    struct lflow_conj_node *lflow_conj;
+    bool found = false;
+    HMAP_FOR_EACH_WITH_HASH (lflow_conj, hmap_node, uuid_hash(lflow_uuid),
+                             &conj_ids->lflow_conj_ids) {
+        if (uuid_equals(&lflow_conj->lflow_uuid, lflow_uuid)) {
+            found = true;
+            break;
+        }
+    }
+    if (!found) {
+        return;
+    }
+    ovs_assert(lflow_conj->n_conjs);
+    uint32_t conj_id = lflow_conj->start_conj_id;
+    for (uint32_t i = 0; i < lflow_conj->n_conjs; i++) {
+        ovs_assert(conj_id);
+        struct conj_id_node *conj_id_node;
+        HMAP_FOR_EACH_WITH_HASH (conj_id_node, hmap_node, conj_id,
+                                 &conj_ids->conj_id_allocations) {
+            if (conj_id_node->conj_id == conj_id) {
+                hmap_remove(&conj_ids->conj_id_allocations,
+                            &conj_id_node->hmap_node);
+                free(conj_id_node);
+                break;
+            }
+        }
+        conj_id++;
+    }
+
+    hmap_remove(&conj_ids->lflow_conj_ids, &lflow_conj->hmap_node);
+    free(lflow_conj);
+}
+
+void
+lflow_conj_ids_init(struct conj_ids *conj_ids)
+{
+    hmap_init(&conj_ids->conj_id_allocations);
+    hmap_init(&conj_ids->lflow_conj_ids);
+}
+
+void
+lflow_conj_ids_destroy(struct conj_ids *conj_ids) {
+    struct conj_id_node *conj_id_node, *next;
+    HMAP_FOR_EACH_SAFE (conj_id_node, next, hmap_node,
+                        &conj_ids->conj_id_allocations) {
+        hmap_remove(&conj_ids->conj_id_allocations, &conj_id_node->hmap_node);
+        free(conj_id_node);
+    }
+    hmap_destroy(&conj_ids->conj_id_allocations);
+
+    struct lflow_conj_node *lflow_conj, *l_c_next;
+    HMAP_FOR_EACH_SAFE (lflow_conj, l_c_next, hmap_node,
+                        &conj_ids->lflow_conj_ids) {
+        hmap_remove(&conj_ids->lflow_conj_ids, &lflow_conj->hmap_node);
+        free(lflow_conj);
+    }
+    hmap_destroy(&conj_ids->lflow_conj_ids);
+}
+
+void lflow_conj_ids_clear(struct conj_ids *conj_ids) {
+    lflow_conj_ids_destroy(conj_ids);
+    lflow_conj_ids_init(conj_ids);
+}
diff --git a/controller/lflow-conj-ids.h b/controller/lflow-conj-ids.h
new file mode 100644
index 000000000..4fcabfe48
--- /dev/null
+++ b/controller/lflow-conj-ids.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LFLOW_CONJ_IDS_H
+#define LFLOW_CONJ_IDS_H 1
+
+#include "openvswitch/hmap.h"
+#include "uuid.h"
+
+struct conj_ids {
+    /* Allocated conjunction ids. Contains struct conj_id_node. */
+    struct hmap conj_id_allocations;
+    /* A map from lflows to the conjunction ids used. Contains struct
+     * lflow_conj_node. */
+    struct hmap lflow_conj_ids;
+};
+
+struct conj_id_node {
+    struct hmap_node hmap_node;
+    uint32_t conj_id;
+};
+
+struct lflow_conj_node {
+    struct hmap_node hmap_node;
+    struct uuid lflow_uuid;
+    uint32_t start_conj_id;
+    uint32_t n_conjs;
+};
+
+uint32_t lflow_conj_ids_alloc(struct conj_ids *, const struct uuid *lflow_uuid,
+                              uint32_t n_conjs);
+bool lflow_conj_ids_alloc_specified(struct conj_ids *,
+                                    const struct uuid *lflow_uuid,
+                                    uint32_t start_conj_id, uint32_t n_conjs);
+void lflow_conj_ids_free(struct conj_ids *, const struct uuid *lflow_uuid);
+void lflow_conj_ids_init(struct conj_ids *);
+void lflow_conj_ids_destroy(struct conj_ids *);
+void lflow_conj_ids_clear(struct conj_ids *);
+
+#endif /* controller/lflow-conj-ids.h */
diff --git a/controller/lflow.c b/controller/lflow.c
index 59f5d3a07..01908cf71 100644
--- a/controller/lflow.c
+++ b/controller/lflow.c
@@ -73,7 +73,7 @@ struct condition_aux {
     struct lflow_resource_ref *lfrr;
 };
 
-static bool
+static void
 consider_logical_flow(const struct sbrec_logical_flow *lflow,
                       struct hmap *dhcp_opts, struct hmap *dhcpv6_opts,
                       struct hmap *nd_ra_opts,
@@ -365,14 +365,9 @@ add_logical_flows(struct lflow_ctx_in *l_ctx_in,
     controller_event_opts_init(&controller_event_opts);
 
     SBREC_LOGICAL_FLOW_TABLE_FOR_EACH (lflow, l_ctx_in->logical_flow_table) {
-        if (!consider_logical_flow(lflow, &dhcp_opts, &dhcpv6_opts,
-                                   &nd_ra_opts, &controller_event_opts,
-                                   l_ctx_in, l_ctx_out)) {
-            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 5);
-            VLOG_ERR_RL(&rl, "Conjunction id overflow when processing lflow "
-                        UUID_FMT, UUID_ARGS(&lflow->header_.uuid));
-            l_ctx_out->conj_id_overflow = true;
-        }
+        consider_logical_flow(lflow, &dhcp_opts, &dhcpv6_opts,
+                              &nd_ra_opts, &controller_event_opts,
+                              l_ctx_in, l_ctx_out);
     }
 
     dhcp_opts_destroy(&dhcp_opts);
@@ -434,18 +429,17 @@ lflow_handle_changed_flows(struct lflow_ctx_in *l_ctx_in,
     HMAP_FOR_EACH (ofrn, hmap_node, &flood_remove_nodes) {
         /* Delete entries from lflow resource reference. */
         lflow_resource_destroy_lflow(l_ctx_out->lfrr, &ofrn->sb_uuid);
+        /* Delete conj_ids owned by the lflow. */
+        lflow_conj_ids_free(l_ctx_out->conj_ids, &ofrn->sb_uuid);
         /* Reprocessing the lflow if the sb record is not deleted. */
         lflow = sbrec_logical_flow_table_get_for_uuid(
             l_ctx_in->logical_flow_table, &ofrn->sb_uuid);
         if (lflow) {
             VLOG_DBG("re-add lflow "UUID_FMT,
                      UUID_ARGS(&lflow->header_.uuid));
-            if (!consider_logical_flow(lflow, &dhcp_opts, &dhcpv6_opts,
-                                       &nd_ra_opts, &controller_event_opts,
-                                       l_ctx_in, l_ctx_out)) {
-                ret = false;
-                break;
-            }
+            consider_logical_flow(lflow, &dhcp_opts, &dhcpv6_opts,
+                                  &nd_ra_opts, &controller_event_opts,
+                                  l_ctx_in, l_ctx_out);
         }
     }
     HMAP_FOR_EACH_SAFE (ofrn, next, hmap_node, &flood_remove_nodes) {
@@ -528,6 +522,7 @@ lflow_handle_changed_ref(enum ref_type ref_type, const char *ref_name,
     /* Secondly, for each lflow that is actually removed, reprocessing it. */
     HMAP_FOR_EACH (ofrn, hmap_node, &flood_remove_nodes) {
         lflow_resource_destroy_lflow(l_ctx_out->lfrr, &ofrn->sb_uuid);
+        lflow_conj_ids_free(l_ctx_out->conj_ids, &ofrn->sb_uuid);
 
         const struct sbrec_logical_flow *lflow =
             sbrec_logical_flow_table_get_for_uuid(l_ctx_in->logical_flow_table,
@@ -540,13 +535,9 @@ lflow_handle_changed_ref(enum ref_type ref_type, const char *ref_name,
             continue;
         }
 
-        if (!consider_logical_flow(lflow, &dhcp_opts, &dhcpv6_opts,
-                                   &nd_ra_opts, &controller_event_opts,
-                                   l_ctx_in, l_ctx_out)) {
-            ret = false;
-            l_ctx_out->conj_id_overflow = true;
-            break;
-        }
+        consider_logical_flow(lflow, &dhcp_opts, &dhcpv6_opts,
+                              &nd_ra_opts, &controller_event_opts,
+                              l_ctx_in, l_ctx_out);
         *changed = true;
     }
     HMAP_FOR_EACH_SAFE (ofrn, ofrn_next, hmap_node, &flood_remove_nodes) {
@@ -568,17 +559,6 @@ lflow_handle_changed_ref(enum ref_type ref_type, const char *ref_name,
     return ret;
 }
 
-static bool
-update_conj_id_ofs(uint32_t *conj_id_ofs, uint32_t n_conjs)
-{
-    if (*conj_id_ofs + n_conjs < *conj_id_ofs) {
-        /* overflow */
-        return true;
-    }
-    *conj_id_ofs += n_conjs;
-    return false;
-}
-
 static void
 lflow_parse_ctrl_meter(const struct sbrec_logical_flow *lflow,
                        struct ovn_extend_table *meter_table,
@@ -762,7 +742,7 @@ convert_match_to_expr(const struct sbrec_logical_flow *lflow,
     return expr_simplify(e);
 }
 
-static bool
+static void
 consider_logical_flow__(const struct sbrec_logical_flow *lflow,
                         const struct sbrec_datapath_binding *dp,
                         struct hmap *dhcp_opts, struct hmap *dhcpv6_opts,
@@ -774,7 +754,7 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
     if (!get_local_datapath(l_ctx_in->local_datapaths, dp->tunnel_key)) {
         VLOG_DBG("lflow "UUID_FMT" is not for local datapath, skip",
                  UUID_ARGS(&lflow->header_.uuid));
-        return true;
+        return;
     }
 
     const char *io_port = smap_get(&lflow->tags, "in_out_port");
@@ -787,14 +767,14 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
         if (!pb) {
             VLOG_DBG("lflow "UUID_FMT" matches inport/outport %s that's not "
                      "found, skip", UUID_ARGS(&lflow->header_.uuid), io_port);
-            return true;
+            return;
         }
         char buf[16];
         get_unique_lport_key(dp->tunnel_key, pb->tunnel_key, buf, sizeof buf);
         if (!sset_contains(l_ctx_in->related_lport_ids, buf)) {
             VLOG_DBG("lflow "UUID_FMT" matches inport/outport %s that's not "
                      "local, skip", UUID_ARGS(&lflow->header_.uuid), io_port);
-            return true;
+            return;
         }
     }
 
@@ -837,7 +817,7 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
         free(error);
         ovnacts_free(ovnacts.data, ovnacts.size);
         ofpbuf_uninit(&ovnacts);
-        return true;
+        return;
     }
 
     struct lookup_port_aux aux = {
@@ -859,8 +839,6 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
 
     struct lflow_cache_value *lcv =
         lflow_cache_get(l_ctx_out->lflow_cache, &lflow->header_.uuid);
-    uint32_t conj_id_ofs =
-        lcv ? lcv->conj_id_ofs : *l_ctx_out->conj_id_ofs;
     enum lflow_cache_type lcv_type =
         lcv ? lcv->type : LCACHE_T_NONE;
 
@@ -869,9 +847,19 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
     size_t matches_size = 0;
 
     bool pg_addr_set_ref = false;
-    uint32_t n_conjs = 0;
 
-    bool conj_id_overflow = false;
+    if (lcv_type == LCACHE_T_MATCHES
+        && lcv->n_conjs
+        && !lflow_conj_ids_alloc_specified(l_ctx_out->conj_ids,
+                                           &lflow->header_.uuid,
+                                           lcv->conj_id_ofs, lcv->n_conjs)) {
+        /* This should happen very rarely. */
+        VLOG_DBG("lflow "UUID_FMT" match cached with conjunctions, but the"
+                 " cached ids are not available anymore. Drop the cache.",
+                 UUID_ARGS(&lflow->header_.uuid));
+        lflow_cache_delete(l_ctx_out->lflow_cache, &lflow->header_.uuid);
+        lcv_type = LCACHE_T_NONE;
+    }
 
     /* Get match expr, either from cache or from lflow match. */
     switch (lcv_type) {
@@ -912,17 +900,21 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
     }
 
     /* Get matches, either from cache or from expr computed above. */
+    uint32_t start_conj_id = 0;
+    uint32_t n_conjs = 0;
     switch (lcv_type) {
     case LCACHE_T_NONE:
     case LCACHE_T_EXPR:
         matches = xmalloc(sizeof *matches);
         n_conjs = expr_to_matches(expr, lookup_port_cb, &aux, matches);
-        matches_size = expr_matches_prepare(matches, conj_id_ofs);
         if (hmap_is_empty(matches)) {
             VLOG_DBG("lflow "UUID_FMT" matches are empty, skip",
-                    UUID_ARGS(&lflow->header_.uuid));
+                     UUID_ARGS(&lflow->header_.uuid));
             goto done;
         }
+        start_conj_id = lflow_conj_ids_alloc(l_ctx_out->conj_ids,
+                                           &lflow->header_.uuid, n_conjs);
+        matches_size = expr_matches_prepare(matches, start_conj_id - 1);
         break;
     case LCACHE_T_MATCHES:
         matches = lcv->expr_matches;
@@ -935,23 +927,18 @@ consider_logical_flow__(const struct sbrec_logical_flow *lflow,
     /* Update cache if needed. */
     switch (lcv_type) {
     case LCACHE_T_NONE:
-        /* Entry not already in cache, update conjunction id offset and
-         * add the entry to the cache.
-         */
-        conj_id_overflow = update_conj_id_ofs(l_ctx_out->conj_id_ofs, n_conjs);
-
         /* Cache new entry if caching is enabled. */
         if (lflow_cache_is_enabled(l_ctx_out->lflow_cache)) {
             if (cached_expr
                 && !lflow_ref_lookup(&l_ctx_out->lfrr->lflow_ref_table,
                                      &lflow->header_.uuid)) {
                 lflow_cache_add_matches(l_ctx_out->lflow_cache,
-                                        &lflow->header_.uuid, matches,
-                                        matches_size);
+                                        &lflow->header_.uuid, start_conj_id,
+                                        n_conjs, matches, matches_size);
                 matches = NULL;
             } else if (cached_expr) {
                 lflow_cache_add_expr(l_ctx_out->lflow_cache,
-                                     &lflow->header_.uuid, conj_id_ofs,
+                                     &lflow->header_.uuid,
                                      cached_expr, expr_size(cached_expr));
                 cached_expr = NULL;
             }
@@ -973,10 +960,9 @@ done:
     expr_destroy(cached_expr);
     expr_matches_destroy(matches);
     free(matches);
-    return !conj_id_overflow;
 }
 
-static bool
+static void
 consider_logical_flow(const struct sbrec_logical_flow *lflow,
                       struct hmap *dhcp_opts, struct hmap *dhcpv6_opts,
                       struct hmap *nd_ra_opts,
@@ -986,30 +972,27 @@ consider_logical_flow(const struct sbrec_logical_flow *lflow,
 {
     const struct sbrec_logical_dp_group *dp_group = lflow->logical_dp_group;
     const struct sbrec_datapath_binding *dp = lflow->logical_datapath;
-    bool ret = true;
 
     if (!dp_group && !dp) {
         VLOG_DBG("lflow "UUID_FMT" has no datapath binding, skip",
                  UUID_ARGS(&lflow->header_.uuid));
-        return true;
+        return;
     }
     ovs_assert(!dp_group || !dp);
 
-    if (dp && !consider_logical_flow__(lflow, dp,
-                                       dhcp_opts, dhcpv6_opts, nd_ra_opts,
-                                       controller_event_opts,
-                                       l_ctx_in, l_ctx_out)) {
-        ret = false;
+    if (dp) {
+        consider_logical_flow__(lflow, dp,
+                                dhcp_opts, dhcpv6_opts, nd_ra_opts,
+                                controller_event_opts,
+                                l_ctx_in, l_ctx_out);
+        return;
     }
     for (size_t i = 0; dp_group && i < dp_group->n_datapaths; i++) {
-        if (!consider_logical_flow__(lflow, dp_group->datapaths[i],
-                                     dhcp_opts,  dhcpv6_opts, nd_ra_opts,
-                                     controller_event_opts,
-                                     l_ctx_in, l_ctx_out)) {
-            ret = false;
-        }
+        consider_logical_flow__(lflow, dp_group->datapaths[i],
+                                dhcp_opts,  dhcpv6_opts, nd_ra_opts,
+                                controller_event_opts,
+                                l_ctx_in, l_ctx_out);
     }
-    return ret;
 }
 
 static void
@@ -1933,13 +1916,9 @@ lflow_add_flows_for_datapath(const struct sbrec_datapath_binding *dp,
     const struct sbrec_logical_flow *lflow;
     SBREC_LOGICAL_FLOW_FOR_EACH_EQUAL (
         lflow, lf_row, l_ctx_in->sbrec_logical_flow_by_logical_datapath) {
-        if (!consider_logical_flow__(lflow, dp, &dhcp_opts, &dhcpv6_opts,
-                                     &nd_ra_opts, &controller_event_opts,
-                                     l_ctx_in, l_ctx_out)) {
-            handled = false;
-            l_ctx_out->conj_id_overflow = true;
-            goto lflow_processing_end;
-        }
+        consider_logical_flow__(lflow, dp, &dhcp_opts, &dhcpv6_opts,
+                                &nd_ra_opts, &controller_event_opts,
+                                l_ctx_in, l_ctx_out);
     }
     sbrec_logical_flow_index_destroy_row(lf_row);
 
@@ -1963,16 +1942,11 @@ lflow_add_flows_for_datapath(const struct sbrec_datapath_binding *dp,
         sbrec_logical_flow_index_set_logical_dp_group(lf_row, ldpg);
         SBREC_LOGICAL_FLOW_FOR_EACH_EQUAL (
             lflow, lf_row, l_ctx_in->sbrec_logical_flow_by_logical_dp_group) {
-            if (!consider_logical_flow__(lflow, dp, &dhcp_opts, &dhcpv6_opts,
-                                         &nd_ra_opts, &controller_event_opts,
-                                         l_ctx_in, l_ctx_out)) {
-                handled = false;
-                l_ctx_out->conj_id_overflow = true;
-                goto lflow_processing_end;
-            }
+            consider_logical_flow__(lflow, dp, &dhcp_opts, &dhcpv6_opts,
+                                    &nd_ra_opts, &controller_event_opts,
+                                    l_ctx_in, l_ctx_out);
         }
     }
-lflow_processing_end:
     sbrec_logical_flow_index_destroy_row(lf_row);
 
     dhcp_opts_destroy(&dhcp_opts);
diff --git a/controller/lflow.h b/controller/lflow.h
index 61ffbc0cc..b2f3385e5 100644
--- a/controller/lflow.h
+++ b/controller/lflow.h
@@ -35,6 +35,7 @@
 
 #include <stdint.h>
 #include "lflow-cache.h"
+#include "lflow-conj-ids.h"
 #include "openvswitch/hmap.h"
 #include "openvswitch/uuid.h"
 #include "openvswitch/list.h"
@@ -156,8 +157,7 @@ struct lflow_ctx_out {
     struct ovn_extend_table *meter_table;
     struct lflow_resource_ref *lfrr;
     struct lflow_cache *lflow_cache;
-    uint32_t *conj_id_ofs;
-    bool conj_id_overflow;
+    struct conj_ids *conj_ids;
     struct simap *hairpin_lb_ids;
     struct id_pool *hairpin_id_pool;
 };
diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c
index 92ba50d65..986ac4e03 100644
--- a/controller/ovn-controller.c
+++ b/controller/ovn-controller.c
@@ -39,6 +39,7 @@
 #include "openvswitch/hmap.h"
 #include "lflow.h"
 #include "lflow-cache.h"
+#include "lflow-conj-ids.h"
 #include "lib/vswitch-idl.h"
 #include "local_data.h"
 #include "lport.h"
@@ -2146,7 +2147,6 @@ non_vif_data_ovs_iface_handler(struct engine_node *node, void *data OVS_UNUSED)
 }
 
 struct lflow_output_persistent_data {
-    uint32_t conj_id_ofs;
     struct lflow_cache *lflow_cache;
 };
 
@@ -2164,6 +2164,8 @@ struct ed_type_lflow_output {
     struct ovn_extend_table meter_table;
     /* lflow resource cross reference */
     struct lflow_resource_ref lflow_resource_ref;
+    /* conjunciton ID usage information of lflows */
+    struct conj_ids conj_ids;
 
     /* Data which is persistent and not cleared during
      * full recompute. */
@@ -2289,9 +2291,8 @@ init_lflow_ctx(struct engine_node *node,
     l_ctx_out->group_table = &fo->group_table;
     l_ctx_out->meter_table = &fo->meter_table;
     l_ctx_out->lfrr = &fo->lflow_resource_ref;
-    l_ctx_out->conj_id_ofs = &fo->pd.conj_id_ofs;
+    l_ctx_out->conj_ids = &fo->conj_ids;
     l_ctx_out->lflow_cache = fo->pd.lflow_cache;
-    l_ctx_out->conj_id_overflow = false;
     l_ctx_out->hairpin_id_pool = fo->hd.pool;
     l_ctx_out->hairpin_lb_ids = &fo->hd.ids;
 }
@@ -2304,8 +2305,8 @@ en_lflow_output_init(struct engine_node *node OVS_UNUSED,
     ovn_desired_flow_table_init(&data->flow_table);
     ovn_extend_table_init(&data->group_table);
     ovn_extend_table_init(&data->meter_table);
-    data->pd.conj_id_ofs = 1;
     lflow_resource_init(&data->lflow_resource_ref);
+    lflow_conj_ids_init(&data->conj_ids);
     simap_init(&data->hd.ids);
     data->hd.pool = id_pool_create(1, UINT32_MAX - 1);
     return data;
@@ -2319,6 +2320,7 @@ en_lflow_output_cleanup(void *data)
     ovn_extend_table_destroy(&flow_output_data->group_table);
     ovn_extend_table_destroy(&flow_output_data->meter_table);
     lflow_resource_destroy(&flow_output_data->lflow_resource_ref);
+    lflow_conj_ids_destroy(&flow_output_data->conj_ids);
     lflow_cache_destroy(flow_output_data->pd.lflow_cache);
     simap_destroy(&flow_output_data->hd.ids);
     id_pool_destroy(flow_output_data->hd.pool);
@@ -2367,37 +2369,18 @@ en_lflow_output_run(struct engine_node *node, void *data)
         ovn_extend_table_clear(group_table, false /* desired */);
         ovn_extend_table_clear(meter_table, false /* desired */);
         lflow_resource_clear(lfrr);
+        lflow_conj_ids_clear(&fo->conj_ids);
     }
 
     struct controller_engine_ctx *ctrl_ctx = engine_get_context()->client_ctx;
 
     fo->pd.lflow_cache = ctrl_ctx->lflow_cache;
 
-    if (!lflow_cache_is_enabled(fo->pd.lflow_cache)) {
-        fo->pd.conj_id_ofs = 1;
-    }
-
     struct lflow_ctx_in l_ctx_in;
     struct lflow_ctx_out l_ctx_out;
     init_lflow_ctx(node, rt_data, non_vif_data, fo, &l_ctx_in, &l_ctx_out);
     lflow_run(&l_ctx_in, &l_ctx_out);
 
-    if (l_ctx_out.conj_id_overflow) {
-        /* Conjunction ids overflow. There can be many holes in between.
-         * Destroy lflow cache and call lflow_run() again. */
-        ovn_desired_flow_table_clear(lflow_table);
-        ovn_extend_table_clear(group_table, false /* desired */);
-        ovn_extend_table_clear(meter_table, false /* desired */);
-        lflow_resource_clear(lfrr);
-        fo->pd.conj_id_ofs = 1;
-        lflow_cache_flush(fo->pd.lflow_cache);
-        l_ctx_out.conj_id_overflow = false;
-        lflow_run(&l_ctx_in, &l_ctx_out);
-        if (l_ctx_out.conj_id_overflow) {
-            VLOG_WARN("Conjunction id overflow.");
-        }
-    }
-
     engine_set_node_state(node, EN_UPDATED);
 }
 
@@ -4079,7 +4062,6 @@ lflow_cache_flush_cmd(struct unixctl_conn *conn OVS_UNUSED,
     VLOG_INFO("User triggered lflow cache flush.");
     struct lflow_output_persistent_data *fo_pd = arg_;
     lflow_cache_flush(fo_pd->lflow_cache);
-    fo_pd->conj_id_ofs = 1;
     engine_set_force_recompute(true);
     poll_immediate_wake();
     unixctl_command_reply(conn, NULL);
diff --git a/controller/test-lflow-cache.c b/controller/test-lflow-cache.c
index 95e619ee0..6f7a3c389 100644
--- a/controller/test-lflow-cache.c
+++ b/controller/test-lflow-cache.c
@@ -36,19 +36,22 @@ static void
 test_lflow_cache_add__(struct lflow_cache *lc, const char *op_type,
                        const struct uuid *lflow_uuid,
                        unsigned int conj_id_ofs,
+                       unsigned int n_conjs,
                        struct expr *e)
 {
     printf("ADD %s:\n", op_type);
     printf("  conj-id-ofs: %u\n", conj_id_ofs);
+    printf("  n_conjs: %u\n", n_conjs);
 
     if (!strcmp(op_type, "expr")) {
-        lflow_cache_add_expr(lc, lflow_uuid, conj_id_ofs, expr_clone(e),
+        lflow_cache_add_expr(lc, lflow_uuid, expr_clone(e),
                              TEST_LFLOW_CACHE_VALUE_SIZE);
     } else if (!strcmp(op_type, "matches")) {
         struct hmap *matches = xmalloc(sizeof *matches);
         ovs_assert(expr_to_matches(e, NULL, NULL, matches) == 0);
         ovs_assert(hmap_count(matches) == 1);
-        lflow_cache_add_matches(lc, lflow_uuid, matches,
+        lflow_cache_add_matches(lc, lflow_uuid,
+                                conj_id_ofs, n_conjs, matches,
                                 TEST_LFLOW_CACHE_VALUE_SIZE);
     } else {
         OVS_NOT_REACHED();
@@ -68,6 +71,7 @@ test_lflow_cache_lookup__(struct lflow_cache *lc,
     }
 
     printf("  conj_id_ofs: %"PRIu32"\n", lcv->conj_id_ofs);
+    printf("  n_conjs: %"PRIu32"\n", lcv->n_conjs);
     switch (lcv->type) {
     case LCACHE_T_EXPR:
         printf("  type: expr\n");
@@ -139,6 +143,12 @@ test_lflow_cache_operations(struct ovs_cmdl_context *ctx)
                 goto done;
             }
 
+            unsigned int n_conjs;
+            if (!test_read_uint_value(ctx, shift++, "n_conjs",
+                                      &n_conjs)) {
+                goto done;
+            }
+
             if (n_lflow_uuids == n_allocated_lflow_uuids) {
                 lflow_uuids = x2nrealloc(lflow_uuids, &n_allocated_lflow_uuids,
                                          sizeof *lflow_uuids);
@@ -146,7 +156,8 @@ test_lflow_cache_operations(struct ovs_cmdl_context *ctx)
             struct uuid *lflow_uuid = &lflow_uuids[n_lflow_uuids++];
 
             uuid_generate(lflow_uuid);
-            test_lflow_cache_add__(lc, op_type, lflow_uuid, conj_id_ofs, e);
+            test_lflow_cache_add__(lc, op_type, lflow_uuid, conj_id_ofs,
+                                   n_conjs, e);
             test_lflow_cache_lookup__(lc, lflow_uuid);
         } else if (!strcmp(op, "add-del")) {
             const char *op_type = test_read_value(ctx, shift++, "op_type");
@@ -160,9 +171,16 @@ test_lflow_cache_operations(struct ovs_cmdl_context *ctx)
                 goto done;
             }
 
+            unsigned int n_conjs;
+            if (!test_read_uint_value(ctx, shift++, "n_conjs",
+                                      &n_conjs)) {
+                goto done;
+            }
+
             struct uuid lflow_uuid;
             uuid_generate(&lflow_uuid);
-            test_lflow_cache_add__(lc, op_type, &lflow_uuid, conj_id_ofs, e);
+            test_lflow_cache_add__(lc, op_type, &lflow_uuid, conj_id_ofs,
+                                   n_conjs, e);
             test_lflow_cache_lookup__(lc, &lflow_uuid);
             test_lflow_cache_delete__(lc, &lflow_uuid);
             test_lflow_cache_lookup__(lc, &lflow_uuid);
@@ -246,10 +264,10 @@ test_lflow_cache_negative(struct ovs_cmdl_context *ctx OVS_UNUSED)
         ovs_assert(expr_to_matches(e, NULL, NULL, matches) == 0);
         ovs_assert(hmap_count(matches) == 1);
 
-        lflow_cache_add_expr(lcs[i], NULL, 0, NULL, 0);
-        lflow_cache_add_expr(lcs[i], NULL, 0, e, expr_size(e));
-        lflow_cache_add_matches(lcs[i], NULL, NULL, 0);
-        lflow_cache_add_matches(lcs[i], NULL, matches,
+        lflow_cache_add_expr(lcs[i], NULL, NULL, 0);
+        lflow_cache_add_expr(lcs[i], NULL, e, expr_size(e));
+        lflow_cache_add_matches(lcs[i], NULL, 0, 0, NULL, 0);
+        lflow_cache_add_matches(lcs[i], NULL, 0, 0, matches,
                                 TEST_LFLOW_CACHE_VALUE_SIZE);
         lflow_cache_destroy(lcs[i]);
     }
@@ -260,7 +278,7 @@ test_lflow_cache_main(int argc, char *argv[])
 {
     set_program_name(argv[0]);
     static const struct ovs_cmdl_command commands[] = {
-        {"lflow_cache_operations", NULL, 3, INT_MAX,
+        {"lflow_cache_operations", NULL, 4, INT_MAX,
          test_lflow_cache_operations, OVS_RO},
         {"lflow_cache_negative", NULL, 0, 0,
          test_lflow_cache_negative, OVS_RO},
diff --git a/tests/ovn-lflow-cache.at b/tests/ovn-lflow-cache.at
index 97d24d0dc..593d3eaac 100644
--- a/tests/ovn-lflow-cache.at
+++ b/tests/ovn-lflow-cache.at
@@ -7,8 +7,8 @@ AT_SETUP([unit test -- lflow-cache single add/lookup])
 AT_CHECK(
     [ovstest test-lflow-cache lflow_cache_operations \
         true 2 \
-        add expr 2 \
-        add matches 3 | grep -v 'Mem usage (KB)'],
+        add expr 2 1 \
+        add matches 3 2 | grep -v 'Mem usage (KB)'],
     [0], [dnl
 Enabled: true
 high-watermark  : 0
@@ -18,8 +18,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 2
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 2
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 1
@@ -29,8 +31,10 @@ cache-matches   : 0
 trim count      : 0
 ADD matches:
   conj-id-ofs: 3
+  n_conjs: 2
 LOOKUP:
-  conj_id_ofs: 0
+  conj_id_ofs: 3
+  n_conjs: 2
   type: matches
 Enabled: true
 high-watermark  : 2
@@ -45,8 +49,8 @@ AT_SETUP([unit test -- lflow-cache single add/lookup/del])
 AT_CHECK(
     [ovstest test-lflow-cache lflow_cache_operations \
         true 2 \
-        add-del expr 2 \
-        add-del matches 3 | grep -v 'Mem usage (KB)'],
+        add-del expr 2 1 \
+        add-del matches 3 1 | grep -v 'Mem usage (KB)'],
     [0], [dnl
 Enabled: true
 high-watermark  : 0
@@ -56,8 +60,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 2
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 2
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 DELETE
 LOOKUP:
@@ -70,8 +76,10 @@ cache-matches   : 0
 trim count      : 0
 ADD matches:
   conj-id-ofs: 3
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 0
+  conj_id_ofs: 3
+  n_conjs: 1
   type: matches
 DELETE
 LOOKUP:
@@ -89,8 +97,8 @@ AT_SETUP([unit test -- lflow-cache disabled single add/lookup/del])
 AT_CHECK(
     [ovstest test-lflow-cache lflow_cache_operations \
         false 2 \
-        add expr 2 \
-        add matches 3 | grep -v 'Mem usage (KB)'],
+        add expr 2 1 \
+        add matches 3 1 | grep -v 'Mem usage (KB)'],
     [0], [dnl
 Enabled: false
 high-watermark  : 0
@@ -100,6 +108,7 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 2
+  n_conjs: 1
 LOOKUP:
   not found
 Enabled: false
@@ -110,6 +119,7 @@ cache-matches   : 0
 trim count      : 0
 ADD matches:
   conj-id-ofs: 3
+  n_conjs: 1
 LOOKUP:
   not found
 Enabled: false
@@ -125,14 +135,14 @@ AT_SETUP([unit test -- lflow-cache disable/enable/flush])
 AT_CHECK(
     [ovstest test-lflow-cache lflow_cache_operations \
         true 9 \
-        add expr 2 \
-        add matches 3 \
+        add expr 2 1 \
+        add matches 3 1 \
         disable \
-        add expr 5 \
-        add matches 6 \
+        add expr 5 1 \
+        add matches 6 1 \
         enable 1000 1024 \
-        add expr 8 \
-        add matches 9 \
+        add expr 8 1 \
+        add matches 9 1 \
         flush | grep -v 'Mem usage (KB)'],
     [0], [dnl
 Enabled: true
@@ -143,8 +153,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 2
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 2
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 1
@@ -154,8 +166,10 @@ cache-matches   : 0
 trim count      : 0
 ADD matches:
   conj-id-ofs: 3
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 0
+  conj_id_ofs: 3
+  n_conjs: 1
   type: matches
 Enabled: true
 high-watermark  : 2
@@ -173,6 +187,7 @@ dnl At "disable" the cache was flushed.
 trim count      : 1
 ADD expr:
   conj-id-ofs: 5
+  n_conjs: 1
 LOOKUP:
   not found
 Enabled: false
@@ -183,6 +198,7 @@ cache-matches   : 0
 trim count      : 1
 ADD matches:
   conj-id-ofs: 6
+  n_conjs: 1
 LOOKUP:
   not found
 Enabled: false
@@ -200,8 +216,10 @@ cache-matches   : 0
 trim count      : 1
 ADD expr:
   conj-id-ofs: 8
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 8
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 1
@@ -211,8 +229,10 @@ cache-matches   : 0
 trim count      : 1
 ADD matches:
   conj-id-ofs: 9
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 0
+  conj_id_ofs: 9
+  n_conjs: 1
   type: matches
 Enabled: true
 high-watermark  : 2
@@ -234,15 +254,15 @@ AT_SETUP([unit test -- lflow-cache set limit])
 AT_CHECK(
     [ovstest test-lflow-cache lflow_cache_operations \
         true 9 \
-        add expr 2 \
-        add matches 3 \
+        add expr 2 1 \
+        add matches 3 1 \
         enable 1 1024 \
-        add expr 5 \
-        add matches 6 \
-        add expr 7 \
+        add expr 5 1 \
+        add matches 6 1 \
+        add expr 7 1 \
         enable 1 1 \
-        add expr 9 \
-        add matches 10 | grep -v 'Mem usage (KB)'],
+        add expr 9 1 \
+        add matches 10 1 | grep -v 'Mem usage (KB)'],
     [0], [dnl
 Enabled: true
 high-watermark  : 0
@@ -252,8 +272,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 2
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 2
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 1
@@ -263,8 +285,10 @@ cache-matches   : 0
 trim count      : 0
 ADD matches:
   conj-id-ofs: 3
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 0
+  conj_id_ofs: 3
+  n_conjs: 1
   type: matches
 Enabled: true
 high-watermark  : 2
@@ -284,8 +308,10 @@ cache-matches   : 0
 trim count      : 1
 ADD expr:
   conj-id-ofs: 5
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 5
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 1
@@ -295,8 +321,10 @@ cache-matches   : 0
 trim count      : 1
 ADD matches:
   conj-id-ofs: 6
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 0
+  conj_id_ofs: 6
+  n_conjs: 1
   type: matches
 dnl
 dnl Cache is full but we can evict the expr entry because we're adding
@@ -310,6 +338,7 @@ cache-matches   : 1
 trim count      : 1
 ADD expr:
   conj-id-ofs: 7
+  n_conjs: 1
 LOOKUP:
   not found
 dnl
@@ -335,6 +364,7 @@ cache-matches   : 0
 trim count      : 2
 ADD expr:
   conj-id-ofs: 9
+  n_conjs: 1
 LOOKUP:
   not found
 dnl
@@ -349,6 +379,7 @@ cache-matches   : 0
 trim count      : 2
 ADD matches:
   conj-id-ofs: 10
+  n_conjs: 1
 LOOKUP:
   not found
 dnl
@@ -369,11 +400,11 @@ AT_CHECK(
     [ovstest test-lflow-cache lflow_cache_operations \
         true 12 \
         enable 1000 1024 trim-limit 100 trim-wmark-perc 100 \
-        add expr 1 \
-        add expr 2 \
-        add expr 3 \
-        add expr 4 \
-        add expr 5 \
+        add expr 1 1 \
+        add expr 2 1 \
+        add expr 3 1 \
+        add expr 4 1 \
+        add expr 5 1 \
         del \
         enable 1000 1024 trim-limit 0 trim-wmark-perc 100 \
         del \
@@ -396,8 +427,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 1
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 1
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 1
@@ -407,8 +440,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 2
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 2
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 2
@@ -418,8 +453,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 3
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 3
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 3
@@ -429,8 +466,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 4
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 4
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 4
@@ -440,8 +479,10 @@ cache-matches   : 0
 trim count      : 0
 ADD expr:
   conj-id-ofs: 5
+  n_conjs: 1
 LOOKUP:
-  conj_id_ofs: 5
+  conj_id_ofs: 0
+  n_conjs: 0
   type: expr
 Enabled: true
 high-watermark  : 5
diff --git a/tests/ovn.at b/tests/ovn.at
index 5d5e00957..6aa619fd9 100644
--- a/tests/ovn.at
+++ b/tests/ovn.at
@@ -14936,6 +14936,18 @@ grep conjunction.*conjunction | wc -l`])
 OVS_WAIT_UNTIL([test 0 = `as hv1 ovs-ofctl dump-flows br-int | \
 grep conjunction.*conjunction.*conjunction | wc -l`])
 
+# Verify that conjunction IDs are consistent between recopmutes
+old_conj_ids=`as hv1 ovs-ofctl dump-flows br-int | grep conj_id= | \
+              awk -F 'conj_id=' '{ print $2 }' | awk -F ',' '{ print $1 }' | sort`
+echo $old_conj_ids
+
+as hv1 ovn-appctl -t ovn-controller recompute
+ovn-nbctl --wait=hv sync
+new_conj_ids=`as hv1 ovs-ofctl dump-flows br-int | grep conj_id= | \
+              awk -F 'conj_id=' '{ print $2 }' | awk -F ',' '{ print $1 }' | sort`
+echo $new_conj_ids
+AT_CHECK([test x"$old_conj_ids" = x"$new_conj_ids"])
+
 OVN_CLEANUP([hv1])
 AT_CLEANUP
 ])
@@ -15076,9 +15088,10 @@ check ovn-nbctl --wait=hv sync
 # Check OVS flows, the less restrictive flows should have been installed.
 AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=44 | ofctl_strip_all | \
     grep "priority=1003" | \
-    sed 's/conjunction([[^)]]*)/conjunction()/g' | sort], [0], [dnl
- table=44, priority=1003,conj_id=2,ip,metadata=0x1 actions=resubmit(,45)
- table=44, priority=1003,conj_id=3,ip,metadata=0x1 actions=resubmit(,45)
+    sed 's/conjunction([[^)]]*)/conjunction()/g' | \
+    sed 's/conj_id=[[0-9]]*,/conj_id=xxx,/g' | sort], [0], [dnl
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.3 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.4 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_src=10.0.0.1 actions=resubmit(,45)
@@ -15120,9 +15133,10 @@ check ovn-nbctl --wait=hv sync
 # Check OVS flows, the second less restrictive allow ACL should have been installed.
 AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=44 | ofctl_strip_all | \
     grep "priority=1003" | \
-    sed 's/conjunction([[^)]]*)/conjunction()/g' | sort], [0], [dnl
- table=44, priority=1003,conj_id=2,ip,metadata=0x1 actions=resubmit(,45)
- table=44, priority=1003,conj_id=3,ip,metadata=0x1 actions=resubmit(,45)
+    sed 's/conjunction([[^)]]*)/conjunction()/g' | \
+    sed 's/conj_id=[[0-9]]*,/conj_id=xxx,/g' | sort], [0], [dnl
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.3 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.4 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_src=10.0.0.1 actions=resubmit(,45)
@@ -15137,9 +15151,10 @@ check ovn-nbctl --wait=hv sync
 # Check OVS flows, the 10.0.0.1 conjunction should have been reinstalled.
 AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=44 | ofctl_strip_all | \
     grep "priority=1003" | \
-    sed 's/conjunction([[^)]]*)/conjunction()/g' | sort], [0], [dnl
- table=44, priority=1003,conj_id=2,ip,metadata=0x1 actions=resubmit(,45)
- table=44, priority=1003,conj_id=3,ip,metadata=0x1 actions=resubmit(,45)
+    sed 's/conjunction([[^)]]*)/conjunction()/g' | \
+    sed 's/conj_id=[[0-9]]*,/conj_id=xxx,/g' | sort], [0], [dnl
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.3 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.4 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_src=10.0.0.1 actions=conjunction(),conjunction()
@@ -15176,9 +15191,10 @@ check ovn-nbctl --wait=hv sync
 # Check OVS flows, the less restrictive flows should have been installed.
 AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=44 | ofctl_strip_all | \
    grep "priority=1003" | \
-   sed 's/conjunction([[^)]]*)/conjunction()/g' | sort], [0], [dnl
- table=44, priority=1003,conj_id=2,ip,metadata=0x1 actions=resubmit(,45)
- table=44, priority=1003,conj_id=3,ip,metadata=0x1 actions=resubmit(,45)
+   sed 's/conjunction([[^)]]*)/conjunction()/g' | \
+   sed 's/conj_id=[[0-9]]*,/conj_id=xxx,/g' | sort], [0], [dnl
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.3 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.4 actions=conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_src=10.0.0.1 actions=resubmit(,45)
@@ -15196,10 +15212,11 @@ check ovn-nbctl --wait=hv sync
 # New non-conjunctive flows should be added to match on 'udp'.
 AT_CHECK([as hv1 ovs-ofctl dump-flows br-int table=44 | ofctl_strip_all | \
    grep "priority=1003" | \
-   sed 's/conjunction([[^)]]*)/conjunction()/g' | sort], [0], [dnl
- table=44, priority=1003,conj_id=2,ip,metadata=0x1 actions=resubmit(,45)
- table=44, priority=1003,conj_id=3,ip,metadata=0x1 actions=resubmit(,45)
- table=44, priority=1003,conj_id=4,ip,metadata=0x1 actions=resubmit(,45)
+   sed 's/conjunction([[^)]]*)/conjunction()/g' | \
+   sed 's/conj_id=[[0-9]]*,/conj_id=xxx,/g' | sort], [0], [dnl
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
+ table=44, priority=1003,conj_id=xxx,ip,metadata=0x1 actions=resubmit(,45)
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.3 actions=conjunction(),conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_dst=10.0.0.4 actions=conjunction(),conjunction(),conjunction()
  table=44, priority=1003,ip,metadata=0x1,nw_src=10.0.0.1 actions=resubmit(,45)
-- 
2.30.2



More information about the dev mailing list