[ovs-dev] [PATCH] ofproto/bond: Fix a race condition in updating post recirculation rules

[Andy Zhou]

When updating post recirc rules, rule management requires calls to
hmap APIs, which requires proper locking to ensure mutual exclsion in
accessing the hmap internal data structure. The locking currently is
missing from the output_normal() xlate path, thus causing
a race condition.

The race condition leads to segfault crash of ovs-vswitchd, with the
following stack trace:

The crash was found by adding and deleting bond interfaces repeatedly
with on-going traffic hitting the bond interfaces.  The same test was
ran over multiple days with this patch to ensure the same crash was
not seen.

The patch added the necessary lock annotation that would have caught
the bug.

Reported-by: Salvator Cambria <salvatore.cambria at citrix.com>
Signed-off-by: Andy Zhou <azhou at nicira.com>
---
 ofproto/bond.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/ofproto/bond.c b/ofproto/bond.c
index c4b3a3a..89979a0 100644
--- a/ofproto/bond.c
+++ b/ofproto/bond.c
@@ -320,6 +320,7 @@ add_pr_rule(struct bond *bond, const struct match *match,
 
 static void
 update_recirc_rules(struct bond *bond)
+    OVS_REQ_WRLOCK(rwlock)
 {
     struct match match;
     struct bond_pr_rule_op *pr_op, *next_op;
@@ -923,8 +924,9 @@ bond_may_recirc(const struct bond *bond, uint32_t *recirc_id,
     }
 }
 
-void
-bond_update_post_recirc_rules(struct bond* bond, const bool force)
+static void
+bond_update_post_recirc_rules__(struct bond* bond, const bool force)
+    OVS_REQ_WRLOCK(rwlock)
 {
    struct bond_entry *e;
    bool update_rules = force;  /* Always update rules if caller forces it. */
@@ -945,6 +947,14 @@ bond_update_post_recirc_rules(struct bond* bond, const bool force)
         update_recirc_rules(bond);
    }
 }
+
+void
+bond_update_post_recirc_rules(struct bond* bond, const bool force)
+{
+    ovs_rwlock_wrlock(&rwlock);
+    bond_update_post_recirc_rules__(bond, force);
+    ovs_rwlock_unlock(&rwlock);
+}
 
 /* Rebalancing. */
 
@@ -1203,7 +1213,7 @@ bond_rebalance(struct bond *bond)
     }
 
     if (use_recirc && rebalanced) {
-        bond_update_post_recirc_rules(bond,true);
+        bond_update_post_recirc_rules__(bond,true);
     }
 
 done:
-- 
1.9.1