[ovs-dev] [PATCH 13/14] Add support for multiple OpenFlow controllers on a single bridge.

Ben Pfaff blp at nicira.com
Fri Apr 9 00:07:11 UTC 2010


With this commit, Open vSwitch permits a bridge to have any number of
OpenFlow controllers.  When multiple controllers are configured, Open
vSwitch connects to all of them simultaneously.  Details of configuration
are in the vswitch schema documentation.

OpenFlow 1.0 does not specify how multiple controllers coordinate in
interacting with a single switch, so more than one controller should be
specified only if the controllers are themselves designed to coordinate
with each other.

An upcoming commit will provide a simple means for coordination between
multiple controllers.

Feature #2495.
---
 ofproto/fail-open.c          |  158 +++++++++++---
 ofproto/fail-open.h          |    8 +-
 ofproto/ofproto.c            |  516 ++++++++++++++++++++++++++----------------
 ofproto/ofproto.h            |    7 +-
 ofproto/status.c             |    9 -
 tests/ovs-vsctl.at           |   81 +++++++
 utilities/ovs-openflowd.8.in |   24 +-
 utilities/ovs-openflowd.c    |  110 ++++++----
 utilities/ovs-vsctl.8.in     |   20 +-
 utilities/ovs-vsctl.c        |  217 +++++++++++++-----
 vswitchd/bridge.c            |  202 +++++++++--------
 vswitchd/ovs-brcompatd.c     |    4 +-
 vswitchd/vswitch.ovsschema   |    4 +-
 vswitchd/vswitch.xml         |  146 +++++++++---
 14 files changed, 1009 insertions(+), 497 deletions(-)

diff --git a/ofproto/fail-open.c b/ofproto/fail-open.c
index eca1417..c3ff897 100644
--- a/ofproto/fail-open.c
+++ b/ofproto/fail-open.c
@@ -68,8 +68,8 @@
 
 struct fail_open {
     struct ofproto *ofproto;
-    struct rconn *controller;
-    int trigger_duration;
+    struct rconn **controllers;
+    size_t n_controllers;
     int last_disconn_secs;
     struct status_category *ss_cat;
     long long int next_bogus_packet_in;
@@ -78,11 +78,54 @@ struct fail_open {
 
 static void fail_open_recover(struct fail_open *);
 
-/* Returns true if 'fo' should be in fail-open mode, otherwise false. */
-static inline bool
-should_fail_open(const struct fail_open *fo)
+/* Returns the number of seconds of disconnection after which fail-open mode
+ * should activate. */
+static int
+trigger_duration(const struct fail_open *fo)
 {
-    return rconn_failure_duration(fo->controller) >= fo->trigger_duration;
+    if (!fo->n_controllers) {
+        /* Shouldn't ever arrive here, but if we do, never fail open. */
+        return INT_MAX;
+    } else {
+        /* Otherwise, every controller must have a chance to send an
+         * inactivity probe and reconnect before we fail open, so take the
+         * maximum probe interval and multiply by 3:
+         *
+         *  - The first interval is the idle time before sending an inactivity
+         *    probe.
+         *
+         *  - The second interval is the time allowed for a response to the
+         *    inactivity probe.
+         *
+         *  - The third interval is the time allowed to reconnect after no
+         *    response is received.
+         */
+        int max_probe_interval;
+        size_t i;
+
+        max_probe_interval = 0;
+        for (i = 0; i < fo->n_controllers; i++) {
+            int probe_interval = rconn_get_probe_interval(fo->controllers[i]);
+            max_probe_interval = MAX(max_probe_interval, probe_interval);
+        }
+        return max_probe_interval * 3;
+    }
+}
+
+/* Returns the number of seconds for which all controllers have been
+ * disconnected.  */
+static int
+failure_duration(const struct fail_open *fo)
+{
+    int min_failure_duration;
+    size_t i;
+
+    min_failure_duration = 0;
+    for (i = 0; i < fo->n_controllers; i++) {
+        int failure_duration = rconn_failure_duration(fo->controllers[i]);
+        min_failure_duration = MAX(min_failure_duration, failure_duration);
+    }
+    return min_failure_duration;
 }
 
 /* Returns true if 'fo' is currently in fail-open mode, otherwise false. */
@@ -92,8 +135,39 @@ fail_open_is_active(const struct fail_open *fo)
     return fo->last_disconn_secs != 0;
 }
 
+/* Returns true if at least one controller is connected (regardless of whether
+ * those controllers are believed to have authenticated and accepted this
+ * switch), false if none of them are connected. */
+static bool
+any_controller_is_connected(const struct fail_open *fo)
+{
+    size_t i;
+
+    for (i = 0; i < fo->n_controllers; i++) {
+        if (rconn_is_connected(fo->controllers[i])) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* Returns true if at least one controller is believed to have authenticated
+ * and accepted this switch, false otherwise. */
+static bool
+any_controller_is_admitted(const struct fail_open *fo)
+{
+    size_t i;
+
+    for (i = 0; i < fo->n_controllers; i++) {
+        if (rconn_is_admitted(fo->controllers[i])) {
+            return true;
+        }
+    }
+    return false;
+}
+
 static void
-send_bogus_packet_in(struct fail_open *fo)
+send_bogus_packet_in(struct fail_open *fo, struct rconn *rconn)
 {
     uint8_t mac[ETH_ADDR_LEN];
     struct ofpbuf *opi;
@@ -107,17 +181,29 @@ send_bogus_packet_in(struct fail_open *fo)
     ofpbuf_uninit(&b);
 
     /* Send. */
-    rconn_send_with_limit(fo->controller, opi, fo->bogus_packet_counter, 1);
+    rconn_send_with_limit(rconn, opi, fo->bogus_packet_counter, 1);
 }
 
-/* Enter fail-open mode if we should be in it.  Handle reconnecting to a
- * controller from fail-open mode. */
+static void
+send_bogus_packet_ins(struct fail_open *fo)
+{
+    size_t i;
+
+    for (i = 0; i < fo->n_controllers; i++) {
+        if (rconn_is_connected(fo->controllers[i])) {
+            send_bogus_packet_in(fo, fo->controllers[i]);
+        }
+    }
+}
+
+/* Enter fail-open mode if we should be in it. */
 void
 fail_open_run(struct fail_open *fo)
 {
+    int disconn_secs = failure_duration(fo);
+
     /* Enter fail-open mode if 'fo' is not in it but should be.  */
-    if (should_fail_open(fo)) {
-        int disconn_secs = rconn_failure_duration(fo->controller);
+    if (disconn_secs >= trigger_duration(fo)) {
         if (!fail_open_is_active(fo)) {
             VLOG_WARN("Could not connect to controller (or switch failed "
                       "controller's post-connection admission control "
@@ -137,10 +223,10 @@ fail_open_run(struct fail_open *fo)
 
     /* Schedule a bogus packet-in if we're connected and in fail-open. */
     if (fail_open_is_active(fo)) {
-        if (rconn_is_connected(fo->controller)) {
+        if (any_controller_is_connected(fo)) {
             bool expired = time_msec() >= fo->next_bogus_packet_in;
             if (expired) {
-                send_bogus_packet_in(fo);
+                send_bogus_packet_ins(fo);
             }
             if (expired || fo->next_bogus_packet_in == LLONG_MAX) {
                 fo->next_bogus_packet_in = time_msec() + 2000;
@@ -157,7 +243,7 @@ fail_open_run(struct fail_open *fo)
 void
 fail_open_maybe_recover(struct fail_open *fo)
 {
-    if (rconn_is_admitted(fo->controller)) {
+    if (any_controller_is_admitted(fo)) {
         fail_open_recover(fo);
     }
 }
@@ -188,8 +274,8 @@ fail_open_wait(struct fail_open *fo)
 void
 fail_open_flushed(struct fail_open *fo)
 {
-    int disconn_secs = rconn_failure_duration(fo->controller);
-    bool open = disconn_secs >= fo->trigger_duration;
+    int disconn_secs = failure_duration(fo);
+    bool open = disconn_secs >= trigger_duration(fo);
     if (open) {
         union ofp_action action;
         flow_t flow;
@@ -210,23 +296,28 @@ static void
 fail_open_status_cb(struct status_reply *sr, void *fo_)
 {
     struct fail_open *fo = fo_;
-    int cur_duration = rconn_failure_duration(fo->controller);
+    int cur_duration = failure_duration(fo);
+    int trigger = trigger_duration(fo);
 
-    status_reply_put(sr, "trigger-duration=%d", fo->trigger_duration);
+    status_reply_put(sr, "trigger-duration=%d", trigger);
     status_reply_put(sr, "current-duration=%d", cur_duration);
     status_reply_put(sr, "triggered=%s",
-                     cur_duration >= fo->trigger_duration ? "true" : "false");
+                     cur_duration >= trigger ? "true" : "false");
 }
 
+/* Creates and returns a new struct fail_open for 'ofproto', registering switch
+ * status with 'switch_status'.
+ *
+ * The caller should register its set of controllers with
+ * fail_open_set_controllers().  (There should be at least one controller,
+ * otherwise there isn't any point in having the struct fail_open around.) */
 struct fail_open *
-fail_open_create(struct ofproto *ofproto,
-                 int trigger_duration, struct switch_status *switch_status,
-                 struct rconn *controller)
+fail_open_create(struct ofproto *ofproto, struct switch_status *switch_status)
 {
     struct fail_open *fo = xmalloc(sizeof *fo);
     fo->ofproto = ofproto;
-    fo->controller = controller;
-    fo->trigger_duration = trigger_duration;
+    fo->controllers = NULL;
+    fo->n_controllers = 0;
     fo->last_disconn_secs = 0;
     fo->ss_cat = switch_status_register(switch_status, "fail-open",
                                         fail_open_status_cb, fo);
@@ -235,18 +326,29 @@ fail_open_create(struct ofproto *ofproto,
     return fo;
 }
 
+/* Registers the 'n' rconns in 'rconns' as connections to the controller for
+ * 'fo'.  The caller must ensure that all of the rconns remain valid until 'fo'
+ * is destroyed or a new set is registered in a subsequent call.
+ *
+ * Takes ownership of the 'rconns' array, but not of the rconns that it points
+ * to (of which the caller retains ownership). */
 void
-fail_open_set_trigger_duration(struct fail_open *fo, int trigger_duration)
+fail_open_set_controllers(struct fail_open *fo,
+                          struct rconn **rconns, size_t n)
 {
-    fo->trigger_duration = trigger_duration;
+    free(fo->controllers);
+    fo->controllers = rconns;
+    fo->n_controllers = n;
 }
 
+/* Destroys 'fo'. */
 void
 fail_open_destroy(struct fail_open *fo)
 {
     if (fo) {
         fail_open_recover(fo);
-        /* We don't own fo->controller. */
+        free(fo->controllers);
+        /* We don't own the rconns behind fo->controllers. */
         switch_status_unregister(fo->ss_cat);
         rconn_packet_counter_destroy(fo->bogus_packet_counter);
         free(fo);
diff --git a/ofproto/fail-open.h b/ofproto/fail-open.h
index 900d587..c4b8716 100644
--- a/ofproto/fail-open.h
+++ b/ofproto/fail-open.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2009 Nicira Networks.
+ * Copyright (c) 2008, 2009, 2010 Nicira Networks.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,10 +32,8 @@ struct switch_status;
  * creates flows with this priority). */
 #define FAIL_OPEN_PRIORITY 70000
 
-struct fail_open *fail_open_create(struct ofproto *, int trigger_duration,
-                                   struct switch_status *,
-                                   struct rconn *controller);
-void fail_open_set_trigger_duration(struct fail_open *, int trigger_duration);
+struct fail_open *fail_open_create(struct ofproto *, struct switch_status *);
+void fail_open_set_controllers(struct fail_open *, struct rconn **, size_t n);
 void fail_open_destroy(struct fail_open *);
 void fail_open_wait(struct fail_open *);
 bool fail_open_is_active(const struct fail_open *);
diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c
index 10d8161..00ebfe2 100644
--- a/ofproto/ofproto.c
+++ b/ofproto/ofproto.c
@@ -165,27 +165,67 @@ static void rule_post_uninstall(struct ofproto *, struct rule *);
 static void send_flow_removed(struct ofproto *p, struct rule *rule,
                               long long int now, uint8_t reason);
 
-struct ofconn {
-    struct list node;
-    struct rconn *rconn;
-    struct pktbuf *pktbuf;
-    int miss_send_len;
-
-    struct rconn_packet_counter *packet_in_counter;
+/* ofproto supports two kinds of OpenFlow connections:
+ *
+ *   - "Controller connections": Connections to ordinary OpenFlow controllers.
+ *     ofproto maintains persistent connections to these controllers and by
+ *     default sends them asynchronous messages such as packet-ins.
+ *
+ *   - "Management connections", e.g. from ovs-ofctl.  When these connections
+ *     drop, it is the other side's responsibility to reconnect them if
+ *     necessary.  ofproto does not send them asynchronous messages by default.
+ */
+enum ofconn_type {
+    OFCONN_CONTROLLER,          /* An OpenFlow controller. */
+    OFCONN_MANAGEMENT           /* A (transient) management connection. */
+};
 
-    /* Number of OpenFlow messages queued as replies to OpenFlow requests, and
-     * the maximum number before we stop reading OpenFlow requests.  */
+/* An OpenFlow connection. */
+struct ofconn {
+    struct ofproto *ofproto;    /* The ofproto that owns this connection. */
+    struct list node;           /* In struct ofproto's "all_conns" list. */
+    struct rconn *rconn;        /* OpenFlow connection. */
+    enum ofconn_type type;      /* Type. */
+
+    /* OFPT_PACKET_IN related data. */
+    struct rconn_packet_counter *packet_in_counter; /* # queued on 'rconn'. */
+    struct pinsched *schedulers[2]; /* Indexed by reason code; see below. */
+    struct pktbuf *pktbuf;         /* OpenFlow packet buffers. */
+    int miss_send_len;             /* Bytes to send of buffered packets. */
+
+    /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
+     * requests, and the maximum number before we stop reading OpenFlow
+     * requests.  */
 #define OFCONN_REPLY_MAX 100
     struct rconn_packet_counter *reply_counter;
+
+    /* type == OFCONN_CONTROLLER only. */
+    struct hmap_node hmap_node;  /* In struct ofproto's "controllers" map. */
+    struct discovery *discovery; /* Controller discovery object, if enabled. */
+    struct status_category *ss;  /* Switch status category. */
 };
 
-static struct ofconn *ofconn_create(struct ofproto *, struct rconn *);
+/* We use OFPR_NO_MATCH and OFPR_ACTION as indexes into struct ofconn's
+ * "schedulers" array.  Their values are 0 and 1, and their meanings and values
+ * coincide with _ODPL_MISS_NR and _ODPL_ACTION_NR, so this is convenient.  In
+ * case anything ever changes, check their values here.  */
+#define N_SCHEDULERS 2
+BUILD_ASSERT_DECL(OFPR_NO_MATCH == 0);
+BUILD_ASSERT_DECL(OFPR_NO_MATCH == _ODPL_MISS_NR);
+BUILD_ASSERT_DECL(OFPR_ACTION == 1);
+BUILD_ASSERT_DECL(OFPR_ACTION == _ODPL_ACTION_NR);
+
+static struct ofconn *ofconn_create(struct ofproto *, struct rconn *,
+                                    enum ofconn_type);
 static void ofconn_destroy(struct ofconn *);
 static void ofconn_run(struct ofconn *, struct ofproto *);
 static void ofconn_wait(struct ofconn *);
 static void queue_tx(struct ofpbuf *msg, const struct ofconn *ofconn,
                      struct rconn_packet_counter *counter);
 
+static void send_packet_in(struct ofproto *, struct ofpbuf *odp_msg);
+static void do_send_packet_in(struct ofpbuf *odp_msg, void *ofconn);
+
 struct ofproto {
     /* Settings. */
     uint64_t datapath_id;       /* Datapath ID. */
@@ -206,11 +246,8 @@ struct ofproto {
 
     /* Configuration. */
     struct switch_status *switch_status;
-    struct status_category *ss_cat;
     struct in_band *in_band;
-    struct discovery *discovery;
     struct fail_open *fail_open;
-    struct pinsched *miss_sched, *action_sched;
     struct netflow *netflow;
     struct ofproto_sflow *sflow;
 
@@ -221,8 +258,8 @@ struct ofproto {
     struct tag_set revalidate_set;
 
     /* OpenFlow connections. */
-    struct list all_conns;
-    struct ofconn *controller;
+    struct hmap controllers;   /* Controller "struct ofconn"s. */
+    struct list all_conns;     /* Contains "struct ofconn"s. */
     struct pvconn **listeners;
     size_t n_listeners;
     struct pvconn **snoops;
@@ -242,8 +279,7 @@ static const struct ofhooks default_ofhooks;
 
 static uint64_t pick_datapath_id(const struct ofproto *);
 static uint64_t pick_fallback_dpid(void);
-static void send_packet_in_miss(struct ofpbuf *, void *ofproto);
-static void send_packet_in_action(struct ofpbuf *, void *ofproto);
+
 static void update_used(struct ofproto *);
 static void update_stats(struct ofproto *, struct rule *,
                          const struct odp_flow_stats *);
@@ -318,9 +354,7 @@ ofproto_create(const char *datapath, const char *datapath_type,
     /* Initialize submodules. */
     p->switch_status = switch_status_create(p);
     p->in_band = NULL;
-    p->discovery = NULL;
     p->fail_open = NULL;
-    p->miss_sched = p->action_sched = NULL;
     p->netflow = NULL;
     p->sflow = NULL;
 
@@ -332,9 +366,7 @@ ofproto_create(const char *datapath, const char *datapath_type,
 
     /* Initialize OpenFlow connections. */
     list_init(&p->all_conns);
-    p->controller = ofconn_create(p, rconn_create(5, 8));
-    p->controller->pktbuf = pktbuf_create();
-    p->controller->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
+    hmap_init(&p->controllers);
     p->listeners = NULL;
     p->n_listeners = 0;
     p->snoops = NULL;
@@ -351,10 +383,6 @@ ofproto_create(const char *datapath, const char *datapath_type,
         p->ml = mac_learning_create();
     }
 
-    /* Register switch status category. */
-    p->ss_cat = switch_status_register(p->switch_status, "remote",
-                                       rconn_status_cb, p->controller->rconn);
-
     /* Pick final datapath ID. */
     p->datapath_id = pick_datapath_id(p);
     VLOG_INFO("using datapath ID %016"PRIx64, p->datapath_id);
@@ -369,103 +397,205 @@ ofproto_set_datapath_id(struct ofproto *p, uint64_t datapath_id)
     uint64_t old_dpid = p->datapath_id;
     p->datapath_id = datapath_id ? datapath_id : pick_datapath_id(p);
     if (p->datapath_id != old_dpid) {
+        struct ofconn *ofconn;
+
         VLOG_INFO("datapath ID changed to %016"PRIx64, p->datapath_id);
-        rconn_reconnect(p->controller->rconn);
+
+        /* Force all active connections to reconnect, since there is no way to
+         * notify a controller that the datapath ID has changed. */
+        LIST_FOR_EACH (ofconn, struct ofconn, node, &p->all_conns) {
+            rconn_reconnect(ofconn->rconn);
+        }
     }
 }
 
-void
-ofproto_set_controller(struct ofproto *p, const struct ofproto_controller *c)
+static bool
+is_discovery_controller(const struct ofproto_controller *c)
 {
-    int rate_limit, burst_limit;
-    bool in_band;
+    return !strcmp(c->target, "discover");
+}
 
-    if (c) {
-        int probe_interval;
-        bool discovery;
+static bool
+is_in_band_controller(const struct ofproto_controller *c)
+{
+    return is_discovery_controller(c) || c->band == OFPROTO_IN_BAND;
+}
 
-        discovery = !strcmp(c->target, "discover");
-        in_band = discovery || c->band == OFPROTO_IN_BAND;
+static void
+add_controller(struct ofproto *ofproto, const struct ofproto_controller *c)
+{
+    struct discovery *discovery;
+    struct ofconn *ofconn;
 
-        rconn_set_max_backoff(p->controller->rconn, c->max_backoff);
+    if (is_discovery_controller(c)) {
+        int error = discovery_create(c->accept_re, c->update_resolv_conf,
+                                     ofproto->dpif, ofproto->switch_status,
+                                     &discovery);
+        if (error) {
+            return;
+        }
+    } else {
+        discovery = NULL;
+    }
 
-        probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
-        rconn_set_probe_interval(p->controller->rconn, probe_interval);
+    ofconn = ofconn_create(ofproto, rconn_create(5, 8), OFCONN_CONTROLLER);
+    ofconn->pktbuf = pktbuf_create();
+    ofconn->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;
+    if (discovery) {
+        ofconn->discovery = discovery;
+    } else {
+        rconn_connect(ofconn->rconn, c->target);
+    }
+    hmap_insert(&ofproto->controllers, &ofconn->hmap_node,
+                hash_string(c->target, 0));
+}
 
-        if (discovery != (p->discovery != NULL)) {
-            rconn_disconnect(p->controller->rconn);
-            if (discovery) {
-                if (discovery_create(c->accept_re, c->update_resolv_conf,
-                                     p->dpif, p->switch_status,
-                                     &p->discovery)) {
-                    return;
-                }
+static void
+update_controller(struct ofconn *ofconn, const struct ofproto_controller *c)
+{
+    struct ofproto *ofproto = ofconn->ofproto;
+    int probe_interval;
+    int i;
+
+    rconn_set_max_backoff(ofconn->rconn, c->max_backoff);
+
+    probe_interval = c->probe_interval ? MAX(c->probe_interval, 5) : 0;
+    rconn_set_probe_interval(ofconn->rconn, probe_interval);
+
+    if (ofconn->discovery) {
+        discovery_set_update_resolv_conf(ofconn->discovery,
+                                         c->update_resolv_conf);
+        discovery_set_accept_controller_re(ofconn->discovery, c->accept_re);
+    }
+
+    for (i = 0; i < N_SCHEDULERS; i++) {
+        struct pinsched **s = &ofconn->schedulers[i];
+
+        if (c->rate_limit > 0) {
+            if (!*s) {
+                *s = pinsched_create(c->rate_limit, c->burst_limit,
+                                     ofproto->switch_status);
             } else {
-                discovery_destroy(p->discovery);
-                p->discovery = NULL;
+                pinsched_set_limits(*s, c->rate_limit, c->burst_limit);
             }
+        } else {
+            pinsched_destroy(*s);
+            *s = NULL;
         }
+    }
+}
 
-        if (discovery) {
-            discovery_set_update_resolv_conf(p->discovery,
-                                             c->update_resolv_conf);
-            discovery_set_accept_controller_re(p->discovery, c->accept_re);
-        } else {
-            if (strcmp(rconn_get_name(p->controller->rconn), c->target)) {
-                rconn_connect(p->controller->rconn, c->target);
-            }
+static const char *
+ofconn_get_target(const struct ofconn *ofconn)
+{
+    return ofconn->discovery ? "discover" : rconn_get_name(ofconn->rconn);
+}
+
+static struct ofconn *
+find_controller_by_target(struct ofproto *ofproto, const char *target)
+{
+    struct ofconn *ofconn;
+
+    HMAP_FOR_EACH_WITH_HASH (ofconn, struct ofconn, hmap_node,
+                             hash_string(target, 0), &ofproto->controllers) {
+        if (!strcmp(ofconn_get_target(ofconn), target)) {
+            return ofconn;
         }
-    } else {
-        rconn_disconnect(p->controller->rconn);
-        in_band = false;
     }
+    return NULL;
+}
 
-    if (in_band != (p->in_band != NULL)) {
-        if (in_band) {
-            int error;
+void
+ofproto_set_controllers(struct ofproto *p,
+                        const struct ofproto_controller *controllers,
+                        size_t n_controllers)
+{
+    struct shash new_controllers;
+    struct rconn **in_band_rconns;
+    enum ofproto_fail_mode fail_mode;
+    struct ofconn *ofconn, *next;
+    bool ss_exists;
+    size_t n_in_band;
+    size_t i;
 
-            error = in_band_create(p, p->dpif, p->switch_status, &p->in_band);
-            if (!error) {
-                in_band_set_remotes(p->in_band, &p->controller->rconn, 1);
-            }
+    shash_init(&new_controllers);
+    for (i = 0; i < n_controllers; i++) {
+        const struct ofproto_controller *c = &controllers[i];
+
+        shash_add_once(&new_controllers, c->target, &controllers[i]);
+        if (!find_controller_by_target(p, c->target)) {
+            add_controller(p, c);
+        }
+    }
+
+    in_band_rconns = xmalloc(n_controllers * sizeof *in_band_rconns);
+    n_in_band = 0;
+    fail_mode = OFPROTO_FAIL_STANDALONE;
+    ss_exists = false;
+    HMAP_FOR_EACH_SAFE (ofconn, next, struct ofconn, hmap_node,
+                        &p->controllers) {
+        struct ofproto_controller *c;
+
+        c = shash_find_data(&new_controllers, ofconn_get_target(ofconn));
+        if (!c) {
+            ofconn_destroy(ofconn);
         } else {
-            in_band_destroy(p->in_band);
-            p->in_band = NULL;
+            update_controller(ofconn, c);
+
+            if (ofconn->ss) {
+                ss_exists = true;
+            }
+            if (is_in_band_controller(c)) {
+                in_band_rconns[n_in_band++] = ofconn->rconn;
+            }
+
+            if (c->fail == OFPROTO_FAIL_SECURE) {
+                fail_mode = OFPROTO_FAIL_SECURE;
+            }
+        }
+    }
+    shash_destroy(&new_controllers);
+
+    if (n_in_band) {
+        if (!p->in_band) {
+            in_band_create(p, p->dpif, p->switch_status, &p->in_band);
+        }
+        if (p->in_band) {
+            in_band_set_remotes(p->in_band, in_band_rconns, n_in_band);
         }
-        rconn_reconnect(p->controller->rconn);
+    } else {
+        in_band_destroy(p->in_band);
+        p->in_band = NULL;
     }
+    free(in_band_rconns);
+
+    if (!hmap_is_empty(&p->controllers)
+        && fail_mode == OFPROTO_FAIL_STANDALONE) {
+        struct rconn **rconns;
+        size_t n;
 
-    if (c && c->fail == OFPROTO_FAIL_STANDALONE) {
-        struct rconn *rconn = p->controller->rconn;
-        int trigger_duration = rconn_get_probe_interval(rconn) * 3;
         if (!p->fail_open) {
-            p->fail_open = fail_open_create(p, trigger_duration,
-                                            p->switch_status, rconn);
-        } else {
-            fail_open_set_trigger_duration(p->fail_open, trigger_duration);
+            p->fail_open = fail_open_create(p, p->switch_status);
+        }
+
+        n = 0;
+        rconns = xmalloc(hmap_count(&p->controllers) * sizeof *rconns);
+        HMAP_FOR_EACH (ofconn, struct ofconn, hmap_node, &p->controllers) {
+            rconns[n++] = ofconn->rconn;
         }
+
+        fail_open_set_controllers(p->fail_open, rconns, n);
+        /* p->fail_open takes ownership of 'rconns'. */
     } else {
         fail_open_destroy(p->fail_open);
         p->fail_open = NULL;
     }
 
-    rate_limit = c ? c->rate_limit : 0;
-    burst_limit = c ? c->burst_limit : 0;
-    if (rate_limit > 0) {
-        if (!p->miss_sched) {
-            p->miss_sched = pinsched_create(rate_limit, burst_limit,
-                                                  p->switch_status);
-            p->action_sched = pinsched_create(rate_limit, burst_limit,
-                                                    NULL);
-        } else {
-            pinsched_set_limits(p->miss_sched, rate_limit, burst_limit);
-            pinsched_set_limits(p->action_sched, rate_limit, burst_limit);
-        }
-    } else {
-        pinsched_destroy(p->miss_sched);
-        p->miss_sched = NULL;
-        pinsched_destroy(p->action_sched);
-        p->action_sched = NULL;
+    if (!hmap_is_empty(&p->controllers) && !ss_exists) {
+        ofconn = CONTAINER_OF(hmap_first(&p->controllers),
+                              struct ofconn, hmap_node);
+        ofconn->ss = switch_status_register(p->switch_status, "remote",
+                                            rconn_status_cb, ofconn->rconn);
     }
 }
 
@@ -628,27 +758,10 @@ ofproto_get_datapath_id(const struct ofproto *ofproto)
     return ofproto->datapath_id;
 }
 
-void
-ofproto_get_controller(const struct ofproto *p, struct ofproto_controller *c)
+bool
+ofproto_has_controller(const struct ofproto *ofproto)
 {
-    memset(c, 0, sizeof *c);
-    if (p->discovery) {
-        struct discovery *d = p->discovery;
-
-        c->target = "discover";
-        c->accept_re = (char *) discovery_get_accept_controller_re(d);
-        c->update_resolv_conf = discovery_get_update_resolv_conf(d);
-    } else if (p->controller) {
-        c->target = (char *) rconn_get_name(p->controller->rconn);
-    } else {
-        return;
-    }
-
-    c->max_backoff = rconn_get_max_backoff(p->controller->rconn);
-    c->probe_interval = rconn_get_probe_interval(p->controller->rconn);
-    c->fail = p->fail_open ? OFPROTO_FAIL_STANDALONE : OFPROTO_FAIL_SECURE;
-    c->band = p->in_band ? OFPROTO_IN_BAND : OFPROTO_OUT_OF_BAND;
-    pinsched_get_limits(p->miss_sched, &c->rate_limit, &c->burst_limit);
+    return !hmap_is_empty(&ofproto->controllers);
 }
 
 void
@@ -697,6 +810,7 @@ ofproto_destroy(struct ofproto *p)
                         &p->all_conns) {
         ofconn_destroy(ofconn);
     }
+    hmap_destroy(&p->controllers);
 
     dpif_close(p->dpif);
     netdev_monitor_destroy(p->netdev_monitor);
@@ -706,14 +820,9 @@ ofproto_destroy(struct ofproto *p)
     shash_destroy(&p->port_by_name);
 
     switch_status_destroy(p->switch_status);
-    discovery_destroy(p->discovery);
-    pinsched_destroy(p->miss_sched);
-    pinsched_destroy(p->action_sched);
     netflow_destroy(p->netflow);
     ofproto_sflow_destroy(p->sflow);
 
-    switch_status_unregister(p->ss_cat);
-
     for (i = 0; i < p->n_listeners; i++) {
         pvconn_close(p->listeners[i]);
     }
@@ -802,21 +911,6 @@ ofproto_run1(struct ofproto *p)
     if (p->in_band) {
         in_band_run(p->in_band);
     }
-    if (p->discovery) {
-        char *controller_name;
-        if (rconn_is_connectivity_questionable(p->controller->rconn)) {
-            discovery_question_connectivity(p->discovery);
-        }
-        if (discovery_run(p->discovery, &controller_name)) {
-            if (controller_name) {
-                rconn_connect(p->controller->rconn, controller_name);
-            } else {
-                rconn_disconnect(p->controller->rconn);
-            }
-        }
-    }
-    pinsched_run(p->miss_sched, send_packet_in_miss, p);
-    pinsched_run(p->action_sched, send_packet_in_action, p);
 
     LIST_FOR_EACH_SAFE (ofconn, next_ofconn, struct ofconn, node,
                         &p->all_conns) {
@@ -835,7 +929,8 @@ ofproto_run1(struct ofproto *p)
 
         retval = pvconn_accept(p->listeners[i], OFP_VERSION, &vconn);
         if (!retval) {
-            ofconn_create(p, rconn_new_from_vconn("passive", vconn));
+            ofconn_create(p, rconn_new_from_vconn("passive", vconn),
+                          OFCONN_MANAGEMENT);
         } else if (retval != EAGAIN) {
             VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
         }
@@ -847,7 +942,19 @@ ofproto_run1(struct ofproto *p)
 
         retval = pvconn_accept(p->snoops[i], OFP_VERSION, &vconn);
         if (!retval) {
-            rconn_add_monitor(p->controller->rconn, vconn);
+            /* Arbitrarily pick the first controller in the list for
+             * monitoring.  We could do something smarter or more flexible
+             * later, if it ever proves useful. */
+            LIST_FOR_EACH (ofconn, struct ofconn, node, &p->all_conns) {
+                if (ofconn->type == OFCONN_CONTROLLER) {
+                    rconn_add_monitor(ofconn->rconn, vconn);
+                    goto success;
+                }
+            }
+            VLOG_INFO_RL(&rl, "no controller connection to monitor");
+            vconn_close(vconn);
+
+        success:;
         } else if (retval != EAGAIN) {
             VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
         }
@@ -920,14 +1027,9 @@ ofproto_wait(struct ofproto *p)
     if (p->in_band) {
         in_band_wait(p->in_band);
     }
-    if (p->discovery) {
-        discovery_wait(p->discovery);
-    }
     if (p->fail_open) {
         fail_open_wait(p->fail_open);
     }
-    pinsched_wait(p->miss_sched);
-    pinsched_wait(p->action_sched);
     if (p->sflow) {
         ofproto_sflow_wait(p->sflow);
     }
@@ -964,7 +1066,7 @@ ofproto_get_revalidate_set(struct ofproto *ofproto)
 bool
 ofproto_is_alive(const struct ofproto *p)
 {
-    return p->discovery || rconn_is_alive(p->controller->rconn);
+    return !hmap_is_empty(&p->controllers);
 }
 
 int
@@ -1341,14 +1443,16 @@ init_ports(struct ofproto *p)
 }
 
 static struct ofconn *
-ofconn_create(struct ofproto *p, struct rconn *rconn)
+ofconn_create(struct ofproto *p, struct rconn *rconn, enum ofconn_type type)
 {
-    struct ofconn *ofconn = xmalloc(sizeof *ofconn);
+    struct ofconn *ofconn = xzalloc(sizeof *ofconn);
+    ofconn->ofproto = p;
     list_push_back(&p->all_conns, &ofconn->node);
     ofconn->rconn = rconn;
+    ofconn->type = type;
+    ofconn->packet_in_counter = rconn_packet_counter_create ();
     ofconn->pktbuf = NULL;
     ofconn->miss_send_len = 0;
-    ofconn->packet_in_counter = rconn_packet_counter_create ();
     ofconn->reply_counter = rconn_packet_counter_create ();
     return ofconn;
 }
@@ -1356,7 +1460,13 @@ ofconn_create(struct ofproto *p, struct rconn *rconn)
 static void
 ofconn_destroy(struct ofconn *ofconn)
 {
+    if (ofconn->type == OFCONN_CONTROLLER) {
+        hmap_remove(&ofconn->ofproto->controllers, &ofconn->hmap_node);
+    }
+    discovery_destroy(ofconn->discovery);
+
     list_remove(&ofconn->node);
+    switch_status_unregister(ofconn->ss);
     rconn_destroy(ofconn->rconn);
     rconn_packet_counter_destroy(ofconn->packet_in_counter);
     rconn_packet_counter_destroy(ofconn->reply_counter);
@@ -1368,6 +1478,25 @@ static void
 ofconn_run(struct ofconn *ofconn, struct ofproto *p)
 {
     int iteration;
+    size_t i;
+
+    if (ofconn->discovery) {
+        char *controller_name;
+        if (rconn_is_connectivity_questionable(ofconn->rconn)) {
+            discovery_question_connectivity(ofconn->discovery);
+        }
+        if (discovery_run(ofconn->discovery, &controller_name)) {
+            if (controller_name) {
+                rconn_connect(ofconn->rconn, controller_name);
+            } else {
+                rconn_disconnect(ofconn->rconn);
+            }
+        }
+    }
+
+    for (i = 0; i < N_SCHEDULERS; i++) {
+        pinsched_run(ofconn->schedulers[i], do_send_packet_in, ofconn);
+    }
 
     rconn_run(ofconn->rconn);
 
@@ -1387,7 +1516,7 @@ ofconn_run(struct ofconn *ofconn, struct ofproto *p)
         }
     }
 
-    if (ofconn != p->controller && !rconn_is_alive(ofconn->rconn)) {
+    if (!ofconn->discovery && !rconn_is_alive(ofconn->rconn)) {
         ofconn_destroy(ofconn);
     }
 }
@@ -1921,7 +2050,7 @@ handle_set_config(struct ofproto *p, struct ofconn *ofconn,
     }
     flags = ntohs(osc->flags);
 
-    if (ofconn == p->controller) {
+    if (ofconn->type == OFCONN_CONTROLLER) {
         switch (flags & OFPC_FRAG_MASK) {
         case OFPC_FRAG_NORMAL:
             dpif_set_drop_frags(p->dpif, false);
@@ -3281,7 +3410,6 @@ static void
 handle_odp_miss_msg(struct ofproto *p, struct ofpbuf *packet)
 {
     struct odp_msg *msg = packet->data;
-    uint16_t in_port = odp_port_to_ofp_port(msg->port);
     struct rule *rule;
     struct ofpbuf payload;
     flow_t flow;
@@ -3317,7 +3445,7 @@ handle_odp_miss_msg(struct ofproto *p, struct ofpbuf *packet)
         }
 
         COVERAGE_INC(ofproto_packet_in);
-        pinsched_send(p->miss_sched, in_port, packet, send_packet_in_miss, p);
+        send_packet_in(p, packet);
         return;
     }
 
@@ -3338,8 +3466,7 @@ handle_odp_miss_msg(struct ofproto *p, struct ofpbuf *packet)
     rule_execute(p, rule, &payload, &flow);
     rule_reinstall(p, rule);
 
-    if (rule->super && rule->super->cr.priority == FAIL_OPEN_PRIORITY
-        && rconn_is_connected(p->controller->rconn)) {
+    if (rule->super && rule->super->cr.priority == FAIL_OPEN_PRIORITY) {
         /*
          * Extra-special case for fail-open mode.
          *
@@ -3350,7 +3477,7 @@ handle_odp_miss_msg(struct ofproto *p, struct ofpbuf *packet)
          *
          * See the top-level comment in fail-open.c for more information.
          */
-        pinsched_send(p->miss_sched, in_port, packet, send_packet_in_miss, p);
+        send_packet_in(p, packet);
     } else {
         ofpbuf_delete(packet);
     }
@@ -3364,8 +3491,7 @@ handle_odp_msg(struct ofproto *p, struct ofpbuf *packet)
     switch (msg->type) {
     case _ODPL_ACTION_NR:
         COVERAGE_INC(ofproto_ctlr_action);
-        pinsched_send(p->action_sched, odp_port_to_ofp_port(msg->port), packet,
-                      send_packet_in_action, p);
+        send_packet_in(p, packet);
         break;
 
     case _ODPL_SFLOW_NR:
@@ -3609,67 +3735,69 @@ update_used(struct ofproto *p)
 }
 
 static void
-do_send_packet_in(struct ofconn *ofconn, uint32_t buffer_id,
-                  const struct ofpbuf *packet, int send_len)
+do_send_packet_in(struct ofpbuf *packet, void *ofconn_)
 {
+    struct ofconn *ofconn = ofconn_;
+    struct ofproto *ofproto = ofconn->ofproto;
     struct odp_msg *msg = packet->data;
     struct ofpbuf payload;
     struct ofpbuf *opi;
-    uint8_t reason;
+    uint32_t buffer_id;
+    int send_len;
 
     /* Extract packet payload from 'msg'. */
     payload.data = msg + 1;
     payload.size = msg->length - sizeof *msg;
 
-    /* Construct ofp_packet_in message. */
-    reason = msg->type == _ODPL_ACTION_NR ? OFPR_ACTION : OFPR_NO_MATCH;
-    opi = make_packet_in(buffer_id, odp_port_to_ofp_port(msg->port), reason,
-                         &payload, send_len);
+    /* Construct packet-in message. */
+    send_len = INT_MAX;
+    if (msg->type == _ODPL_ACTION_NR) {
+        buffer_id = UINT32_MAX;
+    } else {
+        if (ofproto->fail_open && fail_open_is_active(ofproto->fail_open)) {
+            buffer_id = pktbuf_get_null();
+        } else {
+            buffer_id = pktbuf_save(ofconn->pktbuf, &payload, msg->port);
+        }
+        if (buffer_id != UINT32_MAX) {
+            send_len = ofconn->miss_send_len;
+        }
+    }
+    opi = make_packet_in(buffer_id, odp_port_to_ofp_port(msg->port),
+                         msg->type, &payload, send_len);
 
     /* Send. */
     rconn_send_with_limit(ofconn->rconn, opi, ofconn->packet_in_counter, 100);
-}
 
-static void
-send_packet_in_action(struct ofpbuf *packet, void *p_)
-{
-    struct ofproto *p = p_;
-    struct ofconn *ofconn;
-    struct odp_msg *msg;
-
-    msg = packet->data;
-    LIST_FOR_EACH (ofconn, struct ofconn, node, &p->all_conns) {
-        if (ofconn == p->controller || ofconn->miss_send_len) {
-            do_send_packet_in(ofconn, UINT32_MAX, packet, msg->arg);
-        }
-    }
     ofpbuf_delete(packet);
 }
 
 static void
-send_packet_in_miss(struct ofpbuf *packet, void *p_)
+send_packet_in(struct ofproto *ofproto, struct ofpbuf *packet)
 {
-    struct ofproto *p = p_;
-    bool in_fail_open = p->fail_open && fail_open_is_active(p->fail_open);
-    struct ofconn *ofconn;
-    struct ofpbuf payload;
-    struct odp_msg *msg;
+    struct odp_msg *msg = packet->data;
+    struct ofconn *ofconn, *prev;
 
-    msg = packet->data;
-    payload.data = msg + 1;
-    payload.size = msg->length - sizeof *msg;
-    LIST_FOR_EACH (ofconn, struct ofconn, node, &p->all_conns) {
-        if (ofconn->miss_send_len) {
-            struct pktbuf *pb = ofconn->pktbuf;
-            uint32_t buffer_id = (in_fail_open
-                                  ? pktbuf_get_null()
-                                  : pktbuf_save(pb, &payload, msg->port));
-            int send_len = (buffer_id != UINT32_MAX ? ofconn->miss_send_len
-                            : UINT32_MAX);
-            do_send_packet_in(ofconn, buffer_id, packet, send_len);
+    assert(msg->type == _ODPL_MISS_NR || msg->type == _ODPL_ACTION_NR);
+
+    prev = NULL;
+    LIST_FOR_EACH (ofconn, struct ofconn, node, &ofproto->all_conns) {
+        if (ofconn->miss_send_len
+            || (msg->type == _ODPL_ACTION_NR
+                && ofconn->type == OFCONN_CONTROLLER)) {
+            if (prev) {
+                pinsched_send(prev->schedulers[msg->type], msg->port,
+                              ofpbuf_clone(packet), do_send_packet_in, prev);
+            }
+            prev = ofconn;
         }
     }
-    ofpbuf_delete(packet);
+    if (prev) {
+        pinsched_send(prev->schedulers[msg->type], msg->port,
+                      packet, do_send_packet_in, prev);
+    } else {
+        ofpbuf_delete(packet);
+    }
 }
 
 static uint64_t
diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h
index 29f9ee9..22ad610 100644
--- a/ofproto/ofproto.h
+++ b/ofproto/ofproto.h
@@ -100,8 +100,8 @@ bool ofproto_is_alive(const struct ofproto *);
 
 /* Configuration. */
 void ofproto_set_datapath_id(struct ofproto *, uint64_t datapath_id);
-void ofproto_set_controller(struct ofproto *,
-                            const struct ofproto_controller *);
+void ofproto_set_controllers(struct ofproto *,
+                             const struct ofproto_controller *, size_t n);
 void ofproto_set_desc(struct ofproto *,
                       const char *mfr_desc, const char *hw_desc,
                       const char *sw_desc, const char *serial_desc,
@@ -115,8 +115,7 @@ int ofproto_set_stp(struct ofproto *, bool enable_stp);
 
 /* Configuration querying. */
 uint64_t ofproto_get_datapath_id(const struct ofproto *);
-void ofproto_get_controller(const struct ofproto *,
-                            struct ofproto_controller *);
+bool ofproto_has_controller(const struct ofproto *);
 void ofproto_get_listeners(const struct ofproto *, struct svec *);
 void ofproto_get_snoops(const struct ofproto *, struct svec *);
 void ofproto_get_all_flows(struct ofproto *p, struct ds *);
diff --git a/ofproto/status.c b/ofproto/status.c
index dbedf10..133bde0 100644
--- a/ofproto/status.c
+++ b/ofproto/status.c
@@ -130,7 +130,6 @@ static void
 config_status_cb(struct status_reply *sr, void *ofproto_)
 {
     const struct ofproto *ofproto = ofproto_;
-    struct ofproto_controller controller;
     uint64_t datapath_id;
     struct svec listeners;
     size_t i;
@@ -146,14 +145,6 @@ config_status_cb(struct status_reply *sr, void *ofproto_)
         status_reply_put(sr, "management%zu=%s", i, listeners.names[i]);
     }
     svec_destroy(&listeners);
-
-    ofproto_get_controller(ofproto, &controller);
-    if (controller.probe_interval) {
-        status_reply_put(sr, "probe-interval=%d", controller.probe_interval);
-    }
-    if (controller.max_backoff) {
-        status_reply_put(sr, "max-backoff=%d", controller.max_backoff);
-    }
 }
 
 static void
diff --git a/tests/ovs-vsctl.at b/tests/ovs-vsctl.at
index c634703..e9b2106 100644
--- a/tests/ovs-vsctl.at
+++ b/tests/ovs-vsctl.at
@@ -354,6 +354,87 @@ CHECK_IFACES([a], [a1], [a2], [a3])
 OVS_VSCTL_CLEANUP
 AT_CLEANUP
 
+AT_SETUP([controllers])
+AT_KEYWORDS([controller ovs-vsctl])
+OVS_VSCTL_SETUP
+AT_CHECK([RUN_OVS_VSCTL_TOGETHER(
+  [add-br br0], 
+
+  [set-controller tcp:1.2.3.4],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [set-controller br0 tcp:4.5.6.7],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [del-controller],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [set-controller default tcp:8.9.10.11],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [del-controller default],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [del-controller br0],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [set-controller default tcp:1.2.3.4 tcp:4.5.6.7],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0],
+
+  [set-controller br0 tcp:8.9.10.11 tcp:5.4.3.2],
+  [get-controller],
+  [get-controller default],
+  [get-controller br0])], [0], [
+
+tcp:1.2.3.4
+tcp:1.2.3.4
+tcp:1.2.3.4
+
+tcp:1.2.3.4
+tcp:1.2.3.4
+tcp:4.5.6.7
+
+
+
+tcp:4.5.6.7
+
+tcp:8.9.10.11
+tcp:8.9.10.11
+tcp:4.5.6.7
+
+
+
+tcp:4.5.6.7
+
+
+
+
+
+tcp:1.2.3.4\ntcp:4.5.6.7
+tcp:1.2.3.4\ntcp:4.5.6.7
+tcp:1.2.3.4\ntcp:4.5.6.7
+
+tcp:1.2.3.4\ntcp:4.5.6.7
+tcp:1.2.3.4\ntcp:4.5.6.7
+tcp:5.4.3.2\ntcp:8.9.10.11
+], [], [OVS_VSCTL_CLEANUP])
+OVS_VSCTL_CLEANUP
+AT_CLEANUP
+
 dnl ----------------------------------------------------------------------
 AT_BANNER([ovs-vsctl unit tests -- fake bridges])
 
diff --git a/utilities/ovs-openflowd.8.in b/utilities/ovs-openflowd.8.in
index 4d2f211..7a78bf2 100644
--- a/utilities/ovs-openflowd.8.in
+++ b/utilities/ovs-openflowd.8.in
@@ -6,12 +6,12 @@ ovs\-openflowd \- OpenFlow switch implementation
 .
 .SH SYNOPSIS
 .B ovs\-openflowd
-[\fIoptions\fR] \fIdatapath\fR [\fIcontroller\fR]
+[\fIoptions\fR] \fIdatapath\fR [\fIcontroller\fR\&...]
 .
 .SH DESCRIPTION
 The \fBovs\-openflowd\fR program implements an OpenFlow switch using a
-flow-based datapath.  \fBovs\-openflowd\fR connects to an OpenFlow controller
-over TCP or SSL.
+flow-based datapath.  \fBovs\-openflowd\fR connects to one or more
+OpenFlow controllers over TCP or SSL.
 .PP
 The mandatory \fIdatapath\fR argument argument specifies the local datapath
 to relay.  It takes one of the following forms:
@@ -19,17 +19,17 @@ to relay.  It takes one of the following forms:
 .so lib/dpif.man
 .
 .PP
-The optional \fIcontroller\fR argument specifies how to connect to
+The optional \fIcontroller\fR arguments specify how to connect to
 the OpenFlow controller.  It takes one of the following forms:
 .
 .so lib/vconn-active.man
 .
 .PP
-If \fIcontroller\fR is omitted, \fBovs\-openflowd\fR attempts to discover the
-location of the controller automatically (see below).
+If no \fIcontroller\fR is specified, \fBovs\-openflowd\fR attempts to
+discover the location of a controller automatically (see below).
 .
-.SS "Contacting the Controller"
-The OpenFlow switch must be able to contact the OpenFlow controller
+.SS "Contacting Controllers"
+The OpenFlow switch must be able to contact the OpenFlow controllers
 over the network.  It can do so in one of two ways:
 .
 .IP out-of-band
@@ -60,8 +60,8 @@ manually or discovered automatically:
 .RS
 .IP "controller discovery"
 To make \fBovs\-openflowd\fR discover the location of the controller
-automatically, do not specify the location of the controller on the
-\fBovs\-openflowd\fR command line.
+automatically, do not specify a controller on the \fBovs\-openflowd\fR
+command line.
 .IP
 In this mode, \fBovs\-openflowd\fR will broadcast a DHCP request with vendor
 class identifier \fBOpenFlow\fR across the network devices added to
@@ -159,8 +159,8 @@ the local port network device, and start the DHCP client afterward.
 .TP
 \fB--datapath-id=\fIdpid\fR
 Sets \fIdpid\fR, which must consist of exactly 16 hexadecimal digits,
-as the datapath ID that the switch will use to identify itself to the
-OpenFlow controller.
+as the datapath ID that the switch will use to identify itself to
+OpenFlow controllers.
 .IP
 If this option is omitted, the default datapath ID is taken from the
 Ethernet address of the datapath's local port (which is typically
diff --git a/utilities/ovs-openflowd.c b/utilities/ovs-openflowd.c
index 80fbb24..7633f5a 100644
--- a/utilities/ovs-openflowd.c
+++ b/utilities/ovs-openflowd.c
@@ -51,7 +51,8 @@
 /* Settings that may be configured by the user. */
 struct ofsettings {
     /* Controller configuration. */
-    struct ofproto_controller controller;
+    struct ofproto_controller *controllers;
+    size_t n_controllers;
 
     /* Datapath. */
     uint64_t datapath_id;       /* Datapath ID. */
@@ -171,7 +172,7 @@ main(int argc, char *argv[])
     if (error) {
         ovs_fatal(error, "failed to configure STP");
     }
-    ofproto_set_controller(ofproto, &s.controller);
+    ofproto_set_controllers(ofproto, s.controllers, s.n_controllers);
 
     daemonize_complete();
 
@@ -266,16 +267,18 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
         {0, 0, 0, 0},
     };
     char *short_options = long_options_to_short_options(long_options);
+    struct ofproto_controller controller_opts;
 
     /* Set defaults that we can figure out before parsing options. */
-    s->controller.max_backoff = 8;
-    s->controller.probe_interval = 0;
-    s->controller.fail = OFPROTO_FAIL_STANDALONE;
-    s->controller.band = OFPROTO_IN_BAND;
-    s->controller.accept_re = NULL;
-    s->controller.update_resolv_conf = true;
-    s->controller.rate_limit = 0;
-    s->controller.burst_limit = 0;
+    controller_opts.target = NULL;
+    controller_opts.max_backoff = 8;
+    controller_opts.probe_interval = 0;
+    controller_opts.fail = OFPROTO_FAIL_STANDALONE;
+    controller_opts.band = OFPROTO_IN_BAND;
+    controller_opts.accept_re = NULL;
+    controller_opts.update_resolv_conf = true;
+    controller_opts.rate_limit = 0;
+    controller_opts.burst_limit = 0;
     s->datapath_id = 0;
     s->mfr_desc = NULL;
     s->hw_desc = NULL;
@@ -325,26 +328,26 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
             break;
 
         case OPT_ACCEPT_VCONN:
-            s->controller.accept_re = optarg;
+            controller_opts.accept_re = optarg;
             break;
 
         case OPT_NO_RESOLV_CONF:
-            s->controller.update_resolv_conf = false;
+            controller_opts.update_resolv_conf = false;
             break;
 
         case OPT_FAIL_MODE:
             if (!strcmp(optarg, "open")) {
-                s->controller.fail = OFPROTO_FAIL_STANDALONE;
+                controller_opts.fail = OFPROTO_FAIL_STANDALONE;
             } else if (!strcmp(optarg, "closed")) {
-                s->controller.fail = OFPROTO_FAIL_SECURE;
+                controller_opts.fail = OFPROTO_FAIL_SECURE;
             } else {
                 ovs_fatal(0, "--fail argument must be \"open\" or \"closed\"");
             }
             break;
 
         case OPT_INACTIVITY_PROBE:
-            s->controller.probe_interval = atoi(optarg);
-            if (s->controller.probe_interval < 5) {
+            controller_opts.probe_interval = atoi(optarg);
+            if (controller_opts.probe_interval < 5) {
                 ovs_fatal(0, "--inactivity-probe argument must be at least 5");
             }
             break;
@@ -362,28 +365,28 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
             break;
 
         case OPT_MAX_BACKOFF:
-            s->controller.max_backoff = atoi(optarg);
-            if (s->controller.max_backoff < 1) {
+            controller_opts.max_backoff = atoi(optarg);
+            if (controller_opts.max_backoff < 1) {
                 ovs_fatal(0, "--max-backoff argument must be at least 1");
-            } else if (s->controller.max_backoff > 3600) {
-                s->controller.max_backoff = 3600;
+            } else if (controller_opts.max_backoff > 3600) {
+                controller_opts.max_backoff = 3600;
             }
             break;
 
         case OPT_RATE_LIMIT:
             if (optarg) {
-                s->controller.rate_limit = atoi(optarg);
-                if (s->controller.rate_limit < 1) {
+                controller_opts.rate_limit = atoi(optarg);
+                if (controller_opts.rate_limit < 1) {
                     ovs_fatal(0, "--rate-limit argument must be at least 1");
                 }
             } else {
-                s->controller.rate_limit = 1000;
+                controller_opts.rate_limit = 1000;
             }
             break;
 
         case OPT_BURST_LIMIT:
-            s->controller.burst_limit = atoi(optarg);
-            if (s->controller.burst_limit < 1) {
+            controller_opts.burst_limit = atoi(optarg);
+            if (controller_opts.burst_limit < 1) {
                 ovs_fatal(0, "--burst-limit argument must be at least 1");
             }
             break;
@@ -397,11 +400,11 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
             break;
 
         case OPT_OUT_OF_BAND:
-            s->controller.band = OFPROTO_OUT_OF_BAND;
+            controller_opts.band = OFPROTO_OUT_OF_BAND;
             break;
 
         case OPT_IN_BAND:
-            s->controller.band = OFPROTO_IN_BAND;
+            controller_opts.band = OFPROTO_IN_BAND;
             break;
 
         case OPT_NETFLOW:
@@ -457,25 +460,46 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
                   "use --help for usage");
     }
 
-    /* Local and remote vconns. */
-    dp_parse_name(argv[0], &s->dp_name, &s->dp_type);
-
-    s->controller.target = argc > 1 ? argv[1] : "discover";
-    if (!strcmp(s->controller.target, "discover")
-        && s->controller.band == OFPROTO_OUT_OF_BAND) {
-        ovs_fatal(0, "Cannot perform discovery with out-of-band control");
-    }
-
     /* Set accept_controller_regex. */
-    if (!s->controller.accept_re) {
-        s->controller.accept_re
+    if (!controller_opts.accept_re) {
+        controller_opts.accept_re
             = stream_ssl_is_configured() ? "^ssl:.*" : "^tcp:.*";
     }
 
     /* Rate limiting. */
-    if (s->controller.rate_limit && s->controller.rate_limit < 100) {
+    if (controller_opts.rate_limit && controller_opts.rate_limit < 100) {
         VLOG_WARN("Rate limit set to unusually low value %d",
-                  s->controller.rate_limit);
+                  controller_opts.rate_limit);
+    }
+
+    /* Local vconns. */
+    dp_parse_name(argv[0], &s->dp_name, &s->dp_type);
+
+    /* Controllers. */
+    s->n_controllers = argc > 1 ? argc - 1 : 1;
+    s->controllers = xmalloc(s->n_controllers * sizeof *s->controllers);
+    if (argc > 1) {
+        size_t i;
+
+        for (i = 0; i < s->n_controllers; i++) {
+            s->controllers[i] = controller_opts;
+            s->controllers[i].target = argv[i + 1];
+        }
+    } else {
+        s->controllers[0] = controller_opts;
+        s->controllers[0].target = "discover";
+    }
+
+    /* Sanity check. */
+    if (controller_opts.band == OFPROTO_OUT_OF_BAND) {
+        size_t i;
+
+        for (i = 0; i < s->n_controllers; i++) {
+            if (!strcmp(s->controllers[i].target, "discover")) {
+                ovs_fatal(0, "Cannot perform discovery with out-of-band "
+                          "control");
+            }
+        }
     }
 }
 
@@ -483,10 +507,10 @@ static void
 usage(void)
 {
     printf("%s: an OpenFlow switch implementation.\n"
-           "usage: %s [OPTIONS] DATAPATH [CONTROLLER]\n"
+           "usage: %s [OPTIONS] DATAPATH [CONTROLLER...]\n"
            "DATAPATH is a local datapath (e.g. \"dp0\").\n"
-           "CONTROLLER is an active OpenFlow connection method; if it is\n"
-           "omitted, then ovs-openflowd performs controller discovery.\n",
+           "Each CONTROLLER is an active OpenFlow connection method.  If\n"
+           "none is given, ovs-openflowd performs controller discovery.\n",
            program_name, program_name);
     vconn_usage(true, true, true);
     printf("\nOpenFlow options:\n"
diff --git a/utilities/ovs-vsctl.8.in b/utilities/ovs-vsctl.8.in
index 031615c..745c66e 100644
--- a/utilities/ovs-vsctl.8.in
+++ b/utilities/ovs-vsctl.8.in
@@ -298,13 +298,14 @@ output.
 .SS "OpenFlow Controller Connectivity"
 .
 \fBovs\-vswitchd\fR can perform all configured bridging and switching
-locally, or it can be configured to connect a given bridge to an
-external OpenFlow controller, such as NOX.  
+locally, or it can be configured to connect a given bridge to one or
+more external OpenFlow controllers, such as NOX.
 .
-If a \fIbridge\fR argument is given, the settings apply only to the
-specified bridge.  Otherwise, they apply to the Open vSwitch instance,
-and its configuration applies to any bridge that has not been explicitly
-configured through a \fIbridge\fR argument.
+For each of these commands, a \fIbridge\fR of \fBdefault\fR applies
+the configuration as the default for any bridge that has not been
+explicitly configured.  Otherwise, \fIbridge\fR must name a bridge,
+and the settings apply only to that bridge.  (Omitting \fIbridge\fR
+entirely usually has the same effect as specifying \fBdefault\fR.)
 .
 .IP "\fBget\-controller\fR [\fIbridge\fR]"
 Prints the configured controller target.
@@ -312,9 +313,10 @@ Prints the configured controller target.
 .IP "\fBdel\-controller\fR [\fIbridge\fR]"
 Deletes the configured controller target.
 .
-.IP "\fBset\-controller\fR [\fIbridge\fR] \fItarget\fR"
-Sets the configured controller target.  The \fItarget\fR may use any of
-the following forms:
+.IP "\fBset\-controller\fR [\fIbridge\fR] \fItarget\fR\&..."
+Sets the configured controller target or targets.  If more than one
+\fItarget\fR is specified, then \fIbridge\fR may not be omitted.  Each
+\fItarget\fR may use any of the following forms:
 .
 .RS
 .so lib/vconn-active.man
diff --git a/utilities/ovs-vsctl.c b/utilities/ovs-vsctl.c
index d39d610..b2b8d07 100644
--- a/utilities/ovs-vsctl.c
+++ b/utilities/ovs-vsctl.c
@@ -475,7 +475,8 @@ struct vsctl_context {
 struct vsctl_bridge {
     struct ovsrec_bridge *br_cfg;
     char *name;
-    struct ovsrec_controller *ctrl;
+    struct ovsrec_controller **ctrl;
+    size_t n_ctrl;
     struct vsctl_bridge *parent;
     int vlan;
 };
@@ -494,7 +495,8 @@ struct vsctl_info {
     struct shash bridges;
     struct shash ports;
     struct shash ifaces;
-    struct ovsrec_controller *ctrl;
+    struct ovsrec_controller **ctrl;
+    size_t n_ctrl;
 };
 
 static char *
@@ -531,7 +533,13 @@ add_bridge(struct vsctl_info *b,
     br->name = xstrdup(name);
     br->parent = parent;
     br->vlan = vlan;
-    br->ctrl = parent ? parent->br_cfg->controller : br_cfg->controller;
+    if (parent) {
+        br->ctrl = parent->br_cfg->controller;
+        br->n_ctrl = parent->br_cfg->n_controller;
+    } else {
+        br->ctrl = br_cfg->controller;
+        br->n_ctrl = br_cfg->n_controller;
+    }
     shash_add(&b->bridges, br->name, br);
     return br;
 }
@@ -596,6 +604,7 @@ get_info(const struct ovsrec_open_vswitch *ovs, struct vsctl_info *info)
     shash_init(&info->ifaces);
 
     info->ctrl = ovs->controller;
+    info->n_ctrl = ovs->n_controller;
 
     shash_init(&bridges);
     shash_init(&ports);
@@ -1426,6 +1435,29 @@ cmd_iface_to_br(struct vsctl_context *ctx)
     free_info(&info);
 }
 
+/* Print targets of the 'n_controllers' in 'controllers' on the output for
+ * 'ctx'. */
+static void
+print_controllers(struct vsctl_context *ctx,
+                  struct ovsrec_controller **controllers,
+                  size_t n_controllers)
+{
+    /* Print the targets in sorted order for reproducibility. */
+    struct svec targets;
+    size_t i;
+
+    svec_init(&targets);
+    for (i = 0; i < n_controllers; i++) {
+        svec_add(&targets, controllers[i]->target);
+    }
+
+    svec_sort(&targets);
+    for (i = 0; i < targets.n; i++) {
+        ds_put_format(&ctx->output, "%s\n", targets.names[i]);
+    }
+    svec_destroy(&targets);
+}
+
 static void
 cmd_get_controller(struct vsctl_context *ctx)
 {
@@ -1433,21 +1465,14 @@ cmd_get_controller(struct vsctl_context *ctx)
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
-        /* Return the controller from the "Open_vSwitch" table */
-        if (info.ctrl) {
-            ds_put_format(&ctx->output, "%s\n", info.ctrl->target);
-        }
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
+        print_controllers(ctx, info.ctrl, info.n_ctrl);
     } else {
-        /* Return the controller for a particular bridge. */
         struct vsctl_bridge *br = find_bridge(&info, ctx->argv[1], true);
-
-        /* If no controller is explicitly defined for the requested
-         * bridge, fallback to the "Open_vSwitch" table's controller. */
-        if (br->ctrl) {
-            ds_put_format(&ctx->output, "%s\n", br->ctrl->target);
-        } else if (info.ctrl) {
-            ds_put_format(&ctx->output, "%s\n", info.ctrl->target);
+        if (br->n_ctrl) {
+            print_controllers(ctx, br->ctrl, br->n_ctrl);
+        } else {
+            print_controllers(ctx, info.ctrl, info.n_ctrl);
         }
     }
 
@@ -1455,60 +1480,117 @@ cmd_get_controller(struct vsctl_context *ctx)
 }
 
 static void
+delete_controllers(struct ovsrec_controller **controllers,
+                   size_t n_controllers)
+{
+    size_t i;
+
+    for (i = 0; i < n_controllers; i++) {
+        ovsrec_controller_delete(controllers[i]);
+    }
+}
+
+static void
 cmd_del_controller(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
-        if (info.ctrl) {
-            ovsrec_controller_delete(info.ctrl);
-            ovsrec_open_vswitch_set_controller(ctx->ovs, NULL);
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
+        if (info.n_ctrl) {
+            delete_controllers(info.ctrl, info.n_ctrl);
+            ovsrec_open_vswitch_set_controller(ctx->ovs, NULL, 0);
         }
     } else {
         struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
-
         if (br->ctrl) {
-            ovsrec_controller_delete(br->ctrl);
-            ovsrec_bridge_set_controller(br->br_cfg, NULL);
+            delete_controllers(br->ctrl, br->n_ctrl);
+            ovsrec_bridge_set_controller(br->br_cfg, NULL, 0);
         }
     }
 
     free_info(&info);
 }
 
+static struct ovsrec_controller **
+insert_controllers(struct ovsdb_idl_txn *txn, char *targets[], size_t n)
+{
+    struct ovsrec_controller **controllers;
+    size_t i;
+
+    controllers = xmalloc(n * sizeof *controllers);
+    for (i = 0; i < n; i++) {
+        controllers[i] = ovsrec_controller_insert(txn);
+        ovsrec_controller_set_target(controllers[i], targets[i]);
+    }
+
+    return controllers;
+}
+
+static void
+set_default_controllers(struct vsctl_context *ctx, char *targets[], size_t n)
+{
+    struct ovsrec_controller **controllers;
+
+    delete_controllers(ctx->ovs->controller, ctx->ovs->n_controller);
+
+    controllers = insert_controllers(ctx->txn, targets, n);
+    ovsrec_open_vswitch_set_controller(ctx->ovs, controllers, n);
+    free(controllers);
+}
+
 static void
 cmd_set_controller(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
-    struct ovsrec_controller *ctrl;
 
     get_info(ctx->ovs, &info);
 
     if (ctx->argc == 2) {
-        /* Set the controller in the "Open_vSwitch" table. */
-        if (info.ctrl) {
-            ovsrec_controller_delete(info.ctrl);
-        }
-        ctrl = ovsrec_controller_insert(ctx->txn);
-        ovsrec_controller_set_target(ctrl, ctx->argv[1]);
-        ovsrec_open_vswitch_set_controller(ctx->ovs, ctrl);
+        /* Set one controller in the "Open_vSwitch" table. */
+        set_default_controllers(ctx, &ctx->argv[1], 1);
+    } else if (!strcmp(ctx->argv[1], "default")) {
+        /* Set one or more controllers in the "Open_vSwitch" table. */
+        set_default_controllers(ctx, &ctx->argv[2], ctx->argc - 2);
     } else {
-        /* Set the controller for a particular bridge. */
+        /* Set one or more controllers for a particular bridge. */
         struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
+        struct ovsrec_controller **controllers;
+        size_t n;
 
-        if (br->ctrl) {
-            ovsrec_controller_delete(br->ctrl);
-        }
-        ctrl = ovsrec_controller_insert(ctx->txn);
-        ovsrec_controller_set_target(ctrl, ctx->argv[2]);
-        ovsrec_bridge_set_controller(br->br_cfg, ctrl);
+        delete_controllers(br->ctrl, br->n_ctrl);
+
+        n = ctx->argc - 2;
+        controllers = insert_controllers(ctx->txn, &ctx->argv[2], n);
+        ovsrec_bridge_set_controller(br->br_cfg, controllers, n);
+        free(controllers);
     }
 
     free_info(&info);
 }
 
+static const char *
+get_fail_mode(struct ovsrec_controller **controllers, size_t n_controllers)
+{
+    const char *fail_mode;
+    size_t i;
+
+    fail_mode = NULL;
+    for (i = 0; i < n_controllers; i++) {
+        const char *s = controllers[i]->fail_mode;
+        if (s) {
+            if (!strcmp(s, "secure")) {
+                return s;
+            } else {
+                fail_mode = s;
+            }
+        }
+    }
+
+    return fail_mode;
+}
+
 static void
 cmd_get_fail_mode(struct vsctl_context *ctx)
 {
@@ -1517,23 +1599,18 @@ cmd_get_fail_mode(struct vsctl_context *ctx)
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
         /* Return the fail-mode from the "Open_vSwitch" table */
-        if (info.ctrl && info.ctrl->fail_mode) {
-            fail_mode = info.ctrl->fail_mode;
-        }
+        fail_mode = get_fail_mode(info.ctrl, info.n_ctrl);
     } else {
         /* Return the fail-mode for a particular bridge. */
         struct vsctl_bridge *br = find_bridge(&info, ctx->argv[1], true);
 
-        /* If no controller or fail-mode is explicitly defined for the 
-         * requested bridge, fallback to the "Open_vSwitch" table's 
-         * setting. */
-        if (br->ctrl && br->ctrl->fail_mode) {
-            fail_mode = br->ctrl->fail_mode;
-        } else if (info.ctrl && info.ctrl->fail_mode) {
-            fail_mode = info.ctrl->fail_mode;
-        }
+        /* If no controller is defined for the requested bridge, fallback to
+         * the "Open_vSwitch" table's controller. */
+        fail_mode = (br->n_ctrl
+                     ? get_fail_mode(br->ctrl, br->n_ctrl)
+                     : get_fail_mode(info.ctrl, info.n_ctrl));
     }
 
     if (fail_mode && strlen(fail_mode)) {
@@ -1544,22 +1621,29 @@ cmd_get_fail_mode(struct vsctl_context *ctx)
 }
 
 static void
+set_fail_mode(struct ovsrec_controller **controllers, size_t n_controllers,
+              const char *fail_mode)
+{
+    size_t i;
+
+    for (i = 0; i < n_controllers; i++) {
+        ovsrec_controller_set_fail_mode(controllers[i], fail_mode);
+    }
+}
+
+static void
 cmd_del_fail_mode(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
-        if (info.ctrl && info.ctrl->fail_mode) {
-            ovsrec_controller_set_fail_mode(info.ctrl, NULL);
-        }
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
+        set_fail_mode(info.ctrl, info.n_ctrl, NULL);
     } else {
         struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
 
-        if (br->ctrl && br->ctrl->fail_mode) {
-            ovsrec_controller_set_fail_mode(br->ctrl, NULL);
-        }
+        set_fail_mode(br->ctrl, br->n_ctrl, NULL);
     }
 
     free_info(&info);
@@ -1569,29 +1653,36 @@ static void
 cmd_set_fail_mode(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
+    const char *bridge;
     const char *fail_mode;
 
     get_info(ctx->ovs, &info);
 
-    fail_mode = (ctx->argc == 2) ? ctx->argv[1] : ctx->argv[2];
+    if (ctx->argc == 2) {
+        bridge = "default";
+        fail_mode = ctx->argv[1];
+    } else {
+        bridge = ctx->argv[1];
+        fail_mode = ctx->argv[2];
+    }
 
     if (strcmp(fail_mode, "standalone") && strcmp(fail_mode, "secure")) {
         vsctl_fatal("fail-mode must be \"standalone\" or \"secure\"");
     }
 
-    if (ctx->argc == 2) {
+    if (!strcmp(bridge, "default")) {
         /* Set the fail-mode in the "Open_vSwitch" table. */
         if (!info.ctrl) {
             vsctl_fatal("no controller declared");
         }
-        ovsrec_controller_set_fail_mode(info.ctrl, fail_mode);
+        set_fail_mode(info.ctrl, info.n_ctrl, fail_mode);
     } else {
-        struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
+        struct vsctl_bridge *br = find_real_bridge(&info, bridge, true);
 
         if (!br->ctrl) {
             vsctl_fatal("no controller declared for %s", br->name);
         }
-        ovsrec_controller_set_fail_mode(br->ctrl, fail_mode);
+        set_fail_mode(br->ctrl, br->n_ctrl, fail_mode);
     }
 
     free_info(&info);
@@ -2530,7 +2621,7 @@ static const struct vsctl_command_syntax all_commands[] = {
     /* Controller commands. */
     {"get-controller", 0, 1, cmd_get_controller, NULL, ""},
     {"del-controller", 0, 1, cmd_del_controller, NULL, ""},
-    {"set-controller", 1, 2, cmd_set_controller, NULL, ""},
+    {"set-controller", 1, INT_MAX, cmd_set_controller, NULL, ""},
     {"get-fail-mode", 0, 1, cmd_get_fail_mode, NULL, ""},
     {"del-fail-mode", 0, 1, cmd_del_fail_mode, NULL, ""},
     {"set-fail-mode", 1, 2, cmd_set_fail_mode, NULL, ""},
diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
index 98a8e64..c94a02a 100644
--- a/vswitchd/bridge.c
+++ b/vswitchd/bridge.c
@@ -157,11 +157,6 @@ struct bridge {
     bool sent_config_request;   /* Successfully sent config request? */
     uint8_t default_ea[ETH_ADDR_LEN]; /* Default MAC. */
 
-    /* Support for remote controllers. */
-    char *controller;           /* NULL if there is no remote controller;
-                                 * "discover" to do controller discovery;
-                                 * otherwise a vconn name. */
-
     /* OpenFlow switch processing. */
     struct ofproto *ofproto;    /* OpenFlow switch. */
 
@@ -207,9 +202,9 @@ static void bridge_destroy(struct bridge *);
 static struct bridge *bridge_lookup(const char *name);
 static unixctl_cb_func bridge_unixctl_dump_flows;
 static int bridge_run_one(struct bridge *);
-static const struct ovsrec_controller *bridge_get_controller(
-                      const struct ovsrec_open_vswitch *ovs_cfg,
-                      const struct bridge *br);
+static size_t bridge_get_controllers(const struct ovsrec_open_vswitch *ovs_cfg,
+                                     const struct bridge *br,
+                                     struct ovsrec_controller ***controllersp);
 static void bridge_reconfigure_one(const struct ovsrec_open_vswitch *,
                                    struct bridge *);
 static void bridge_reconfigure_controller(const struct ovsrec_open_vswitch *,
@@ -744,8 +739,10 @@ bridge_reconfigure(const struct ovsrec_open_vswitch *ovs_cfg)
         /* Set sFlow configuration on this bridge. */
         if (br->cfg->sflow) {
             const struct ovsrec_sflow *sflow_cfg = br->cfg->sflow;
-            const struct ovsrec_controller *ctrl;
+            struct ovsrec_controller **controllers;
             struct ofproto_sflow_options oso;
+            size_t n_controllers;
+            size_t i;
 
             memset(&oso, 0, sizeof oso);
 
@@ -770,8 +767,14 @@ bridge_reconfigure(const struct ovsrec_open_vswitch *ovs_cfg)
             oso.sub_id = sflow_bridge_number++;
             oso.agent_device = sflow_cfg->agent;
 
-            ctrl = bridge_get_controller(ovs_cfg, br);
-            oso.control_ip = ctrl ? ctrl->local_ip : NULL;
+            oso.control_ip = NULL;
+            n_controllers = bridge_get_controllers(ovs_cfg, br, &controllers);
+            for (i = 0; i < n_controllers; i++) {
+                if (controllers[i]->local_ip) {
+                    oso.control_ip = controllers[i]->local_ip;
+                    break;
+                }
+            }
             ofproto_set_sflow(br->ofproto, &oso);
 
             svec_destroy(&oso.targets);
@@ -1051,7 +1054,7 @@ bridge_wait(void)
 
     LIST_FOR_EACH (br, struct bridge, node, &all_bridges) {
         ofproto_wait(br->ofproto);
-        if (br->controller) {
+        if (ofproto_has_controller(br->ofproto)) {
             continue;
         }
 
@@ -1181,7 +1184,6 @@ bridge_destroy(struct bridge *br)
         }
         dpif_close(br->dpif);
         ofproto_destroy(br->ofproto);
-        free(br->controller);
         mac_learning_destroy(br->ml);
         port_array_destroy(&br->ifaces);
         free(br->ports);
@@ -1257,21 +1259,31 @@ bridge_run_one(struct bridge *br)
     return error;
 }
 
-static const struct ovsrec_controller *
-bridge_get_controller(const struct ovsrec_open_vswitch *ovs_cfg,
-                      const struct bridge *br)
+static size_t
+bridge_get_controllers(const struct ovsrec_open_vswitch *ovs_cfg,
+                       const struct bridge *br,
+                       struct ovsrec_controller ***controllersp)
 {
-    const struct ovsrec_controller *controller;
+    struct ovsrec_controller **controllers;
+    size_t n_controllers;
 
-    controller = (br->cfg->controller ? br->cfg->controller
-                  : ovs_cfg->controller ? ovs_cfg->controller
-                  : NULL);
+    if (br->cfg->n_controller) {
+        controllers = br->cfg->controller;
+        n_controllers = br->cfg->n_controller;
+    } else {
+        controllers = ovs_cfg->controller;
+        n_controllers = ovs_cfg->n_controller;
+    }
 
-    if (controller && !strcmp(controller->target, "none")) {
-        return NULL;
+    if (n_controllers == 1 && !strcmp(controllers[0]->target, "none")) {
+        controllers = NULL;
+        n_controllers = 0;
     }
 
-    return controller;
+    if (controllersp) {
+        *controllersp = controllers;
+    }
+    return n_controllers;
 }
 
 static bool
@@ -1390,7 +1402,7 @@ bridge_reconfigure_one(const struct ovsrec_open_vswitch *ovs_cfg,
      * user didn't specify one.
      *
      * XXX perhaps we should synthesize a port ourselves in this case. */
-    if (bridge_get_controller(ovs_cfg, br)) {
+    if (bridge_get_controllers(ovs_cfg, br, NULL)) {
         char local_name[IF_NAMESIZE];
         int error;
 
@@ -1505,79 +1517,21 @@ static void
 bridge_reconfigure_controller(const struct ovsrec_open_vswitch *ovs_cfg,
                               struct bridge *br)
 {
-    const struct ovsrec_controller *c;
+    struct ovsrec_controller **controllers;
+    size_t n_controllers;
 
-    c = bridge_get_controller(ovs_cfg, br);
-    if ((br->controller != NULL) != (c != NULL)) {
+    n_controllers = bridge_get_controllers(ovs_cfg, br, &controllers);
+    if (ofproto_has_controller(br->ofproto) != (n_controllers != 0)) {
         ofproto_flush_flows(br->ofproto);
     }
-    free(br->controller);
-    br->controller = c ? xstrdup(c->target) : NULL;
-
-    if (c) {
-        struct ofproto_controller oc;
-
-        if (strcmp(c->target, "discover")) {
-            struct iface *local_iface;
-            struct in_addr ip;
-
-            local_iface = bridge_get_local_iface(br);
-            if (local_iface && c->local_ip && inet_aton(c->local_ip, &ip)) {
-                struct netdev *netdev = local_iface->netdev;
-                struct in_addr mask, gateway;
-
-                if (!c->local_netmask || !inet_aton(c->local_netmask, &mask)) {
-                    mask.s_addr = 0;
-                }
-                if (!c->local_gateway
-                    || !inet_aton(c->local_gateway, &gateway)) {
-                    gateway.s_addr = 0;
-                }
-
-                netdev_turn_flags_on(netdev, NETDEV_UP, true);
-                if (!mask.s_addr) {
-                    mask.s_addr = guess_netmask(ip.s_addr);
-                }
-                if (!netdev_set_in4(netdev, ip, mask)) {
-                    VLOG_INFO("bridge %s: configured IP address "IP_FMT", "
-                              "netmask "IP_FMT,
-                              br->name, IP_ARGS(&ip.s_addr),
-                              IP_ARGS(&mask.s_addr));
-                }
-
-                if (gateway.s_addr) {
-                    if (!netdev_add_router(netdev, gateway)) {
-                        VLOG_INFO("bridge %s: configured gateway "IP_FMT,
-                                  br->name, IP_ARGS(&gateway.s_addr));
-                    }
-                }
-            }
-        }
 
-        oc.target = c->target;
-        oc.max_backoff = c->max_backoff ? *c->max_backoff / 1000 : 8;
-        oc.probe_interval = (c->inactivity_probe
-                             ? *c->inactivity_probe / 1000 : 5);
-        oc.fail = (!c->fail_mode
-                   || !strcmp(c->fail_mode, "standalone")
-                   || !strcmp(c->fail_mode, "open")
-                   ? OFPROTO_FAIL_STANDALONE
-                   : OFPROTO_FAIL_SECURE);
-        oc.band = (!c->connection_mode
-                   || !strcmp(c->connection_mode, "out-of-band")
-                   ? OFPROTO_IN_BAND
-                   : OFPROTO_OUT_OF_BAND);
-        oc.accept_re = c->discover_accept_regex;
-        oc.update_resolv_conf = c->discover_update_resolv_conf;
-        oc.rate_limit = (c->controller_rate_limit
-                         ? *c->controller_rate_limit : 0);
-        oc.burst_limit = (c->controller_burst_limit
-                          ? *c->controller_burst_limit : 0);
-        ofproto_set_controller(br->ofproto, &oc);
-    } else {
+    if (!n_controllers) {
         union ofp_action action;
         flow_t flow;
 
+        /* Clear out controllers. */
+        ofproto_set_controllers(br->ofproto, NULL, 0);
+
         /* Set up a flow that matches every packet and directs them to
          * OFPP_NORMAL (which goes to us). */
         memset(&action, 0, sizeof action);
@@ -1587,8 +1541,74 @@ bridge_reconfigure_controller(const struct ovsrec_open_vswitch *ovs_cfg,
         memset(&flow, 0, sizeof flow);
         ofproto_add_flow(br->ofproto, &flow, OFPFW_ALL, 0,
                          &action, 1, 0);
+    } else {
+        struct ofproto_controller *ocs;
+        size_t i;
+
+        ocs = xmalloc(n_controllers * sizeof *ocs);
+        for (i = 0; i < n_controllers; i++) {
+            struct ovsrec_controller *c = controllers[i];
+            struct ofproto_controller *oc = &ocs[i];
+
+            if (strcmp(c->target, "discover")) {
+                struct iface *local_iface;
+                struct in_addr ip;
+
+                local_iface = bridge_get_local_iface(br);
+                if (local_iface && c->local_ip && inet_aton(c->local_ip, &ip)) {
+                    struct netdev *netdev = local_iface->netdev;
+                    struct in_addr mask, gateway;
+
+                    if (!c->local_netmask || !inet_aton(c->local_netmask, &mask)) {
+                        mask.s_addr = 0;
+                    }
+                    if (!c->local_gateway
+                        || !inet_aton(c->local_gateway, &gateway)) {
+                        gateway.s_addr = 0;
+                    }
+
+                    netdev_turn_flags_on(netdev, NETDEV_UP, true);
+                    if (!mask.s_addr) {
+                        mask.s_addr = guess_netmask(ip.s_addr);
+                    }
+                    if (!netdev_set_in4(netdev, ip, mask)) {
+                        VLOG_INFO("bridge %s: configured IP address "IP_FMT", "
+                                  "netmask "IP_FMT,
+                                  br->name, IP_ARGS(&ip.s_addr),
+                                  IP_ARGS(&mask.s_addr));
+                    }
+
+                    if (gateway.s_addr) {
+                        if (!netdev_add_router(netdev, gateway)) {
+                            VLOG_INFO("bridge %s: configured gateway "IP_FMT,
+                                      br->name, IP_ARGS(&gateway.s_addr));
+                        }
+                    }
+                }
+            }
 
-        ofproto_set_controller(br->ofproto, NULL);
+            oc->target = c->target;
+            oc->max_backoff = c->max_backoff ? *c->max_backoff / 1000 : 8;
+            oc->probe_interval = (c->inactivity_probe
+                                 ? *c->inactivity_probe / 1000 : 5);
+            oc->fail = (!c->fail_mode
+                       || !strcmp(c->fail_mode, "standalone")
+                       || !strcmp(c->fail_mode, "open")
+                       ? OFPROTO_FAIL_STANDALONE
+                       : OFPROTO_FAIL_SECURE);
+            oc->band = (!c->connection_mode
+                       || !strcmp(c->connection_mode, "out-of-band")
+                       ? OFPROTO_IN_BAND
+                       : OFPROTO_OUT_OF_BAND);
+            oc->accept_re = c->discover_accept_regex;
+            oc->update_resolv_conf = c->discover_update_resolv_conf;
+            oc->rate_limit = (c->controller_rate_limit
+                             ? *c->controller_rate_limit : 0);
+            oc->burst_limit = (c->controller_burst_limit
+                              ? *c->controller_burst_limit : 0);
+        }
+        ofproto_set_controllers(br->ofproto, ocs, n_controllers);
+        free(ocs);
     }
 }
 
diff --git a/vswitchd/ovs-brcompatd.c b/vswitchd/ovs-brcompatd.c
index b503705..2950301 100644
--- a/vswitchd/ovs-brcompatd.c
+++ b/vswitchd/ovs-brcompatd.c
@@ -661,8 +661,8 @@ del_bridge(struct ovsdb_idl *idl,
     if (br->sflow) {
         ovsrec_sflow_delete(br->sflow);
     }
-    if (br->controller) {
-        ovsrec_controller_delete(br->controller);
+    for (i = 0; i < br->n_controller; i++) {
+        ovsrec_controller_delete(br->controller[i]);
     }
 
     /* Remove 'br' from the vswitch's list of bridges. */
diff --git a/vswitchd/vswitch.ovsschema b/vswitchd/vswitch.ovsschema
index 8661875..9e3573f 100644
--- a/vswitchd/vswitch.ovsschema
+++ b/vswitchd/vswitch.ovsschema
@@ -9,7 +9,7 @@
        "controller": {
          "type": {"key": {"type": "uuid",
                           "refTable": "Controller"},
-                   "min": 0, "max": 1}},
+                   "min": 0, "max": "unlimited"}},
        "managers": {
          "type": {"key": "string", "min": 0, "max": "unlimited"}},
        "ssl": {
@@ -52,7 +52,7 @@
        "controller": {
          "type": {"key": {"type": "uuid",
                           "refTable": "Controller"},
-                  "min": 0, "max": 1}},
+                  "min": 0, "max": "unlimited"}},
        "other_config": {
          "type": {"key": "string", "value": "string", "min": 0, "max": "unlimited"}},
        "external_ids": {
diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index d3f3efb..ad1263d 100644
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -16,7 +16,7 @@
       </column>
 
       <column name="controller">
-        Default <ref table="Controller"/> used by bridges.  May be
+        Default OpenFlow <ref table="Controller"/> set used by bridges.  May be
         overridden on a per-bridge basis by the <ref table="Bridge"
         column="controller"/> column in <ref table="Bridge"/>.
       </column>
@@ -104,10 +104,11 @@
 
     <group title="OpenFlow Configuration">
       <column name="controller">
-        OpenFlow controller.  If unset, defaults to that specified by
-        <ref column="controller" table="Open_vSwitch"/> in the
-        <ref table="Open_vSwitch"/> table.  If the default is also unset, then
-        no OpenFlow controller will be used.
+        OpenFlow controller set.  If unset, defaults to the set of
+        controllers specified by <ref column="controller"
+        table="Open_vSwitch"/> in the <ref table="Open_vSwitch"/>
+        table.  If the default is also unset, then no OpenFlow
+        controllers will be used.
       </column>
 
       <column name="datapath_id">
@@ -500,13 +501,20 @@
   </table>
 
   <table name="Controller" title="OpenFlow controller configuration.">
-    An OpenFlow controller.
+    <p>An OpenFlow controller.</p>
+
+    <p>Open vSwitch permits a bridge to have any number of OpenFlow
+       controllers.  When multiple controllers are configured, Open vSwitch
+       connects to all of them simultaneously.  OpenFlow 1.0 does not specify
+       how multiple controllers coordinate in interacting with a single switch,
+       so more than one controller should be specified only if the controllers
+       are themselves designed to coordinate with each other.</p>
 
     <group title="Core Features">
       <column name="target">
-        Connection method for controller.
-        The following connection methods are currently
-        supported:
+        <p>Connection method for controller.
+          The following connection methods are currently
+          supported:</p>
         <dl>
           <dt><code>ssl:<var>ip</var></code>[<code>:<var>port</var></code>]</dt>
           <dd>
@@ -523,15 +531,58 @@
             the given <var>ip</var>, which must be expressed as an IP address
             (not a DNS name).</dd>
           <dt><code>discover</code></dt>
-          <dd>Enables controller discovery.</dd>
+          <dd>
+            <p>Enables controller discovery.</p>
+            <p>In controller discovery mode, Open vSwitch broadcasts a DHCP
+              request with vendor class identifier <code>OpenFlow</code> across
+              all of the bridge's network devices.  It will accept any valid
+              DHCP reply that has the same vendor class identifier and includes
+              a vendor-specific option with code 1 whose contents are a string
+              specifying the location of the controller in the same format as
+              <ref column="target"/>.</p>
+            <p>The DHCP reply may also, optionally, include a vendor-specific
+              option with code 2 whose contents are a string specifying the URI
+              to the base of the OpenFlow PKI
+              (e.g. <code>http://192.168.0.1/openflow/pki</code>).  This URI is
+              used only for bootstrapping the OpenFlow PKI at initial switch
+              setup; <code>ovs-vswitchd</code> does not use it at all.</p>
+          </dd>
           <dt><code>none</code></dt>
           <dd>Disables the controller.</dd>
         </dl>
+	<p>When multiple controllers are configured for a single bridge, the
+	  <ref column="target"/> values must be unique.  Duplicate
+	  <ref column="target"/> values yield unspecified results.</p>
       </column>
 
       <column name="connection_mode">
-        Either <code>in-band</code> or <code>out-of-band</code>.  If not
-        specified, the default is implementation-specific.
+	<p>If it is specified, this setting must be one of the following
+	strings that describes how Open vSwitch contacts this OpenFlow
+	controller over the network:</p>
+
+	<dl>
+	  <dt><code>in-band</code></dt>
+	  <dd>In this mode, this controller's OpenFlow traffic travels over the
+	    bridge associated with the controller.  With this setting, Open
+	    vSwitch allows traffic to and from the controller regardless of the
+	    contents of the OpenFlow flow table.  (Otherwise, Open vSwitch
+	    would never be able to connect to the controller, because it did
+	    not have a flow to enable it.)  This is the most common connection
+	    mode because it is not necessary to maintain two independent
+	    networks.</dd>
+	  <dt><code>out-of-band</code></dt>
+	  <dd>In this mode, OpenFlow traffic uses a control network separate
+	    from the bridge associated with this controller, that is, the
+	    bridge does not use any of its own network devices to communicate
+	    with the controller.  The control network must be configured
+	    separately, before or after <code>ovs-vswitchd</code> is started.
+	  </dd>
+	</dl>
+
+        <p>If not specified, the default is implementation-specific.  If
+          <ref column="target"/> is <code>discover</code>, the connection mode
+          is always treated as <code>in-band</code> regardless of the actual
+          setting.</p>
       </column>
     </group>
 
@@ -565,7 +616,7 @@
               times the inactivity probe interval
               (see <ref column="inactivity_probe"/>), then Open vSwitch
               will take over responsibility for setting up flows.  In
-              this mode, Open vSwitch causes the datapath to act like an
+              this mode, Open vSwitch causes the bridge to act like an
               ordinary MAC-learning switch.  Open vSwitch will continue
               to retry connecting to the controller in the background
               and, when the connection succeeds, it will discontinue its
@@ -576,19 +627,20 @@
               connecting to the controller forever.</dd>
           </dl>
         </p>
-        <p>If this value is unset, the default is
-        implementation-specific.</p>
+        <p>If this value is unset, the default is implementation-specific.</p>
+	<p>When more than one controller is configured,
+	  <ref column="fail_mode"/> is considered only when none of the
+	  configured controllers can be contacted.  At that point, the bridge
+	  enters secure mode if any of the controllers'
+	  <ref column="fail_mode"/> is set to <code>secure</code>.  Otherwise,
+	  it enters standalone mode if at least one <ref column="fail_mode"/>
+	  is set to <code>standalone</code>.  If none of the
+	  <ref column="fail_mode"/> values are set, the default is
+	  implementation-defined.</p>
       </column>
     </group>
 
     <group title="OpenFlow Rate Limiting">
-        <column name="controller_burst_limit">
-          In conjunction with <ref column="controller_rate_limit"/>,
-          the maximum number of unused packet credits that the bridge will
-          allow to accumulate, in packets.  If not specified, the default
-          is implementation-specific.
-        </column>
-
         <column name="controller_rate_limit">
           <p>The maximum rate at which packets in unknown flows will be
             forwarded to the OpenFlow controller, in packets per second.  This
@@ -608,11 +660,21 @@
             actual rate that packets are sent to the controller is up to
             twice the specified rate.</p>
         </column>
+
+        <column name="controller_burst_limit">
+          In conjunction with <ref column="controller_rate_limit"/>,
+          the maximum number of unused packet credits that the bridge will
+          allow to accumulate, in packets.  If not specified, the default
+          is implementation-specific.
+        </column>
     </group>
 
-    <group title="Additional Configuration for Discovery">
+    <group title="Additional Discovery Configuration">
+      <p>These values are considered only when <ref column="target"/>
+	is <code>discover</code>.</p>
+
       <column name="discover_accept_regex">
-        If <ref column="target"/> is <code>discover</code>, a POSIX
+        A POSIX
         extended regular expression against which the discovered controller
         location is validated.  The regular expression is implicitly
         anchored at the beginning of the controller location string, as
@@ -621,8 +683,7 @@
       </column>
 
       <column name="discover_update_resolv_conf">
-        If <ref column="target"/> is <code>discover</code>,
-        whether to update <code>/etc/resolv.conf</code> when the
+        Whether to update <code>/etc/resolv.conf</code> when the
         controller is discovered.  If not specified, the default
         is implementation-specific.  Open vSwitch will only modify
         <code>/etc/resolv.conf</code> if the DHCP response that it receives
@@ -630,20 +691,35 @@
       </column>
     </group>
 
-    <group title="Additional Configuration without Discovery">
-      <column name="local_gateway">
-        If <ref column="target"/> is not <code>discover</code>, the IP
-        address of the gateway to configure on the local port.
-      </column>
+    <group title="Additional In-Band Configuration">
+      <p>These values are considered only in in-band control mode (see
+	<ref column="connection_mode"/>) and only when <ref column="target"/>
+	is not <code>discover</code>.  (For controller discovery, the network
+	configuration obtained via DHCP is used instead.)</p>
+
+      <p>When multiple controllers are configured on a single bridge, there
+	should be only one set of unique values in these columns.  If different
+	values are set for these columns in different controllers, the effect
+	is unspecified.</p>
 
       <column name="local_ip">
-        If <ref column="target"/> is not <code>discover</code>, the IP
-        address to configure on the local port.
+        The IP address to configure on the local port,
+        e.g. <code>192.168.0.123</code>.  If this value is unset, then
+        <ref column="local_netmask"/> and <ref column="local_gateway"/> are
+        ignored.
       </column>
 
       <column name="local_netmask">
-        If <ref column="target"/> is not <code>discover</code>, the IP
-        netmask to configure on the local port.
+        The IP netmask to configure on the local port,
+        e.g. <code>255.255.255.0</code>.  If <ref column="local_ip"/> is set
+        but this value is unset, then the default is chosen based on whether
+        the IP address is class A, B, or C.
+      </column>
+
+      <column name="local_gateway">
+        The IP address of the gateway to configure on the local port, as a
+        string, e.g. <code>192.168.0.1</code>.  Leave this column unset if
+        this network has no gateway.
       </column>
     </group>
   </table>
-- 
1.6.6.1





More information about the dev mailing list