[ovs-dev] [PATCH ovn v3 04/13] ovn-ic: Interconnection controller with AZ registeration.

Han Zhou hzhou at ovn.org
Tue Jan 28 02:55:29 UTC 2020


This patch introduces interconnection controller, ovn-ic, and
implements the basic AZ registration feature: taking the AZ
name from NB DB and create an Availability_Zone entry in
IC-SB DB.

Signed-off-by: Han Zhou <hzhou at ovn.org>
---
 Makefile.am          |   1 +
 ic/.gitignore        |   2 +
 ic/automake.mk       |  10 ++
 ic/ovn-ic.8.xml      | 120 +++++++++++++
 ic/ovn-ic.c          | 467 +++++++++++++++++++++++++++++++++++++++++++++++++++
 ovn-nb.ovsschema     |   5 +-
 ovn-nb.xml           |   7 +
 tests/automake.mk    |   4 +-
 tests/ovn-ic.at      |  28 +++
 tests/ovn-macros.at  | 161 +++++++++++++++---
 tests/testsuite.at   |   1 +
 tutorial/ovs-sandbox |  78 ++++++++-
 12 files changed, 854 insertions(+), 30 deletions(-)
 create mode 100644 ic/.gitignore
 create mode 100644 ic/automake.mk
 create mode 100644 ic/ovn-ic.8.xml
 create mode 100644 ic/ovn-ic.c
 create mode 100644 tests/ovn-ic.at

diff --git a/Makefile.am b/Makefile.am
index 8eed7a7..d524c0d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -500,4 +500,5 @@ include selinux/automake.mk
 include controller/automake.mk
 include controller-vtep/automake.mk
 include northd/automake.mk
+include ic/automake.mk
 include build-aux/automake.mk
diff --git a/ic/.gitignore b/ic/.gitignore
new file mode 100644
index 0000000..1b73eb4
--- /dev/null
+++ b/ic/.gitignore
@@ -0,0 +1,2 @@
+/ovn-ic
+/ovn-ic.8
diff --git a/ic/automake.mk b/ic/automake.mk
new file mode 100644
index 0000000..8e71bc3
--- /dev/null
+++ b/ic/automake.mk
@@ -0,0 +1,10 @@
+# ovn-ic
+bin_PROGRAMS += ic/ovn-ic
+ic_ovn_ic_SOURCES = ic/ovn-ic.c
+ic_ovn_ic_LDADD = \
+	lib/libovn.la \
+	$(OVSDB_LIBDIR)/libovsdb.la \
+	$(OVS_LIBDIR)/libopenvswitch.la
+man_MANS += ic/ovn-ic.8
+EXTRA_DIST += ic/ovn-ic.8.xml
+CLEANFILES += ic/ovn-ic.8
diff --git a/ic/ovn-ic.8.xml b/ic/ovn-ic.8.xml
new file mode 100644
index 0000000..f5ef994
--- /dev/null
+++ b/ic/ovn-ic.8.xml
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manpage program="ovn-ic" section="8" title="ovn-ic">
+    <h1>Name</h1>
+    <p>ovn-ic -- Open Virtual Network interconnection controller</p>
+
+    <h1>Synopsis</h1>
+    <p><code>ovn-ic</code> [<var>options</var>]</p>
+
+    <h1>Description</h1>
+    <p>
+      <code>ovn-ic</code>, OVN interconnection controller, is a centralized
+      daemon which communicates with global interconnection databases IC_NB/IC_SB
+      to configure and exchange data with local NB/SB for interconnecting
+      with other OVN deployments.
+    </p>
+
+    <h1>Options</h1>
+    <dl>
+      <dt><code>--ovnnb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Northbound Database.  If the
+        <env>OVN_NB_DB</env> environment variable is set, its value is used
+        as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovnnb_db.sock</code>.
+      </dd>
+      <dt><code>--ovnsb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Southbound Database.  If the
+        <env>OVN_SB_DB</env> environment variable is set, its value is used
+        as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovnsb_db.sock</code>.
+      </dd>
+      <dt><code>--ic-nb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Interconnection Northbound
+        Database.  If the <env>OVN_IC_NB_DB</env> environment variable is set,
+        its value is used as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovn_ic_nb_db.sock</code>.
+      </dd>
+      <dt><code>--ic-sb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Interconnection Southbound
+        Database.  If the <env>OVN_IC_SB_DB</env> environment variable is set,
+        its value is used as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovn_ic_sb_db.sock</code>.
+      </dd>
+    </dl>
+    <p>
+      <var>database</var> in the above options must be an OVSDB active or
+      passive connection method, as described in <code>ovsdb</code>(7).
+    </p>
+
+    <h2>Daemon Options</h2>
+    <xi:include href="lib/daemon.xml" xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h2>Logging Options</h2>
+    <xi:include href="lib/vlog.xml" xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h2>PKI Options</h2>
+    <p>
+      PKI configuration is required in order to use SSL for the connections to
+      the Northbound and Southbound databases.
+    </p>
+    <xi:include href="lib/ssl.xml" xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h2>Other Options</h2>
+    <xi:include href="lib/unixctl.xml"
+     xmlns:xi="http://www.w3.org/2003/XInclude"/>
+    <h3></h3>
+    <xi:include href="lib/common.xml"
+     xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h1>Runtime Management Commands</h1>
+    <p>
+      <code>ovs-appctl</code> can send commands to a running
+      <code>ovn-ic</code> process.  The currently supported commands
+      are described below.
+      <dl>
+      <dt><code>exit</code></dt>
+      <dd>
+        Causes <code>ovn-ic</code> to gracefully terminate.
+      </dd>
+
+      <dt><code>pause</code></dt>
+      <dd>
+        Pauses the ovn-ic operation from processing any database changes.
+        This will also instruct ovn-ic to drop any lock on SB DB.
+      </dd>
+
+      <dt><code>resume</code></dt>
+      <dd>
+        Resumes the ovn-ic operation to process database contents.  This will
+        also instruct ovn-northd to aspire for the lock on SB DB.
+      </dd>
+
+      <dt><code>is-paused</code></dt>
+      <dd>
+        Returns "true" if ovn-ic is currently paused, "false" otherwise.
+      </dd>
+
+      <dt><code>status</code></dt>
+      <dd>
+        Prints this server's status.  Status will be "active" if ovn-ic has
+        acquired OVSDB lock on SB DB, "standby" if it has not or "paused" if
+        this instance is paused.
+      </dd>
+      </dl>
+
+    </p>
+
+    <h1>Active-Standby for High Availability</h1>
+    <p>
+      You may run <code>ovn-ic</code> more than once in an OVN deployment.
+      When connected to a standalone or clustered DB setup, OVN will
+      automatically ensure that only one of them is active at a time.
+      If multiple instances of <code>ovn-ic</code> are running and the
+      active <code>ovn-ic</code> fails, one of the hot standby instances
+      of <code>ovn-ic</code> will automatically take over.
+    </p>
+</manpage>
diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c
new file mode 100644
index 0000000..a6d27c0
--- /dev/null
+++ b/ic/ovn-ic.c
@@ -0,0 +1,467 @@
+/*
+ * Copyright (c) 2020 eBay Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include <getopt.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitmap.h"
+#include "command-line.h"
+#include "daemon.h"
+#include "dirs.h"
+#include "openvswitch/dynamic-string.h"
+#include "fatal-signal.h"
+#include "hash.h"
+#include "openvswitch/hmap.h"
+#include "lib/ovn-ic-nb-idl.h"
+#include "lib/ovn-ic-sb-idl.h"
+#include "lib/ovn-nb-idl.h"
+#include "lib/ovn-sb-idl.h"
+#include "lib/ovn-util.h"
+#include "openvswitch/poll-loop.h"
+#include "smap.h"
+#include "sset.h"
+#include "stream.h"
+#include "stream-ssl.h"
+#include "unixctl.h"
+#include "util.h"
+#include "uuid.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(ovn_ic);
+
+static unixctl_cb_func ovn_ic_exit;
+static unixctl_cb_func ovn_ic_pause;
+static unixctl_cb_func ovn_ic_resume;
+static unixctl_cb_func ovn_ic_is_paused;
+static unixctl_cb_func ovn_ic_status;
+
+struct ic_context {
+    struct ovsdb_idl *ovnnb_idl;
+    struct ovsdb_idl *ovnsb_idl;
+    struct ovsdb_idl *ovninb_idl;
+    struct ovsdb_idl *ovnisb_idl;
+    struct ovsdb_idl_txn *ovnnb_txn;
+    struct ovsdb_idl_txn *ovnsb_txn;
+    struct ovsdb_idl_txn *ovninb_txn;
+    struct ovsdb_idl_txn *ovnisb_txn;
+};
+
+struct ic_state {
+    bool had_lock;
+    bool paused;
+};
+
+static const char *ovnnb_db;
+static const char *ovnsb_db;
+static const char *ovn_ic_nb_db;
+static const char *ovn_ic_sb_db;
+static const char *unixctl_path;
+
+
+static void
+usage(void)
+{
+    printf("\
+%s: OVN interconnection management daemon\n\
+usage: %s [OPTIONS]\n\
+\n\
+Options:\n\
+  --ovnnb-db=DATABASE       connect to ovn-nb database at DATABASE\n\
+                            (default: %s)\n\
+  --ovnsb-db=DATABASE       connect to ovn-sb database at DATABASE\n\
+                            (default: %s)\n\
+  --unixctl=SOCKET          override default control socket name\n\
+  -h, --help                display this help message\n\
+  -o, --options             list available options\n\
+  -V, --version             display version information\n\
+", program_name, program_name, default_nb_db(), default_sb_db());
+    daemon_usage();
+    vlog_usage();
+    stream_usage("database", true, true, false);
+}
+
+static const struct icsbrec_availability_zone *
+az_run(struct ic_context *ctx)
+{
+    const struct nbrec_nb_global *nb_global =
+        nbrec_nb_global_first(ctx->ovnnb_idl);
+
+    if (!nb_global) {
+        VLOG_INFO("NB Global not exist.");
+        return NULL;
+    }
+
+    /* Delete old AZ if name changes.  Note: if name changed when ovn-ic
+     * is not running, one has to manually delete the old AZ with:
+     * "ovn-ic-sbctl destroy avail <az>". */
+    static char *az_name;
+    const struct icsbrec_availability_zone *az;
+    if (az_name && strcmp(az_name, nb_global->name)) {
+        ICSBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
+            if (!strcmp(az->name, az_name)) {
+                icsbrec_availability_zone_delete(az);
+                break;
+            }
+        }
+        free(az_name);
+        az_name = NULL;
+    }
+
+    if (!nb_global->name[0]) {
+        return NULL;
+    }
+
+    if (!az_name) {
+        az_name = xstrdup(nb_global->name);
+    }
+
+    ICSBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
+        if (!strcmp(az->name, az_name)) {
+            return az;
+        }
+    }
+
+    /* Create AZ in ISB */
+    if (ctx->ovnisb_txn) {
+        VLOG_INFO("Register AZ %s to interconnection DB.", az_name);
+        az = icsbrec_availability_zone_insert(ctx->ovnisb_txn);
+        icsbrec_availability_zone_set_name(az, az_name);
+        return az;
+    }
+    return NULL;
+}
+
+static void
+ovn_db_run(struct ic_context *ctx)
+{
+    const struct icsbrec_availability_zone *az = az_run(ctx);
+    VLOG_DBG("Availability zone: %s", az ? az->name : "not created yet.");
+}
+
+static void
+parse_options(int argc OVS_UNUSED, char *argv[] OVS_UNUSED)
+{
+    enum {
+        DAEMON_OPTION_ENUMS,
+        VLOG_OPTION_ENUMS,
+        SSL_OPTION_ENUMS,
+    };
+    static const struct option long_options[] = {
+        {"ovnsb-db", required_argument, NULL, 'd'},
+        {"ovnnb-db", required_argument, NULL, 'D'},
+        {"ic-sb-db", required_argument, NULL, 'i'},
+        {"ic-nb-db", required_argument, NULL, 'I'},
+        {"unixctl", required_argument, NULL, 'u'},
+        {"help", no_argument, NULL, 'h'},
+        {"options", no_argument, NULL, 'o'},
+        {"version", no_argument, NULL, 'V'},
+        DAEMON_LONG_OPTIONS,
+        VLOG_LONG_OPTIONS,
+        STREAM_SSL_LONG_OPTIONS,
+        {NULL, 0, NULL, 0},
+    };
+    char *short_options = ovs_cmdl_long_options_to_short_options(long_options);
+
+    for (;;) {
+        int c;
+
+        c = getopt_long(argc, argv, short_options, long_options, NULL);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+        DAEMON_OPTION_HANDLERS;
+        VLOG_OPTION_HANDLERS;
+        STREAM_SSL_OPTION_HANDLERS;
+
+        case 'd':
+            ovnsb_db = optarg;
+            break;
+
+        case 'D':
+            ovnnb_db = optarg;
+            break;
+
+        case 'i':
+            ovn_ic_sb_db = optarg;
+            break;
+
+        case 'I':
+            ovn_ic_nb_db = optarg;
+            break;
+
+        case 'u':
+            unixctl_path = optarg;
+            break;
+
+        case 'h':
+            usage();
+            exit(EXIT_SUCCESS);
+
+        case 'o':
+            ovs_cmdl_print_options(long_options);
+            exit(EXIT_SUCCESS);
+
+        case 'V':
+            ovs_print_version(0, 0);
+            exit(EXIT_SUCCESS);
+
+        default:
+            break;
+        }
+    }
+
+    if (!ovnsb_db) {
+        ovnsb_db = default_sb_db();
+    }
+
+    if (!ovnnb_db) {
+        ovnnb_db = default_nb_db();
+    }
+
+    if (!ovn_ic_sb_db) {
+        ovn_ic_sb_db = default_ic_sb_db();
+    }
+
+    if (!ovn_ic_nb_db) {
+        ovn_ic_nb_db = default_ic_nb_db();
+    }
+
+    free(short_options);
+}
+
+static void OVS_UNUSED
+add_column_noalert(struct ovsdb_idl *idl,
+                   const struct ovsdb_idl_column *column)
+{
+    ovsdb_idl_add_column(idl, column);
+    ovsdb_idl_omit_alert(idl, column);
+}
+
+int
+main(int argc, char *argv[])
+{
+    int res = EXIT_SUCCESS;
+    struct unixctl_server *unixctl;
+    int retval;
+    bool exiting;
+    struct ic_state state;
+
+    fatal_ignore_sigpipe();
+    ovs_cmdl_proctitle_init(argc, argv);
+    set_program_name(argv[0]);
+    service_start(&argc, &argv);
+    parse_options(argc, argv);
+
+    daemonize_start(false);
+
+    if (!unixctl_path) {
+        char *abs_unixctl_path = get_abs_unix_ctl_path();
+        retval = unixctl_server_create(abs_unixctl_path, &unixctl);
+        free(abs_unixctl_path);
+    } else {
+        retval = unixctl_server_create(unixctl_path, &unixctl);
+    }
+
+    if (retval) {
+        exit(EXIT_FAILURE);
+    }
+    unixctl_command_register("exit", "", 0, 0, ovn_ic_exit, &exiting);
+    unixctl_command_register("pause", "", 0, 0, ovn_ic_pause, &state);
+    unixctl_command_register("resume", "", 0, 0, ovn_ic_resume, &state);
+    unixctl_command_register("is-paused", "", 0, 0, ovn_ic_is_paused, &state);
+    unixctl_command_register("status", "", 0, 0, ovn_ic_status, &state);
+
+    daemonize_complete();
+
+    /* ovn-ic-nb db. */
+    struct ovsdb_idl_loop ovninb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovn_ic_nb_db, &icnbrec_idl_class, true, true));
+
+    /* ovn-ic-sb db. */
+    struct ovsdb_idl_loop ovnisb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovn_ic_sb_db, &icsbrec_idl_class, true, true));
+
+    /* ovn-nb db. XXX: add only needed tables and columns */
+    struct ovsdb_idl_loop ovnnb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovnnb_db, &nbrec_idl_class, true, true));
+
+    /* ovn-sb db. XXX: add only needed tables and columns */
+    struct ovsdb_idl_loop ovnsb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovnsb_db, &sbrec_idl_class, true, true));
+
+    /* Main loop. */
+    exiting = false;
+    state.had_lock = false;
+    state.paused = false;
+    while (!exiting) {
+        if (!state.paused) {
+            if (!ovsdb_idl_has_lock(ovnsb_idl_loop.idl) &&
+                !ovsdb_idl_is_lock_contended(ovnsb_idl_loop.idl))
+            {
+                /* Ensure that only a single ovn-ic is active in the deployment
+                 * by acquiring a lock called "ovn_ic" on the southbound
+                 * database and then only performing DB transactions if the
+                 * lock is held. */
+                ovsdb_idl_set_lock(ovnsb_idl_loop.idl, "ovn_ic");
+            }
+
+            struct ic_context ctx = {
+                .ovnnb_idl = ovnnb_idl_loop.idl,
+                .ovnnb_txn = ovsdb_idl_loop_run(&ovnnb_idl_loop),
+                .ovnsb_idl = ovnsb_idl_loop.idl,
+                .ovnsb_txn = ovsdb_idl_loop_run(&ovnsb_idl_loop),
+                .ovninb_idl = ovninb_idl_loop.idl,
+                .ovninb_txn = ovsdb_idl_loop_run(&ovninb_idl_loop),
+                .ovnisb_idl = ovnisb_idl_loop.idl,
+                .ovnisb_txn = ovsdb_idl_loop_run(&ovnisb_idl_loop),
+            };
+
+            if (!state.had_lock && ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
+                VLOG_INFO("ovn-ic lock acquired. "
+                        "This ovn-ic instance is now active.");
+                state.had_lock = true;
+            } else if (state.had_lock &&
+                       !ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
+                VLOG_INFO("ovn-ic lock lost. "
+                        "This ovn-ic instance is now on standby.");
+                state.had_lock = false;
+            }
+
+            if (ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
+                ovn_db_run(&ctx);
+            }
+
+            ovsdb_idl_loop_commit_and_wait(&ovnnb_idl_loop);
+            ovsdb_idl_loop_commit_and_wait(&ovnsb_idl_loop);
+            ovsdb_idl_loop_commit_and_wait(&ovninb_idl_loop);
+            ovsdb_idl_loop_commit_and_wait(&ovnisb_idl_loop);
+        } else {
+            /* ovn-ic is paused
+             *    - we still want to handle any db updates and update the
+             *      local IDL. Otherwise, when it is resumed, the local IDL
+             *      copy will be out of sync.
+             *    - but we don't want to create any txns.
+             * */
+            if (ovsdb_idl_has_lock(ovnsb_idl_loop.idl) ||
+                ovsdb_idl_is_lock_contended(ovnsb_idl_loop.idl))
+            {
+                /* make sure we don't hold the lock while paused */
+                VLOG_INFO("This ovn-ic instance is now paused.");
+                ovsdb_idl_set_lock(ovnsb_idl_loop.idl, NULL);
+                state.had_lock = false;
+            }
+
+            ovsdb_idl_run(ovnnb_idl_loop.idl);
+            ovsdb_idl_run(ovnsb_idl_loop.idl);
+            ovsdb_idl_run(ovninb_idl_loop.idl);
+            ovsdb_idl_run(ovnisb_idl_loop.idl);
+            ovsdb_idl_wait(ovnnb_idl_loop.idl);
+            ovsdb_idl_wait(ovnsb_idl_loop.idl);
+            ovsdb_idl_wait(ovninb_idl_loop.idl);
+            ovsdb_idl_wait(ovnisb_idl_loop.idl);
+        }
+
+        unixctl_server_run(unixctl);
+        unixctl_server_wait(unixctl);
+        if (exiting) {
+            poll_immediate_wake();
+        }
+
+        poll_block();
+        if (should_service_stop()) {
+            exiting = true;
+        }
+    }
+
+    unixctl_server_destroy(unixctl);
+    ovsdb_idl_loop_destroy(&ovnnb_idl_loop);
+    ovsdb_idl_loop_destroy(&ovnsb_idl_loop);
+    ovsdb_idl_loop_destroy(&ovninb_idl_loop);
+    ovsdb_idl_loop_destroy(&ovnisb_idl_loop);
+    service_stop();
+
+    exit(res);
+}
+
+static void
+ovn_ic_exit(struct unixctl_conn *conn, int argc OVS_UNUSED,
+            const char *argv[] OVS_UNUSED, void *exiting_)
+{
+    bool *exiting = exiting_;
+    *exiting = true;
+
+    unixctl_command_reply(conn, NULL);
+}
+
+static void
+ovn_ic_pause(struct unixctl_conn *conn, int argc OVS_UNUSED,
+             const char *argv[] OVS_UNUSED, void *state_)
+{
+    struct ic_state *state = state_;
+    state->paused = true;
+
+    unixctl_command_reply(conn, NULL);
+}
+
+static void
+ovn_ic_resume(struct unixctl_conn *conn, int argc OVS_UNUSED,
+              const char *argv[] OVS_UNUSED, void *state_)
+{
+    struct ic_state *state = state_;
+    state->paused = false;
+
+    unixctl_command_reply(conn, NULL);
+}
+
+static void
+ovn_ic_is_paused(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                 const char *argv[] OVS_UNUSED, void *state_)
+{
+    struct ic_state *state = state_;
+    if (state->paused) {
+        unixctl_command_reply(conn, "true");
+    } else {
+        unixctl_command_reply(conn, "false");
+    }
+}
+
+static void
+ovn_ic_status(struct unixctl_conn *conn, int argc OVS_UNUSED,
+              const char *argv[] OVS_UNUSED, void *state_)
+{
+    struct ic_state *state = state_;
+    char *status;
+
+    if (state->paused) {
+        status = "paused";
+    } else {
+        status = state->had_lock ? "active" : "standby";
+    }
+
+    /*
+     * Use a labelled formatted output so we can add more to the status command
+     * later without breaking any consuming scripts
+     */
+    struct ds s = DS_EMPTY_INITIALIZER;
+    ds_put_format(&s, "Status: %s\n", status);
+    unixctl_command_reply(conn, ds_cstr(&s));
+    ds_destroy(&s);
+}
diff --git a/ovn-nb.ovsschema b/ovn-nb.ovsschema
index 7d969fb..bbd6c25 100644
--- a/ovn-nb.ovsschema
+++ b/ovn-nb.ovsschema
@@ -1,10 +1,11 @@
 {
     "name": "OVN_Northbound",
-    "version": "5.19.0",
-    "cksum": "4258826789 24879",
+    "version": "5.20.0",
+    "cksum": "987891875 24923",
     "tables": {
         "NB_Global": {
             "columns": {
+                "name": {"type": "string"},
                 "nb_cfg": {"type": {"key": "integer"}},
                 "sb_cfg": {"type": {"key": "integer"}},
                 "hv_cfg": {"type": {"key": "integer"}},
diff --git a/ovn-nb.xml b/ovn-nb.xml
index 8c9ef90..c7b04ae 100644
--- a/ovn-nb.xml
+++ b/ovn-nb.xml
@@ -36,6 +36,13 @@
       one row.
     </p>
 
+    <group title="Identity">
+      <column name="name">
+        The name of the OVN cluster, which uniquely identifies the OVN cluster
+        throughout all OVN clusters supposed to interconnect with each other.
+      </column>
+    </group>
+
     <group title="Status">
       These columns allow a client to track the overall configuration state of
       the system.
diff --git a/tests/automake.mk b/tests/automake.mk
index c70a213..215fb43 100644
--- a/tests/automake.mk
+++ b/tests/automake.mk
@@ -28,6 +28,7 @@ TESTSUITE_AT = \
 	tests/ovn-ic-sbctl.at \
 	tests/ovn-controller.at \
 	tests/ovn-controller-vtep.at \
+	tests/ovn-ic.at \
 	tests/ovn-macros.at \
 	tests/ovn-performance.at
 
@@ -54,7 +55,7 @@ SYSTEM_KMOD_TESTSUITE = $(srcdir)/tests/system-kmod-testsuite
 SYSTEM_USERSPACE_TESTSUITE = $(srcdir)/tests/system-userspace-testsuite
 DISTCLEANFILES += tests/atconfig tests/atlocal
 
-AUTOTEST_PATH = $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller
+AUTOTEST_PATH = $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller:ic
 
 export ovs_srcdir
 
@@ -103,6 +104,7 @@ valgrind_wrappers = \
 	tests/valgrind/ovn-sbctl \
 	tests/valgrind/ovn-ic-nbctl \
 	tests/valgrind/ovn-ic-sbctl \
+	tests/valgrind/ovn-ic \
 	tests/valgrind/ovs-appctl \
 	tests/valgrind/ovs-ofctl \
 	tests/valgrind/ovs-vsctl \
diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
new file mode 100644
index 0000000..b20105a
--- /dev/null
+++ b/tests/ovn-ic.at
@@ -0,0 +1,28 @@
+AT_BANNER([OVN Interconnection Controller])
+AT_SETUP([ovn-ic -- AZ register])
+
+ovn_init_ic_db
+ovn_start az1
+ovn_start az2
+
+AT_CHECK([ovn-ic-sbctl show], [0], [dnl
+availability-zone az1
+availability-zone az2
+])
+
+ovn_as az1
+ovn-nbctl set NB_Global . name=az3
+AT_CHECK([ovn-ic-sbctl show], [0], [dnl
+availability-zone az2
+availability-zone az3
+])
+
+ovn_as az2
+ovn-nbctl set NB_Global . name=\"\"
+AT_CHECK([ovn-ic-sbctl show], [0], [dnl
+availability-zone az3
+])
+
+OVN_CLEANUP_IC([az1], [az2])
+
+AT_CLEANUP
diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at
index 4045ce0..53a22a8 100644
--- a/tests/ovn-macros.at
+++ b/tests/ovn-macros.at
@@ -48,9 +48,49 @@ m4_define([OVN_CLEANUP],[
     OVN_CLEANUP_VSWITCH([main])
 ])
 
+# OVN_CLEANUP_AZ(az)
+#
+# Gracefully terminate all OVN daemons, including those in the
+# specified sandbox instances.
+m4_define([OVN_CLEANUP_AZ],[
+    as $1/ovn-sb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    as $1/ovn-nb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    as $1/northd
+    OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+    as $1/northd-backup
+    OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+    as $1/ic
+    OVS_APP_EXIT_AND_WAIT([ovn-ic])
+])
+
+# OVN_CLEANUP_IC([az ...])
+#
+# Gracefully terminate all interconnection DBs, and daemons in the
+# specified AZs, if any.
+m4_define([OVN_CLEANUP_IC],[
+    m4_foreach([az], [$@], [
+        OVN_CLEANUP_AZ([az])
+    ])
+    as ovn-ic-sb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    as ovn-ic-nb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    if test -d "$ovs_base"/main; then
+        OVN_CLEANUP_VSWITCH([main])
+    fi
+])
+
 m4_divert_push([PREPARE_TESTS])
 
-# ovn_init_db DATABASE
+# ovn_init_db DATABASE [AZ]
 #
 # Creates and initializes the given DATABASE (one of "ovn-sb" or "ovn-nb"),
 # starts its ovsdb-server instance, and sets the appropriate environment
@@ -60,36 +100,77 @@ m4_divert_push([PREPARE_TESTS])
 # Usually invoked from ovn_start.
 ovn_init_db () {
     echo "creating $1 database"
-    local d=$ovs_base/$1
+    local as_d=$1
+    if test -n "$2"; then
+        as_d=$2/$as_d
+    fi
+    local d=$ovs_base/$as_d
     mkdir "$d" || return 1
     : > "$d"/.$1.db.~lock~
-    as $1 ovsdb-tool create "$d"/$1.db "$abs_top_srcdir"/$1.ovsschema
-    as $1 start_daemon ovsdb-server --remote=punix:"$d"/$1.sock "$d"/$1.db
+    as $as_d ovsdb-tool create "$d"/$1.db "$abs_top_srcdir"/$1.ovsschema
+    as $as_d start_daemon ovsdb-server --remote=punix:"$d"/$1.sock "$d"/$1.db
     local var=`echo $1_db | tr a-z- A-Z_`
-    AS_VAR_SET([$var], [unix:$ovs_base/$1/$1.sock]); export $var
+    AS_VAR_SET([$var], [unix:"$d"/$1.sock]); export $var
 }
 
-# ovn_start
+# ovn_init_ic_db
+#
+# Creates and initializes ovn-ic-nb and ovn-ic-sb databases and starts their
+# ovsdb-server instances, for OVN interconnection.
+ovn_init_ic_db () {
+    ovn_init_db ovn-ic-nb
+    ovn_init_db ovn-ic-sb
+}
+
+# ovn_start [AZ]
 #
 # Creates and initializes ovn-sb and ovn-nb databases and starts their
 # ovsdb-server instance, sets appropriate environment variables so that
 # ovn-sbctl and ovn-nbctl use them by default, and starts ovn-northd running
 # against them.
 ovn_start () {
-    ovn_init_db ovn-sb; ovn-sbctl init
-    ovn_init_db ovn-nb; ovn-nbctl init
+    if test -n "$1"; then
+        mkdir "$ovs_base"/$1
+    fi
 
+    ovn_init_db ovn-sb $1; ovn-sbctl init
+    ovn_init_db ovn-nb $1; ovn-nbctl init
+    if test -n "$1"; then
+        ovn-nbctl set NB_Global . name=$1
+    fi
+    local ovn_sb_db=$OVN_SB_DB
+    local ovn_nb_db=$OVN_NB_DB
+
+    local as_d=northd
+    if test -n "$1"; then
+        as_d=$1/$as_d
+    fi
     echo "starting ovn-northd"
-    mkdir "$ovs_base"/northd
-    as northd start_daemon ovn-northd -v \
-               --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock \
-               --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock
+    mkdir "$ovs_base"/$as_d
+    as $as_d start_daemon ovn-northd -v \
+               --ovnnb-db=$ovn_nb_db \
+               --ovnsb-db=$ovn_sb_db
 
+    as_d=northd-backup
+    if test -n "$1"; then
+        as_d=$1/$as_d
+    fi
     echo "starting backup ovn-northd"
-    mkdir "$ovs_base"/northd-backup
-    as northd-backup start_daemon ovn-northd -v \
-               --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock \
-               --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock
+    mkdir "$ovs_base"/$as_d
+    as $as_d start_daemon ovn-northd -v \
+               --ovnnb-db=$ovn_nb_db \
+               --ovnsb-db=$ovn_sb_db
+
+    if test -n "$1"; then
+        as_d=$1/ic
+        echo "starting ovn-ic"
+        mkdir "$ovs_base"/$as_d
+        as $as_d start_daemon ovn-ic -v \
+               --ovnnb-db=$ovn_nb_db \
+               --ovnsb-db=$ovn_sb_db \
+               --ic-nb-db=unix:"$ovs_base"/ovn-ic-nb/ovn-ic-nb.sock \
+               --ic-sb-db=unix:"$ovs_base"/ovn-ic-sb/ovn-ic-sb.sock
+    fi
 }
 
 # Interconnection networks.
@@ -132,24 +213,25 @@ net_attach () {
         || return 1
 }
 
-# ovn_attach NETWORK BRIDGE IP [MASKLEN]
-#
-# First, this command attaches BRIDGE to interconnection network NETWORK, just
-# like "net_attach NETWORK BRIDGE".  Second, it configures (simulated) IP
-# address IP (with network mask length MASKLEN, which defaults to 24) on
-# BRIDGE.  Finally, it configures the Open vSwitch database to work with OVN
-# and starts ovn-controller.
-ovn_attach() {
-    local net=$1 bridge=$2 ip=$3 masklen=${4-24}
+# ovn_az_attach AZ NETWORK BRIDGE IP [MASKLEN]
+ovn_az_attach() {
+    local az=$1 net=$2 bridge=$3 ip=$4 masklen=${5-24}
     net_attach $net $bridge || return 1
 
     mac=`ovs-vsctl get Interface $bridge mac_in_use | sed s/\"//g`
     arp_table="$arp_table $sandbox,$bridge,$ip,$mac"
     ovs-appctl netdev-dummy/ip4addr $bridge $ip/$masklen >/dev/null || return 1
     ovs-appctl ovs/route/add $ip/$masklen $bridge >/dev/null || return 1
+
+    local ovn_remote
+    if test X"$az" = XNONE; then
+        ovn_remote=unix:$ovs_base/ovn-sb/ovn-sb.sock
+    else
+        ovn_remote=unix:$ovs_base/$az/ovn-sb/ovn-sb.sock
+    fi
     ovs-vsctl \
         -- set Open_vSwitch . external-ids:system-id=$sandbox \
-        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
+        -- set Open_vSwitch . external-ids:ovn-remote=$ovn_remote \
         -- set Open_vSwitch . external-ids:ovn-encap-type=geneve,vxlan \
         -- set Open_vSwitch . external-ids:ovn-encap-ip=$ip \
         -- add-br br-int \
@@ -158,6 +240,33 @@ ovn_attach() {
     start_daemon ovn-controller || return 1
 }
 
+# ovn_attach NETWORK BRIDGE IP [MASKLEN]
+#
+# First, this command attaches BRIDGE to interconnection network NETWORK, just
+# like "net_attach NETWORK BRIDGE".  Second, it configures (simulated) IP
+# address IP (with network mask length MASKLEN, which defaults to 24) on
+# BRIDGE.  Finally, it configures the Open vSwitch database to work with OVN
+# and starts ovn-controller.
+ovn_attach() {
+    ovn_az_attach NONE $@
+}
+
+# ovn_setenv AZ
+ovn_setenv () {
+    local d=$ovs_base/$1
+    AS_VAR_SET([OVN_NB_DB], [unix:"$d"/ovn-nb/ovn-nb.sock]); export $var
+    AS_VAR_SET([OVN_SB_DB], [unix:"$d"/ovn-sb/ovn-sb.sock]); export $var
+}
+
+# ovs_as AZ
+ovn_as() {
+    if test "X$2" != X; then
+        (ovn_setenv $1; shift; "$@")
+    else
+        ovn_setenv $1
+    fi
+}
+
 # OVN_POPULATE_ARP
 #
 # This pre-populates the ARP tables of all of the OVN instances that have been
diff --git a/tests/testsuite.at b/tests/testsuite.at
index eff6732..1985923 100644
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -30,4 +30,5 @@ m4_include([tests/ovn-ic-nbctl.at])
 m4_include([tests/ovn-ic-sbctl.at])
 m4_include([tests/ovn-controller.at])
 m4_include([tests/ovn-controller-vtep.at])
+m4_include([tests/ovn-ic.at])
 m4_include([tests/checkpatch.at])
diff --git a/tutorial/ovs-sandbox b/tutorial/ovs-sandbox
index 36cdb6b..1841776 100755
--- a/tutorial/ovs-sandbox
+++ b/tutorial/ovs-sandbox
@@ -58,6 +58,8 @@ gdb_vswitchd_ex=false
 gdb_ovsdb_ex=false
 gdb_ovn_northd=false
 gdb_ovn_northd_ex=false
+gdb_ovn_ic=false
+gdb_ovn_ic_ex=false
 gdb_ovn_controller=false
 gdb_ovn_controller_ex=false
 gdb_ovn_controller_vtep=false
@@ -72,13 +74,20 @@ built=false
 ovn=true
 ovnsb_schema=
 ovnnb_schema=
+ic_sb_schema=
+ic_nb_schema=
 ovn_rbac=true
 n_northds=1
+n_ics=1
 n_controllers=1
 nbdb_model=standalone
 nbdb_servers=3
 sbdb_model=backup
 sbdb_servers=3
+ic_nb_model=clustered
+ic_nb_servers=3
+ic_sb_model=clustered
+ic_sb_servers=3
 dummy=override
 
 for option; do
@@ -125,6 +134,7 @@ General options:
   -g, --gdb-vswitchd   run ovs-vswitchd under gdb
   -d, --gdb-ovsdb      run ovsdb-server under gdb
   --gdb-ovn-northd     run ovn-northd under gdb
+  --gdb-ovn-ic         run ovn-ic under gdb
   --gdb-ovn-controller run ovn-controller under gdb
   --gdb-ovn-controller-vtep run ovn-controller-vtep under gdb
   --dummy=ARG          pass --enable-dummy=ARG to vswitchd (default: override)
@@ -135,10 +145,15 @@ General options:
 OVN options:
   --no-ovn-rbac        disable role-based access control for OVN
   --n-northds=NUMBER   run NUMBER copies of northd (default: 1)
+  --n-ics=NUMBER       run NUMBER copies of ic (default: 1)
   --nbdb-model=standalone|backup|clustered    northbound database model
   --nbdb-servers=N     number of servers in nbdb cluster (default: 3)
   --sbdb-model=standalone|backup|clustered    southbound database model
   --sbdb-servers=N     number of servers in sbdb cluster (default: 3)
+  --ic-nb-model=standalone|backup|clustered   ic-northbound database model
+  --ic-nb-servers=N     number of servers in IC NB cluster (default: 3)
+  --ic-sb-model=standalone|backup|clustered   ic-southbound database model
+  --ic-sb-servers=N     number of servers in IC SB cluster (default: 3)
 
 Other options:
   -h, --help           Print this usage message.
@@ -210,6 +225,9 @@ EOF
         --gdb-ovn-northd)
             gdb_ovn_northd=true
             ;;
+        --gdb-ovn-ic)
+            gdb_ovn_ic=true
+            ;;
         --gdb-ovn-controller)
             gdb_ovn_controller=true
             ;;
@@ -225,6 +243,12 @@ EOF
         --n-northd*)
             prev=n_northds
             ;;
+        --n-ic*=*)
+            n_ics=$optarg
+            ;;
+        --n-ic*)
+            prev=n_ics_
+            ;;
         --n-controller*=*)
             n_controllers=$optarg
             ;;
@@ -259,10 +283,39 @@ EOF
         --sbdb-m*)
             prev=sbdb_model
             ;;
+        --ic-nb-s*=*)
+            ic_nb_servers=$optarg
+            ic_nb_model=clustered
+            ;;
+        --ic-nb-s*)
+            prev=ic_nb_servers
+            ic_nb_model=clustered
+            ;;
+        --ic-nb-m*=*)
+            ic_nb_model=$optarg
+            ;;
+        --ic-nb-m*)
+            prev=ic_nb_model
+            ;;
+        --ic-sb-s*=*)
+            ic_sb_servers=$optarg
+            ic_sb_model=clustered
+            ;;
+        --ic-sb-s*)
+            prev=ic_sb_servers
+            ic_sb_model=clustered
+            ;;
+        --ic-sb-m*=*)
+            ic_sb_model=$optarg
+            ;;
+        --ic-sb-m*)
+            prev=ic_sb_model
+            ;;
         -R|--gdb-run)
             gdb_vswitchd_ex=true
             gdb_ovsdb_ex=true
             gdb_ovn_northd_ex=true
+            gdb_ovn_ic_ex=true
             gdb_ovn_controller_ex=true
             gdb_ovn_controller_vtep_ex=true
             ;;
@@ -327,6 +380,16 @@ if $built; then
             echo >&2 'source directory not found, please use --srcdir'
             exit 1
         fi
+        ic_sb_schema=$srcdir/ovn-ic-sb.ovsschema
+        if test ! -e "$ic_sb_schema"; then
+            echo >&2 'source directory not found, please use --srcdir'
+            exit 1
+        fi
+        ic_nb_schema=$srcdir/ovn-ic-nb.ovsschema
+        if test ! -e "$ic_nb_schema"; then
+            echo >&2 'source directory not found, please use --srcdir'
+            exit 1
+        fi
         vtep_schema=$ovssrcdir/vtep/vtep.ovsschema
         if test ! -e "$vtep_schema"; then
             echo >&2 'source directory not found, please use --srcdir'
@@ -340,7 +403,7 @@ if $built; then
         exit 1
     fi
     PATH=$ovsbuilddir/ovsdb:$ovsbuilddir/vswitchd:$ovsbuilddir/utilities:$ovsbuilddir/vtep:$PATH
-    PATH=$builddir/controller:$builddir/controller-vtep:$builddir/northd:$builddir/utilities:$PATH
+    PATH=$builddir/controller:$builddir/controller-vtep:$builddir/northd:$builddir/ic:$builddir/utilities:$PATH
     export PATH
 else
     case $schema in
@@ -490,6 +553,8 @@ The backup database file is sandbox/${db}2.db
 backup_note=
 ovn_start_db nb "$nbdb_model" "$nbdb_servers" "$ovnnb_schema"
 ovn_start_db sb "$sbdb_model" "$sbdb_servers" "$ovnsb_schema"
+ovn_start_db ic_nb "$ic_nb_model" "$ic_nb_servers" "$ic_nb_schema"
+ovn_start_db ic_sb "$ic_sb_model" "$ic_sb_servers" "$ic_sb_schema"
 
 #Add a small delay to allow ovsdb-server to launch.
 sleep 0.1
@@ -512,6 +577,9 @@ rungdb $gdb_vswitchd $gdb_vswitchd_ex ovs-vswitchd --detach --no-chdir --pidfile
 
 ovn-nbctl init
 ovn-sbctl init
+ovn-ic-nbctl init
+ovn-ic-sbctl init
+ovn-nbctl set NB_Global . name=az-1
 
 ovs-vsctl set open . external-ids:system-id=chassis-1
 ovs-vsctl set open . external-ids:hostname=sandbox
@@ -533,6 +601,14 @@ else
     ovs-vsctl set open . external-ids:ovn-remote=$OVN_SB_DB
     OVN_CTRLR_PKI=""
 fi
+for i in $(seq $n_ics); do
+    if [ $i -eq 1 ]; then inst=""; else inst=$i; fi
+    rungdb $gdb_ovn_ic $gdb_ovn_ic_ex ovn-ic --detach \
+            --no-chdir --pidfile=ovn-ic${inst}.pid -vconsole:off \
+            --log-file=ovn-ic${inst}.log -vsyslog:off \
+            --ovnsb-db="$OVN_SB_DB" --ovnnb-db="$OVN_NB_DB" \
+            --ic-sb-db="$OVN_IC_SB_DB" --ic-nb-db="$OVN_IC_NB_DB"
+done
 for i in $(seq $n_northds); do
     if [ $i -eq 1 ]; then inst=""; else inst=$i; fi
     rungdb $gdb_ovn_northd $gdb_ovn_northd_ex ovn-northd --detach \
-- 
2.1.0



More information about the dev mailing list