[ovs-dev] [PATCH] ovn-ctl: Support starting clustered OVN dbs
nusiddiq at redhat.com
nusiddiq at redhat.com
Wed Mar 28 11:37:47 UTC 2018
From: Numan Siddique <nusiddiq at redhat.com>
This patch adds the options to start clustered OVN db servers.
To support this following options are added - '--db-nb-cluster-local-addr',
'--db-nb-cluster-remote-addr', '--db-sb-cluster-local-addr' and
'--db-sb-cluster-remote-addr'. If only '--db-(nb/sb)-cluster-local-addr' is defined
then clustered db is created (using ovsdb-tool create-cluster). If both are defined,
then the db is added to the cluster (using ovsdb-tool join-cluster)
Presently this patch doesn't handle the schema update scenario when restarting the
clustered ovsdb-servers. This will be handled in a separate patch.
(There are 2 checkpatch warnings 'Line length is >79-characters long' in ovn-ctl.8.xml)
Signed-off-by: Numan Siddique <nusiddiq at redhat.com>
---
ovn/utilities/ovn-ctl | 175 ++++++++++++++++++++++++++++++--------------
ovn/utilities/ovn-ctl.8.xml | 70 ++++++++++++++++++
utilities/ovs-lib.in | 71 +++++++++++++++++-
3 files changed, 259 insertions(+), 57 deletions(-)
diff --git a/ovn/utilities/ovn-ctl b/ovn/utilities/ovn-ctl
index dc0c26159..ed8f41081 100755
--- a/ovn/utilities/ovn-ctl
+++ b/ovn/utilities/ovn-ctl
@@ -95,80 +95,131 @@ promote_ovnsb() {
start_nb_ovsdb() {
# Check and eventually start ovsdb-server for Northbound DB
- if ! pidfile_is_running $DB_NB_PID; then
- upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null
+ if pidfile_is_running $DB_SB_PID; then
+ return
+ fi
- set ovsdb-server
+ mode="standalone"
+ if test ! -z "$DB_NB_CLUSTER_LOCAL_ADDR"; then
+ mode="cluster"
+ elif test ! -z "$DB_NB_SYNC_FROM_ADDR"; then
+ mode="active_passive"
+ echo "$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:\
+$DB_NB_SYNC_FROM_PORT" > $ovnnb_active_conf_file
+ fi
- if test X"$DB_NB_DETACH" != Xno; then
- set "$@" --detach --monitor
+ if test X$mode = "Xcluster"; then
+ if test -z "$DB_NB_CLUSTER_REMOTE_ADDR"; then
+ create_cluster "$DB_NB_FILE" "$DB_NB_SCHEMA" \
+"$DB_NB_CLUSTER_LOCAL_ADDR"
else
- set exec "$@"
+ join_cluster "$DB_NB_FILE" "OVN_Northbound" \
+"$DB_NB_CLUSTER_LOCAL_ADDR" "$DB_NB_CLUSTER_REMOTE_ADDR"
fi
+ else
+ upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null
+ fi
- set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE
- set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID
- set "$@" --remote=db:OVN_Northbound,NB_Global,connections
- set "$@" --unixctl=ovnnb_db.ctl
- set "$@" --private-key=db:OVN_Northbound,SSL,private_key
- set "$@" --certificate=db:OVN_Northbound,SSL,certificate
- set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert
- set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols
- set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers
-
- if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then
- set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR
- fi
+ set ovsdb-server
- if test ! -z "$DB_NB_SYNC_FROM_ADDR"; then
- echo "$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:$DB_NB_SYNC_FROM_PORT" > $ovnnb_active_conf_file
- fi
+ if test X"$DB_NB_DETACH" != Xno; then
+ set "$@" --detach --monitor
+ else
+ set exec "$@"
+ fi
- if test -e $ovnnb_active_conf_file; then
- set "$@" --sync-from=`cat $ovnnb_active_conf_file`
- fi
+ set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE
+ set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID
+ set "$@" --unixctl=ovnnb_db.ctl
+
+ set "$@" --remote=db:OVN_Northbound,NB_Global,connections
+ set "$@" --private-key=db:OVN_Northbound,SSL,private_key
+ set "$@" --certificate=db:OVN_Northbound,SSL,certificate
+ set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert
+ set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols
+ set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers
+
+ if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then
+ set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR
+ fi
+
+ # Set '--sync-from' if ovnnb_active_conf_file is present and mode
+ # is 'active_passive'
+ if test -e $ovnnb_active_conf_file && test X$mode = "Xactive_passive"; then
+ set "$@" --sync-from=`cat $ovnnb_active_conf_file`
+ fi
- $@ $DB_NB_FILE
+ $@ $DB_NB_FILE
+
+ # Initialize the database if it's running standalone, active-passive,
+ # or is the first server in a cluster.
+ if test -z "$DB_NB_CLUSTER_REMOTE_ADDR"; then
ovn-nbctl init
fi
}
start_sb_ovsdb() {
# Check and eventually start ovsdb-server for Southbound DB
- if ! pidfile_is_running $DB_SB_PID; then
- upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null
+ if pidfile_is_running $DB_SB_PID; then
+ return
+ fi
+
+ mode="standalone"
- set ovsdb-server
+ if test ! -z "$DB_SB_CLUSTER_LOCAL_ADDR"; then
+ mode="cluster"
+ elif test ! -z "$DB_SB_SYNC_FROM_ADDR"; then
+ mode="active_passive"
+ echo "$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:\
+$DB_SB_SYNC_FROM_PORT" > $ovnsb_active_conf_file
+ fi
- if test X"$DB_SB_DETACH" != Xno; then
- set "$@" --detach --monitor
+ if test X$mode = "Xcluster"; then
+ if test -z "$DB_SB_CLUSTER_REMOTE_ADDR"; then
+ create_cluster "$DB_SB_FILE" "$DB_SB_SCHEMA" \
+"$DB_SB_CLUSTER_LOCAL_ADDR"
else
- set exec "$@"
+ join_cluster "$DB_SB_FILE" "OVN_Southbound" \
+"$DB_SB_CLUSTER_LOCAL_ADDR" "$DB_SB_CLUSTER_REMOTE_ADDR"
fi
+ else
+ upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null
+ fi
- set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE
- set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID
- set "$@" --remote=db:OVN_Southbound,SB_Global,connections
- set "$@" --unixctl=ovnsb_db.ctl
- set "$@" --private-key=db:OVN_Southbound,SSL,private_key
- set "$@" --certificate=db:OVN_Southbound,SSL,certificate
- set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert
- set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols
- set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers
-
- if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then
- set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR
- fi
+ set ovsdb-server
- if test ! -z "$DB_SB_SYNC_FROM_ADDR"; then
- echo "$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:$DB_SB_SYNC_FROM_PORT" > $ovnsb_active_conf_file
- fi
+ if test X"$DB_SB_DETACH" != Xno; then
+ set "$@" --detach --monitor
+ else
+ set exec "$@"
+ fi
- if test -e $ovnsb_active_conf_file; then
- set "$@" --sync-from=`cat $ovnsb_active_conf_file`
- fi
+ set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE
+ set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID
+ set "$@" --unixctl=ovnsb_db.ctl
+
+ set "$@" --remote=db:OVN_Southbound,SB_Global,connections
+ set "$@" --private-key=db:OVN_Southbound,SSL,private_key
+ set "$@" --certificate=db:OVN_Southbound,SSL,certificate
+ set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert
+ set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols
+ set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers
+
+ if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then
+ set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR
+ fi
+
+ # Set '--sync-from' if ovnsb_active_conf_file is present and mode
+ # is 'active_passive'
+ if test -e $ovnsb_active_conf_file && test X$mode = "Xactive_passive"; then
+ set "$@" --sync-from=`cat $ovnsb_active_conf_file`
+ fi
- $@ $DB_SB_FILE
+ $@ $DB_SB_FILE
+
+ # Initialize the database if it's running standalone, active-passive,
+ # or is the first server in a cluster.
+ if test -z "$DB_SB_CLUSTER_REMOTE_ADDR"; then
ovn-sbctl init
fi
}
@@ -236,7 +287,8 @@ start_northd () {
exit
fi
fi
- ovn_northd_params="--ovnnb-db=unix:$DB_NB_SOCK --ovnsb-db=unix:$DB_SB_SOCK"
+ ovn_northd_params="--ovnnb-db=$OVN_NORTHD_NB_DB \
+ --ovnsb-db=$OVN_NORTHD_SB_DB"
else
ovn_northd_params="`cat $ovn_northd_db_conf_file`"
fi
@@ -406,6 +458,13 @@ set_defaults () {
DB_NB_DETACH="yes"
DB_SB_DETACH="yes"
+
+ DB_NB_CLUSTER_LOCAL_ADDR=""
+ DB_NB_CLUSTER_REMOTE_ADDR=""
+ DB_SB_CLUSTER_LOCAL_ADDR=""
+ DB_SB_CLUSTER_REMOTE_ADDR=""
+ OVN_NORTHD_NB_DB="unix:$DB_NB_SOCK"
+ OVN_NORTHD_SB_DB="unix:$DB_SB_SOCK"
}
set_option () {
@@ -494,6 +553,16 @@ File location options:
--db-sb-sync-from-port=ADDR OVN Southbound active db tcp port (default: $DB_SB_SYNC_FROM_PORT)
--db-sb-sync-from-proto=PROTO OVN Southbound active db transport (default: $DB_SB_SYNC_FROM_PROTO)
--db-sb-create-insecure-remote=yes|no Create ptcp OVN Southbound remote (default: $DB_SB_CREATE_INSECURE_REMOTE)
+ --db-nb-cluster-local-addr=ADDR OVN_Northbound cluster local address \
+ (default: $DB_NB_CLUSTER_LOCAL_ADDR)
+ --db-nb-cluster-remote-addr=ADDR OVN_Northbound cluster remote address \
+ (default: $DB_NB_CLUSTER_REMOTE_ADDR)
+ --db-sb-cluster-local-addr=ADDR OVN_Southbound cluster local address \
+ (default: $DB_SB_CLUSTER_LOCAL_ADDR)
+ --db-sb-cluster-remote-addr=ADDR OVN_Southbound cluster remote address \
+ (default: $DB_SB_CLUSTER_REMOTE_ADDR)
+ --ovn-northd-nb-db=NB DB address(es) (default: $OVN_NORTHD_NB_DB)
+ --ovn-northd-sb-db=SB DB address(es) (default: $OVN_NORTHD_SB_DB)
Default directories with "configure" option and environment variable override:
logs: /usr/local/var/log/openvswitch (--with-logdir, OVS_LOGDIR)
diff --git a/ovn/utilities/ovn-ctl.8.xml b/ovn/utilities/ovn-ctl.8.xml
index 40defc9ec..ae81ef15e 100644
--- a/ovn/utilities/ovn-ctl.8.xml
+++ b/ovn/utilities/ovn-ctl.8.xml
@@ -66,6 +66,43 @@
<p><code>--db-sb-sync-from-addr=<var>IP ADDRESS</var></code></p>
<p><code>--db-sb-sync-from-port=<var>PORT NUMBER</var></code></p>
<p><code>--db-sb-sync-from-proto=<var>PROTO</var></code></p>
+ <p>
+ <code>
+ --ovn-northd-nb-db=<var>PROTO</var>:<var>IP ADDRESS</var>:
+ <var>PORT</var>..
+ </code>
+ </p>
+ <p>
+ <code>
+ --ovn-northd-sb-db=<var>PROTO</var>:<var>IP ADDRESS</var>:
+ <var>PORT</var>..
+ </code>
+ </p>
+ <h1> Clustering options </h1>
+ <p>
+ <code>
+ --db-nb-cluster-local-addr=<var>PROTO</var>:
+ <var>IP ADDRESS</var>:<var>PORT</var>
+ </code>
+ </p>
+ <p>
+ <code>
+ --db-nb-cluster-remote-addr=<var>PROTO</var>:
+ <var>IP ADDRESS</var>:<var>PORT</var>
+ </code>
+ </p>
+ <p>
+ <code>
+ --db-sb-cluster-local-addr=<var>PROTO</var>:
+ <var>IP ADDRESS</var>:<var>PORT</var>
+ </code>
+ </p>
+ <p>
+ <code>
+ --db-sb-cluster-remote-addr=<var>PROTO</var>:
+ <var>IP ADDRESS</var>:<var>PORT</var>
+ </code>
+ </p>
<h1>Configuration files</h1>
<p>Following are the optional configuration files. If present, it should be located in the etc dir</p>
@@ -125,4 +162,37 @@
<p><code># ovn-ctl promote_ovnsb</code></p>
<p><code># ovn-ctl --db-nb-sync-from-addr=x.x.x.x --db-nb-sync-from-port=6641 demote_ovnnb</code></p>
<p><code># ovn-ctl --db-sb-sync-from-addr=x.x.x.x --db-sb-sync-from-port=6642 demote_ovnsb</code></p>
+
+ <h2>Starting OVN ovsdb servers cluster on a node with IP x.x.x.x</h2>
+ <p>
+ <code>
+ # ovn-ctl --db-nb-cluster-local-addr="tcp:x.x.x.x:6643"
+ --db-sb-cluster-local-addr="tcp:x.x.x.x:6644" start_ovsdb
+ </code>
+ </p>
+
+ <h2>Starting OVN ovsdb-servers on a node with IP y.y.y.y and joining a cluster started at x.x.x.x</h2>
+ <p>
+ <code>
+ # ovn-ctl --db-nb-cluster-local-addr="tcp:x.x.x.x:6643"
+ --db-sb-cluster-local-addr="tcp:x.x.x.x:6644"
+ --db-nb-cluster-remote-addr="tcp:y.y.y.y:6643"
+ --db-sb-cluster-remote-addr="tcp:y.y.y.y:6644" start_ovsdb
+ </code>
+ </p>
+
+ <h2>Starting OVN ovsdb-servers cluster and ovn-northd on a node with IP x.x.x.x with other cluster nodes - y.y.y.y and z.z.z.z</h2>
+ <p>
+ <code>
+ # ovn-ctl -db-nb-addr=x.x.x.x --db-nb-port=6641
+ --db-nb-create-insecure-remote=yes
+ --db-nb-cluster-local-addr=tcp:x.x.x.x:6643
+ --db-sb-addr=x.x.x.x --db-sb-port=6642
+ --db-sb-create-insecure-remote=yes
+ --db-sb-cluster-local-addr=tcp:x.x.x.x:6644
+ --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+ --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+ start_northd
+ </code>
+ </p>
</manpage>
diff --git a/utilities/ovs-lib.in b/utilities/ovs-lib.in
index cf6b6d296..6647e5b6c 100644
--- a/utilities/ovs-lib.in
+++ b/utilities/ovs-lib.in
@@ -399,6 +399,14 @@ create_db () {
action "Creating empty database $DB_FILE" ovsdb_tool create "$DB_FILE" "$DB_SCHEMA"
}
+backup_db() {
+ DB_FILE="$1"
+ version=`ovsdb_tool db-version "$DB_FILE"`
+ cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
+ backup=$DB_FILE.backup$version-$cksum
+ action "Backing up database to $backup" cp "$DB_FILE" "$backup" || return 1
+}
+
upgrade_db () {
DB_FILE="$1"
DB_SCHEMA="$2"
@@ -410,10 +418,10 @@ upgrade_db () {
create_db "$DB_FILE" "$DB_SCHEMA"
elif test X"`ovsdb_tool needs-conversion "$DB_FILE" "$DB_SCHEMA"`" = Xyes; then
# Back up the old version.
- version=`ovsdb_tool db-version "$DB_FILE"`
- cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
- backup=$DB_FILE.backup$version-$cksum
- action "Backing up database to $backup" cp "$DB_FILE" "$backup" || return 1
+ backup_db $DB_FILE
+ if [ "$?" != "0" ]; then
+ return 1
+ fi
# Compact database. This is important if the old schema did not enable
# garbage collection (i.e. if it did not have any tables with "isRoot":
@@ -613,3 +621,58 @@ restart () {
flow_restore_complete
add_managers
}
+
+
+create_cluster () {
+ DB_FILE="$1"
+ DB_SCHEMA="$2"
+ LOCAL_ADDR="$3"
+
+ if test ! -e "$DB_FILE"; then
+ action "Creating cluster database $DB_FILE" ovsdb_tool create-cluster \
+"$DB_FILE" "$DB_SCHEMA" "$LOCAL_ADDR"
+ else
+ # DB file exists. Check if it is a standalone db or not. If it is a
+ # standalone db, create a clustered db from the standalone db
+ ovsdb_tool db-is-standalone $DB_FILE
+ if [ "$?" = "0" ]; then
+ backup_db $DB_FILE
+ if [ "$?" != "0" ]; then
+ return 1
+ fi
+
+ rm -f $DB_FILE
+ action "Creating cluster database from standalone db $DB_FILE" \
+ ovsdb_tool create-cluster "$DB_FILE" "$backup" "$LOCAL_ADDR"
+ fi
+
+ # TODO(numans) - If it is a clustered db, then
+ # - upgrade the db if required
+ # - compact the db.
+ fi
+}
+
+join_cluster() {
+ DB_FILE="$1"
+ SCHEMA_NAME="$2"
+ LOCAL_ADDR="$3"
+ REMOTE_ADDR="$4"
+
+ if test ! -e "$DB_FILE"; then
+ ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR" "$REMOTE_ADDR"
+ else
+ # DB file exists. Check if it is a standalone db or not. If it is a
+ # standalone db, backup the db and join the cluster.
+ ovsdb_tool db-is-standalone $DB_FILE
+ if [ "$?" = "0" ]; then
+ backup_db $DB_FILE
+ if [ "$?" != "0" ]; then
+ return 1
+ fi
+
+ rm -f $DB_FILE
+ action "Joining cluster database" \
+ ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR" "$REMOTE_ADDR"
+ fi
+ fi
+}
--
2.14.3
More information about the dev
mailing list