[ovs-dev] [PATCH v2] ovn-ctl: Support starting clustered OVN dbs
Ben Pfaff
blp at ovn.org
Tue Apr 3 23:12:02 UTC 2018
On Wed, Apr 04, 2018 at 12:24:27AM +0530, nusiddiq at redhat.com wrote:
> From: Numan Siddique <nusiddiq at redhat.com>
>
> This patch adds the options to start clustered OVN db servers in ovn-ctl.
> To support this, following options are added - '--db-(nb/sb)-cluster-local-addr',
> '--db-(nb/sb)-cluster-local-port', '--db-(nb/sb)-cluster-local-proto',
> '--db-(nb/sb)-cluster-remote-addr', '--db-(nb/sb)-cluster-remote-port' and
> '--db-(nb/sb)-cluster-remote-proto'.
>
> If only '--db-(nb/sb)-cluster-local-addr' is defined, then clustered db is created
> (using ovsdb-tool create-cluster). If both are defined, then the db is added to
> the cluster (using ovsdb-tool join-cluster)
>
> This patch also adds the support to configure ovn-northd to point to all the servers
> in the cluster using the options - '--ovn-northd-nb-db' and 'ovn-northd-sb-db'.
>
> Presently this patch doesn't handle the schema update scenario when restarting the
> clustered ovsdb-servers. This will be handled in a separate patch.
>
> The initial versions of these commands are tested by Aliasgar <aginwala <aginwala at asu.edu>
> and the discussion on this can be found here -
> https://mail.openvswitch.org/pipermail/ovs-discuss/2018-March/046470.html
>
> (There are 4 checkpatch warnings 'Line length is >79-characters long' in ovn-ctl.8.xml
> which I couldn't resolve without losing proper rendering when "man ovn-ctl" is run.)
>
> Signed-off-by: Numan Siddique <nusiddiq at redhat.com>
Hi Numan, thanks for working on this.
I'm pasting a version of the patch that I spent some time on myself. I
think that your version has better documentation and it may have other
enhancement. My version merges the code for starting the nb and sb
database servers into a single function and it adds support for
upgrading the cluster, but on the other hand I haven't tested it at all.
Do you think you could try to produce a version that merges all of these
benefits?
Thanks,
Ben.
--8<--------------------------cut here-------------------------->8--
diff --git a/ovn/utilities/ovn-ctl b/ovn/utilities/ovn-ctl
index dc0c261590fb..029aa081c7ba 100755
--- a/ovn/utilities/ovn-ctl
+++ b/ovn/utilities/ovn-ctl
@@ -93,89 +93,95 @@ promote_ovnsb() {
ovs-appctl -t $rundir/ovnsb_db.ctl ovsdb-server/disconnect-active-ovsdb-server
}
-start_nb_ovsdb() {
- # Check and eventually start ovsdb-server for Northbound DB
- if ! pidfile_is_running $DB_NB_PID; then
- upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null
+start_ovsdb__() {
+ local DB=$1 db=$2 schema_name=$3 table_name=$4
+ eval local pid=\$DB_${DB}_PID
+ eval local cluster_local_addr=\$DB_${DB}_CLUSTER_LOCAL_ADDR
+ eval local cluster_remote_addr=\$DB_${DB}_CLUSTER_REMOTE_ADDR
+ eval local sync_from_proto=\$DB_${DB}_SYNC_FROM_PROTO
+ eval local sync_from_addr=\$DB_${DB}_SYNC_FROM_ADDR
+ eval local sync_from_port=\$DB_${DB}_SYNC_FROM_PORT
+ eval local file=\$DB_${DB}_FILE
+ eval local schema=\$DB_${DB}_SCHEMA
+ eval local logfile=\$OVN_${DB}_LOGFILE
+ eval local log=\$OVN_${DB}_LOG
+ eval local sock=\$DB_${DB}_SOCK
+ eval local detach=\$DB_${DB}_DETACH
+ eval local create_insecure_remote=\$DB_${DB}_CREATE_INSECURE_REMOTE
+ eval local port=\$DB_${DB}_PORT
+ eval local addr=\$DB_${DB}_ADDR
+ eval local active_conf_file=\$ovn${db}_active_conf_file
+
+ # Check and eventually start ovsdb-server for DB
+ if pidfile_is_running $pid; then
+ return
+ fi
- set ovsdb-server
+ if test ! -z "$cluster_local_addr"; then
+ mode=cluster
+ elif test ! -z "$sync_from_addr"; then
+ mode=active_passive
+ echo "$sync_from_proto:$sync_from_addr:$sync_from_port" > $active_conf_file
+ else
+ mode=standalone
+ fi
- if test X"$DB_NB_DETACH" != Xno; then
- set "$@" --detach --monitor
+ if test $mode = cluster; then
+ local local=$cluster_local_addr
+ local remote=$cluster_remote_addr
+ if test -n "$remote"; then
+ join_cluster "$file" "$schema_name" "$local" "$remote"
else
- set exec "$@"
- fi
-
- set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE
- set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID
- set "$@" --remote=db:OVN_Northbound,NB_Global,connections
- set "$@" --unixctl=ovnnb_db.ctl
- set "$@" --private-key=db:OVN_Northbound,SSL,private_key
- set "$@" --certificate=db:OVN_Northbound,SSL,certificate
- set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert
- set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols
- set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers
-
- if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then
- set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR
- fi
-
- if test ! -z "$DB_NB_SYNC_FROM_ADDR"; then
- echo "$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:$DB_NB_SYNC_FROM_PORT" > $ovnnb_active_conf_file
+ create_cluster "$file" "$schema" "$local"
fi
+ else
+ upgrade_db "$file" "$schema"
+ fi
- if test -e $ovnnb_active_conf_file; then
- set "$@" --sync-from=`cat $ovnnb_active_conf_file`
- fi
+ set ovsdb-server
+ set "$@" $log --log-file=$logfile
+ set "$@" --remote=punix:$sock --pidfile=$pid
+ set "$@" --unixctl=ovnnb_db.ctl
- $@ $DB_NB_FILE
- ovn-nbctl init
+ if test X"$detach" != Xno; then
+ set "$@" --detach --monitor
+ else
+ set exec "$@"
fi
-}
-start_sb_ovsdb() {
- # Check and eventually start ovsdb-server for Southbound DB
- if ! pidfile_is_running $DB_SB_PID; then
- upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null
+ if test $mode != cluster; then
+ set "$@" --remote=db:$schema_name,$table_name,connections
+ set "$@" --private-key=db:$schema_name,SSL,private_key
+ set "$@" --certificate=db:$schema_name,SSL,certificate
+ set "$@" --ca-cert=db:$schema_name,SSL,ca_cert
+ set "$@" --ssl-protocols=db:$schema_name,SSL,ssl_protocols
+ set "$@" --ssl-ciphers=db:$schema_name,SSL,ssl_ciphers
+ fi
- set ovsdb-server
+ if test X"$create_insecure_remote" = Xyes; then
+ set "$@" --remote=ptcp:$port:$addr
+ fi
- if test X"$DB_SB_DETACH" != Xno; then
- set "$@" --detach --monitor
- else
- set exec "$@"
- fi
+ if test $mode = active_passive; then
+ set "$@" --sync-from=`cat $active_conf_file`
+ fi
- set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE
- set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID
- set "$@" --remote=db:OVN_Southbound,SB_Global,connections
- set "$@" --unixctl=ovnsb_db.ctl
- set "$@" --private-key=db:OVN_Southbound,SSL,private_key
- set "$@" --certificate=db:OVN_Southbound,SSL,certificate
- set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert
- set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols
- set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers
-
- if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then
- set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR
- fi
+ "$@" "$file"
- if test ! -z "$DB_SB_SYNC_FROM_ADDR"; then
- echo "$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:$DB_SB_SYNC_FROM_PORT" > $ovnsb_active_conf_file
- fi
-
- if test -e $ovnsb_active_conf_file; then
- set "$@" --sync-from=`cat $ovnsb_active_conf_file`
- fi
+ # Initialize the database if it's running standalone,
+ # active-passive, or is the first server in a cluster.
+ if test -z "$cluster_remote_addr"; then
+ ovn-nbctl init
+ fi
- $@ $DB_SB_FILE
- ovn-sbctl init
+ if test $mode = cluster; then
+ upgrade_cluster "$schema" "unix:$sock"
fi
}
start_ovsdb () {
- start_nb_ovsdb
- start_sb_ovsdb
+ start_ovsdb__ NB nb OVN_Northbound NB_Global
+ start_ovsdb__ SB sb OVN_Southbound SB_Global
}
sync_status() {
@@ -406,6 +412,11 @@ set_defaults () {
DB_NB_DETACH="yes"
DB_SB_DETACH="yes"
+
+ DB_NB_CLUSTER_LOCAL_ADDR=""
+ DB_NB_CLUSTER_REMOTE_ADDR=""
+ DB_SB_CLUSTER_LOCAL_ADDR=""
+ DB_SB_CLUSTER_REMOTE_ADDR=""
}
set_option () {
@@ -494,6 +505,14 @@ File location options:
--db-sb-sync-from-port=ADDR OVN Southbound active db tcp port (default: $DB_SB_SYNC_FROM_PORT)
--db-sb-sync-from-proto=PROTO OVN Southbound active db transport (default: $DB_SB_SYNC_FROM_PROTO)
--db-sb-create-insecure-remote=yes|no Create ptcp OVN Southbound remote (default: $DB_SB_CREATE_INSECURE_REMOTE)
+ --db-nb-cluster-local-addr=ADDR OVN_Northbound cluster local address \
+ (default: $DB_NB_CLUSTER_LOCAL_ADDR)
+ --db-nb-cluster-remote-addr=ADDR OVN_Northbound cluster remote address \
+ (default: $DB_NB_CLUSTER_REMOTE_ADDR)
+ --db-sb-cluster-local-addr=ADDR OVN_Southbound cluster local address \
+ (default: $DB_SB_CLUSTER_LOCAL_ADDR)
+ --db-sb-cluster-remote-addr=ADDR OVN_Southbound cluster remote address \
+ (default: $DB_SB_CLUSTER_REMOTE_ADDR)
Default directories with "configure" option and environment variable override:
logs: /usr/local/var/log/openvswitch (--with-logdir, OVS_LOGDIR)
diff --git a/utilities/ovs-lib.in b/utilities/ovs-lib.in
index 4b08f0f53c7b..cb4d20203992 100644
--- a/utilities/ovs-lib.in
+++ b/utilities/ovs-lib.in
@@ -401,6 +401,14 @@ create_db () {
action "Creating empty database $DB_FILE" ovsdb_tool create "$DB_FILE" "$DB_SCHEMA"
}
+backup_db () {
+ # Back up the old version.
+ version=`ovsdb_tool db-version "$DB_FILE"`
+ cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
+ backup=$DB_FILE.backup$version-$cksum
+ action "Backing up database to $backup" cp "$DB_FILE" "$backup" || return 1
+}
+
upgrade_db () {
DB_FILE="$1"
DB_SCHEMA="$2"
@@ -411,11 +419,7 @@ upgrade_db () {
install_dir `dirname $DB_FILE`
create_db "$DB_FILE" "$DB_SCHEMA"
elif test X"`ovsdb_tool needs-conversion "$DB_FILE" "$DB_SCHEMA"`" = Xyes; then
- # Back up the old version.
- version=`ovsdb_tool db-version "$DB_FILE"`
- cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
- backup=$DB_FILE.backup$version-$cksum
- action "Backing up database to $backup" cp "$DB_FILE" "$backup" || return 1
+ backup_db || return 1
# Compact database. This is important if the old schema did not enable
# garbage collection (i.e. if it did not have any tables with "isRoot":
@@ -443,6 +447,54 @@ upgrade_db () {
fi
}
+upgrade_cluster () {
+ local DB_SCHEMA=$1 DB_SERVER=$2
+ local schema_name=$(schema-name $1) || return 1
+
+ action "Waiting for $schema_name to come up" ovsdb-client -T 30 wait "$DB_SERVER" "$schema_name" connected || return $?
+ local db_version=$(ovsdb-client -T 10 get-schema-version "$DB_SERVER" "$schema_name") || return $?
+ local target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") || return $?
+
+ if ovsdb-tool compare-versions "$db_version" == "$target_version"; then
+ :
+ elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then
+ log_warning_msg "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?"
+ else
+ action "Upgrading database $schema_name from schema version $db_version to $target_version" ovsdb-client -T 30 convert "$DB_SERVER" "$DB_SCHEMA"
+ fi
+}
+
+create_cluster () {
+ DB_FILE="$1"
+ DB_SCHEMA="$2"
+ LOCAL_ADDR="$3"
+
+ if test ! -e "$DB_FILE"; then
+ action "Creating cluster database $DB_FILE" ovsdb_tool create-cluster "$DB_FILE" "$DB_SCHEMA" "$LOCAL_ADDR"
+ elif ovsdb_tool db-is-standalone "$DB_FILE"; then
+ # Convert standalone database to clustered.
+ backup_db || return 1
+ action "Creating cluster database $DB_FILE from existing one" \
+ ovsdb_tool create-cluster "$DB_FILE" "$backup" "$LOCAL_ADDR"
+ fi
+}
+
+join_cluster() {
+ DB_FILE="$1"
+ SCHEMA_NAME="$2"
+ LOCAL_ADDR="$3"
+ REMOTE_ADDR="$4"
+
+ if test ! -e "$DB_FILE"; then
+ ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR" "$REMOTE_ADDR"
+ elif ovsdb_tool db-is-standalone "$DB_FILE"; then
+ # Backup standalone database and join cluster.
+ backup_db || return 1
+ action "Joining $DB_FILE to cluster" \
+ ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR"
+ fi
+}
+
ovs_vsctl () {
ovs-vsctl --no-wait "$@"
}
More information about the dev
mailing list