[ovs-dev] [PATCH v2] ovn-ctl: Support starting clustered OVN dbs
Numan Siddique
nusiddiq at redhat.com
Wed Apr 4 00:41:54 UTC 2018
On Wed, Apr 4, 2018, 4:42 AM Ben Pfaff <blp at ovn.org> wrote:
> On Wed, Apr 04, 2018 at 12:24:27AM +0530, nusiddiq at redhat.com wrote:
> > From: Numan Siddique <nusiddiq at redhat.com>
> >
> > This patch adds the options to start clustered OVN db servers in ovn-ctl.
> > To support this, following options are added -
> '--db-(nb/sb)-cluster-local-addr',
> > '--db-(nb/sb)-cluster-local-port', '--db-(nb/sb)-cluster-local-proto',
> > '--db-(nb/sb)-cluster-remote-addr', '--db-(nb/sb)-cluster-remote-port'
> and
> > '--db-(nb/sb)-cluster-remote-proto'.
> >
> > If only '--db-(nb/sb)-cluster-local-addr' is defined, then clustered db
> is created
> > (using ovsdb-tool create-cluster). If both are defined, then the db is
> added to
> > the cluster (using ovsdb-tool join-cluster)
> >
> > This patch also adds the support to configure ovn-northd to point to all
> the servers
> > in the cluster using the options - '--ovn-northd-nb-db' and
> 'ovn-northd-sb-db'.
> >
> > Presently this patch doesn't handle the schema update scenario when
> restarting the
> > clustered ovsdb-servers. This will be handled in a separate patch.
> >
> > The initial versions of these commands are tested by Aliasgar <aginwala <
> aginwala at asu.edu>
> > and the discussion on this can be found here -
> >
> https://mail.openvswitch.org/pipermail/ovs-discuss/2018-March/046470.html
> >
> > (There are 4 checkpatch warnings 'Line length is >79-characters long' in
> ovn-ctl.8.xml
> > which I couldn't resolve without losing proper rendering when "man
> ovn-ctl" is run.)
> >
> > Signed-off-by: Numan Siddique <nusiddiq at redhat.com>
>
> Hi Numan, thanks for working on this.
>
> I'm pasting a version of the patch that I spent some time on myself. I
> think that your version has better documentation and it may have other
> enhancement. My version merges the code for starting the nb and sb
> database servers into a single function and it adds support for
> upgrading the cluster, but on the other hand I haven't tested it at all.
>
> Do you think you could try to produce a version that merges all of these
> benefits?
>
Thats great. Yeah sure. I will test it out and update the patch with v3.
Thanks
Numan
> Thanks,
>
> Ben.
>
> --8<--------------------------cut here-------------------------->8--
>
> diff --git a/ovn/utilities/ovn-ctl b/ovn/utilities/ovn-ctl
> index dc0c261590fb..029aa081c7ba 100755
> --- a/ovn/utilities/ovn-ctl
> +++ b/ovn/utilities/ovn-ctl
> @@ -93,89 +93,95 @@ promote_ovnsb() {
> ovs-appctl -t $rundir/ovnsb_db.ctl
> ovsdb-server/disconnect-active-ovsdb-server
> }
>
> -start_nb_ovsdb() {
> - # Check and eventually start ovsdb-server for Northbound DB
> - if ! pidfile_is_running $DB_NB_PID; then
> - upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null
> +start_ovsdb__() {
> + local DB=$1 db=$2 schema_name=$3 table_name=$4
> + eval local pid=\$DB_${DB}_PID
> + eval local cluster_local_addr=\$DB_${DB}_CLUSTER_LOCAL_ADDR
> + eval local cluster_remote_addr=\$DB_${DB}_CLUSTER_REMOTE_ADDR
> + eval local sync_from_proto=\$DB_${DB}_SYNC_FROM_PROTO
> + eval local sync_from_addr=\$DB_${DB}_SYNC_FROM_ADDR
> + eval local sync_from_port=\$DB_${DB}_SYNC_FROM_PORT
> + eval local file=\$DB_${DB}_FILE
> + eval local schema=\$DB_${DB}_SCHEMA
> + eval local logfile=\$OVN_${DB}_LOGFILE
> + eval local log=\$OVN_${DB}_LOG
> + eval local sock=\$DB_${DB}_SOCK
> + eval local detach=\$DB_${DB}_DETACH
> + eval local create_insecure_remote=\$DB_${DB}_CREATE_INSECURE_REMOTE
> + eval local port=\$DB_${DB}_PORT
> + eval local addr=\$DB_${DB}_ADDR
> + eval local active_conf_file=\$ovn${db}_active_conf_file
> +
> + # Check and eventually start ovsdb-server for DB
> + if pidfile_is_running $pid; then
> + return
> + fi
>
> - set ovsdb-server
> + if test ! -z "$cluster_local_addr"; then
> + mode=cluster
> + elif test ! -z "$sync_from_addr"; then
> + mode=active_passive
> + echo "$sync_from_proto:$sync_from_addr:$sync_from_port" >
> $active_conf_file
> + else
> + mode=standalone
> + fi
>
> - if test X"$DB_NB_DETACH" != Xno; then
> - set "$@" --detach --monitor
> + if test $mode = cluster; then
> + local local=$cluster_local_addr
> + local remote=$cluster_remote_addr
> + if test -n "$remote"; then
> + join_cluster "$file" "$schema_name" "$local" "$remote"
> else
> - set exec "$@"
> - fi
> -
> - set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE
> - set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID
> - set "$@" --remote=db:OVN_Northbound,NB_Global,connections
> - set "$@" --unixctl=ovnnb_db.ctl
> - set "$@" --private-key=db:OVN_Northbound,SSL,private_key
> - set "$@" --certificate=db:OVN_Northbound,SSL,certificate
> - set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert
> - set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols
> - set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers
> -
> - if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then
> - set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR
> - fi
> -
> - if test ! -z "$DB_NB_SYNC_FROM_ADDR"; then
> - echo
> "$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:$DB_NB_SYNC_FROM_PORT" >
> $ovnnb_active_conf_file
> + create_cluster "$file" "$schema" "$local"
> fi
> + else
> + upgrade_db "$file" "$schema"
> + fi
>
> - if test -e $ovnnb_active_conf_file; then
> - set "$@" --sync-from=`cat $ovnnb_active_conf_file`
> - fi
> + set ovsdb-server
> + set "$@" $log --log-file=$logfile
> + set "$@" --remote=punix:$sock --pidfile=$pid
> + set "$@" --unixctl=ovnnb_db.ctl
>
> - $@ $DB_NB_FILE
> - ovn-nbctl init
> + if test X"$detach" != Xno; then
> + set "$@" --detach --monitor
> + else
> + set exec "$@"
> fi
> -}
>
> -start_sb_ovsdb() {
> - # Check and eventually start ovsdb-server for Southbound DB
> - if ! pidfile_is_running $DB_SB_PID; then
> - upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null
> + if test $mode != cluster; then
> + set "$@" --remote=db:$schema_name,$table_name,connections
> + set "$@" --private-key=db:$schema_name,SSL,private_key
> + set "$@" --certificate=db:$schema_name,SSL,certificate
> + set "$@" --ca-cert=db:$schema_name,SSL,ca_cert
> + set "$@" --ssl-protocols=db:$schema_name,SSL,ssl_protocols
> + set "$@" --ssl-ciphers=db:$schema_name,SSL,ssl_ciphers
> + fi
>
> - set ovsdb-server
> + if test X"$create_insecure_remote" = Xyes; then
> + set "$@" --remote=ptcp:$port:$addr
> + fi
>
> - if test X"$DB_SB_DETACH" != Xno; then
> - set "$@" --detach --monitor
> - else
> - set exec "$@"
> - fi
> + if test $mode = active_passive; then
> + set "$@" --sync-from=`cat $active_conf_file`
> + fi
>
> - set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE
> - set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID
> - set "$@" --remote=db:OVN_Southbound,SB_Global,connections
> - set "$@" --unixctl=ovnsb_db.ctl
> - set "$@" --private-key=db:OVN_Southbound,SSL,private_key
> - set "$@" --certificate=db:OVN_Southbound,SSL,certificate
> - set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert
> - set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols
> - set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers
> -
> - if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then
> - set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR
> - fi
> + "$@" "$file"
>
> - if test ! -z "$DB_SB_SYNC_FROM_ADDR"; then
> - echo
> "$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:$DB_SB_SYNC_FROM_PORT" >
> $ovnsb_active_conf_file
> - fi
> -
> - if test -e $ovnsb_active_conf_file; then
> - set "$@" --sync-from=`cat $ovnsb_active_conf_file`
> - fi
> + # Initialize the database if it's running standalone,
> + # active-passive, or is the first server in a cluster.
> + if test -z "$cluster_remote_addr"; then
> + ovn-nbctl init
> + fi
>
> - $@ $DB_SB_FILE
> - ovn-sbctl init
> + if test $mode = cluster; then
> + upgrade_cluster "$schema" "unix:$sock"
> fi
> }
>
> start_ovsdb () {
> - start_nb_ovsdb
> - start_sb_ovsdb
> + start_ovsdb__ NB nb OVN_Northbound NB_Global
> + start_ovsdb__ SB sb OVN_Southbound SB_Global
> }
>
> sync_status() {
> @@ -406,6 +412,11 @@ set_defaults () {
>
> DB_NB_DETACH="yes"
> DB_SB_DETACH="yes"
> +
> + DB_NB_CLUSTER_LOCAL_ADDR=""
> + DB_NB_CLUSTER_REMOTE_ADDR=""
> + DB_SB_CLUSTER_LOCAL_ADDR=""
> + DB_SB_CLUSTER_REMOTE_ADDR=""
> }
>
> set_option () {
> @@ -494,6 +505,14 @@ File location options:
> --db-sb-sync-from-port=ADDR OVN Southbound active db tcp port (default:
> $DB_SB_SYNC_FROM_PORT)
> --db-sb-sync-from-proto=PROTO OVN Southbound active db transport
> (default: $DB_SB_SYNC_FROM_PROTO)
> --db-sb-create-insecure-remote=yes|no Create ptcp OVN Southbound remote
> (default: $DB_SB_CREATE_INSECURE_REMOTE)
> + --db-nb-cluster-local-addr=ADDR OVN_Northbound cluster local address \
> + (default: $DB_NB_CLUSTER_LOCAL_ADDR)
> + --db-nb-cluster-remote-addr=ADDR OVN_Northbound cluster remote address \
> + (default: $DB_NB_CLUSTER_REMOTE_ADDR)
> + --db-sb-cluster-local-addr=ADDR OVN_Southbound cluster local address \
> + (default: $DB_SB_CLUSTER_LOCAL_ADDR)
> + --db-sb-cluster-remote-addr=ADDR OVN_Southbound cluster remote address \
> + (default: $DB_SB_CLUSTER_REMOTE_ADDR)
>
> Default directories with "configure" option and environment variable
> override:
> logs: /usr/local/var/log/openvswitch (--with-logdir, OVS_LOGDIR)
> diff --git a/utilities/ovs-lib.in b/utilities/ovs-lib.in
> index 4b08f0f53c7b..cb4d20203992 100644
> --- a/utilities/ovs-lib.in
> +++ b/utilities/ovs-lib.in
> @@ -401,6 +401,14 @@ create_db () {
> action "Creating empty database $DB_FILE" ovsdb_tool create
> "$DB_FILE" "$DB_SCHEMA"
> }
>
> +backup_db () {
> + # Back up the old version.
> + version=`ovsdb_tool db-version "$DB_FILE"`
> + cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
> + backup=$DB_FILE.backup$version-$cksum
> + action "Backing up database to $backup" cp "$DB_FILE" "$backup" ||
> return 1
> +}
> +
> upgrade_db () {
> DB_FILE="$1"
> DB_SCHEMA="$2"
> @@ -411,11 +419,7 @@ upgrade_db () {
> install_dir `dirname $DB_FILE`
> create_db "$DB_FILE" "$DB_SCHEMA"
> elif test X"`ovsdb_tool needs-conversion "$DB_FILE" "$DB_SCHEMA"`" =
> Xyes; then
> - # Back up the old version.
> - version=`ovsdb_tool db-version "$DB_FILE"`
> - cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
> - backup=$DB_FILE.backup$version-$cksum
> - action "Backing up database to $backup" cp "$DB_FILE" "$backup"
> || return 1
> + backup_db || return 1
>
> # Compact database. This is important if the old schema did not
> enable
> # garbage collection (i.e. if it did not have any tables with
> "isRoot":
> @@ -443,6 +447,54 @@ upgrade_db () {
> fi
> }
>
> +upgrade_cluster () {
> + local DB_SCHEMA=$1 DB_SERVER=$2
> + local schema_name=$(schema-name $1) || return 1
> +
> + action "Waiting for $schema_name to come up" ovsdb-client -T 30 wait
> "$DB_SERVER" "$schema_name" connected || return $?
> + local db_version=$(ovsdb-client -T 10 get-schema-version "$DB_SERVER"
> "$schema_name") || return $?
> + local target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") ||
> return $?
> +
> + if ovsdb-tool compare-versions "$db_version" == "$target_version";
> then
> + :
> + elif ovsdb-tool compare-versions "$db_version" ">" "$target_version";
> then
> + log_warning_msg "Database $schema_name has newer schema version
> ($db_version) than our local schema ($target_version), possibly an upgrade
> is partially complete?"
> + else
> + action "Upgrading database $schema_name from schema version
> $db_version to $target_version" ovsdb-client -T 30 convert "$DB_SERVER"
> "$DB_SCHEMA"
> + fi
> +}
> +
> +create_cluster () {
> + DB_FILE="$1"
> + DB_SCHEMA="$2"
> + LOCAL_ADDR="$3"
> +
> + if test ! -e "$DB_FILE"; then
> + action "Creating cluster database $DB_FILE" ovsdb_tool
> create-cluster "$DB_FILE" "$DB_SCHEMA" "$LOCAL_ADDR"
> + elif ovsdb_tool db-is-standalone "$DB_FILE"; then
> + # Convert standalone database to clustered.
> + backup_db || return 1
> + action "Creating cluster database $DB_FILE from existing one" \
> + ovsdb_tool create-cluster "$DB_FILE" "$backup"
> "$LOCAL_ADDR"
> + fi
> +}
> +
> +join_cluster() {
> + DB_FILE="$1"
> + SCHEMA_NAME="$2"
> + LOCAL_ADDR="$3"
> + REMOTE_ADDR="$4"
> +
> + if test ! -e "$DB_FILE"; then
> + ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR"
> "$REMOTE_ADDR"
> + elif ovsdb_tool db-is-standalone "$DB_FILE"; then
> + # Backup standalone database and join cluster.
> + backup_db || return 1
> + action "Joining $DB_FILE to cluster" \
> + ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME"
> "$LOCAL_ADDR"
> + fi
> +}
> +
> ovs_vsctl () {
> ovs-vsctl --no-wait "$@"
> }
>
More information about the dev
mailing list