[ovs-dev] [PATCH v5] ovndb-servers.ocf: add LB support for managing ovndb cluster:

aginwala amginwal at gmail.com
Sat Jun 2 16:11:56 UTC 2018


using pacemaker so that controllers can be placed in different fault domains.
More background about the discussions can be found on:
https://mail.openvswitch.org/pipermail/ovs-discuss/2018-May/046770.html

Signed-off-by: aginwala <aginwala at ebay.com>
---
 Documentation/topics/integration.rst | 34 +++++++++++++---
 ovn/utilities/ovndb-servers.ocf      | 75 +++++++++++++++++++++++++++---------
 2 files changed, 85 insertions(+), 24 deletions(-)

diff --git a/Documentation/topics/integration.rst b/Documentation/topics/integration.rst
index 0447faf..5d2d3e4 100644
--- a/Documentation/topics/integration.rst
+++ b/Documentation/topics/integration.rst
@@ -243,12 +243,14 @@ node at which the active server is run, it is not efficient to instruct all the
 ovn-controllers and the ovn-northd to listen to the latest active server's
 ip-address.
 
-This problem can be solved by using a native ocf resource agent
-``ocf:heartbeat:IPaddr2``. The IPAddr2 resource agent is just a resource with
-an ip-address. When we colocate this resource with the active server, pacemaker
-will enable the active server to be connected with a single ip-address all the
-time. This is the ip-address that needs to be given as the parameter while
-creating the `ovndb_servers` resource.
+This problem can be solved by two ways:
+
+1. By using a native ocf resource agent ``ocf:heartbeat:IPaddr2``.
+The IPAddr2 resource agent is just a resource with an ip-address. When we
+colocate this resource with the active server, pacemaker will enable the
+active server to be connected with a single ip-address all the time. This is
+the ip-address that needs to be given as the parameter while creating the
+`ovndb_servers` resource.
 
 Use the following command to create the IPAddr2 resource and colocate it
 with the active server::
@@ -258,3 +260,23 @@ with the active server::
     $ pcs constraint order promote ovndb_servers-master then VirtualIP
     $ pcs constraint colocation add VirtualIP with master ovndb_servers-master \
         score=INFINITY
+
+
+2. Using load balancer vip ip as a master_ip.
+In order to use this feature, one needs to use listen_on_master_ip_only to no.
+Current code for load balancer have been tested to work with tcp protocol
+and needs to be tested/enchanced for ssl. Using load balancer, standby nodes
+will not listen on nb and sb db ports so that load balancer will always
+communicate to the active node and all the traffic will be sent to active node only.
+Standby will continue to sync using LB VIP IP in this case.
+
+Use the following command to create pcs resource using LB VIP IP::
+
+    $ pcs resource create ovndb_servers ocf:ovn:ovndb-servers \
+         master_ip="<load_balance_vip_ip>" \
+         listen_on_master_ip_only="no" \
+         ovn_ctl=<path of the ovn-ctl script> \
+         op monitor interval="10s" \
+         op monitor role=Master interval="15s"
+    $ pcs resource master ovndb_servers-master ovndb_servers \
+        meta notify="true"
diff --git a/ovn/utilities/ovndb-servers.ocf b/ovn/utilities/ovndb-servers.ocf
index 23dc700..9391b89 100755
--- a/ovn/utilities/ovndb-servers.ocf
+++ b/ovn/utilities/ovndb-servers.ocf
@@ -9,6 +9,7 @@
 : ${SB_MASTER_PROTO_DEFAULT="tcp"}
 : ${MANAGE_NORTHD_DEFAULT="no"}
 : ${INACTIVE_PROBE_DEFAULT="5000"}
+: ${LISTEN_ON_MASTER_IP_ONLY_DEFAULT="yes"}
 
 CRM_MASTER="${HA_SBIN_DIR}/crm_master -l reboot"
 CRM_ATTR_REPL_INFO="${HA_SBIN_DIR}/crm_attribute --type crm_config --name OVN_REPL_INFO -s ovn_ovsdb_master_server"
@@ -21,6 +22,10 @@ SB_MASTER_PROTO=${OCF_RESKEY_sb_master_protocol:-${SB_MASTER_PROTO_DEFAULT}}
 MANAGE_NORTHD=${OCF_RESKEY_manage_northd:-${MANAGE_NORTHD_DEFAULT}}
 INACTIVE_PROBE=${OCF_RESKEY_inactive_probe_interval:-${INACTIVE_PROBE_DEFAULT}}
 
+# In order for pacemaker to work with LB, we can set LISTEN_ON_MASTER_IP_ONLY
+# to false and pass LB vip IP while creating pcs resource.
+LISTEN_ON_MASTER_IP_ONLY=${OCF_RESKEY_listen_on_master_ip_only:-${LISTEN_ON_MASTER_IP_ONLY_DEFAULT}}
+
 # Invalid IP address is an address that can never exist in the network, as
 # mentioned in rfc-5737. The ovsdb servers connects to this IP address till
 # a master is promoted and the IPAddr2 resource is started.
@@ -117,6 +122,16 @@ ovsdb_server_metadata() {
   <content type="string" />
   </parameter>
 
+  <parameter name="listen_on_master_ip_only" unique="1">
+  <longdesc lang="en">
+  If set to yes, the OVNDBs will listen on master IP. Otherwise, it will
+  listen on 0.0.0.0. Set to yes when using pacemaker managed vip resource
+  as MASTER_IP; set to no when using external LB VIP.
+  </longdesc>
+  <shortdesc lang="en">Listen on master IP or 0.0.0.0</shortdesc>
+  <content type="string" />
+  </parameter>
+
   </parameters>
 
   <actions>
@@ -157,22 +172,25 @@ ovsdb_server_notify() {
             ${OVN_CTL} --ovn-manage-ovsdb=no start_northd
         fi
 
-        conn=`ovn-nbctl get NB_global . connections`
-        if [ "$conn" == "[]" ]
-        then
-            ovn-nbctl -- --id=@conn_uuid create Connection \
+        # Not needed while listening on 0.0.0.0 as we do not want to allow
+        # local binds. However, it is needed if vip ip is binded to nodes.
+        if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xyes ]; then
+            conn=`ovn-nbctl get NB_global . connections`
+            if [ "$conn" == "[]" ]
+            then
+                ovn-nbctl -- --id=@conn_uuid create Connection \
 target="p${NB_MASTER_PROTO}\:${NB_MASTER_PORT}\:${MASTER_IP}" \
 inactivity_probe=$INACTIVE_PROBE -- set NB_Global . connections=@conn_uuid
-        fi
+            fi
 
-        conn=`ovn-sbctl get SB_global . connections`
-        if [ "$conn" == "[]" ]
-        then
-            ovn-sbctl -- --id=@conn_uuid create Connection \
+            conn=`ovn-sbctl get SB_global . connections`
+            if [ "$conn" == "[]" ]
+            then
+                ovn-sbctl -- --id=@conn_uuid create Connection \
 target="p${SB_MASTER_PROTO}\:${SB_MASTER_PORT}\:${MASTER_IP}" \
 inactivity_probe=$INACTIVE_PROBE -- set SB_Global . connections=@conn_uuid
+            fi
         fi
-
     else
         if [ "$MANAGE_NORTHD" = "yes" ]; then
             # Stop ovn-northd service. Set --ovn-manage-ovsdb=no so that
@@ -295,15 +313,13 @@ ovsdb_server_start() {
 
     set ${OVN_CTL}
 
-    set $@ --db-nb-addr=${MASTER_IP} --db-nb-port=${NB_MASTER_PORT}
-    set $@ --db-sb-addr=${MASTER_IP} --db-sb-port=${SB_MASTER_PORT}
-
-    if [ "x${NB_MASTER_PROTO}" = xtcp ]; then
-        set $@ --db-nb-create-insecure-remote=yes
-    fi
+    if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xno ]; then
+        set $@ --db-nb-port=${NB_MASTER_PORT}
+        set $@ --db-sb-port=${SB_MASTER_PORT}
 
-    if [ "x${SB_MASTER_PROTO}" = xtcp ]; then
-        set $@ --db-sb-create-insecure-remote=yes
+    else
+       set $@ --db-nb-addr=${MASTER_IP} --db-nb-port=${NB_MASTER_PORT}
+       set $@ --db-sb-addr=${MASTER_IP} --db-sb-port=${SB_MASTER_PORT}
     fi
 
     if [ "x${present_master}" = x ]; then
@@ -313,15 +329,33 @@ ovsdb_server_start() {
         # Force all copies to come up as slaves by pointing them into
         # space and let pacemaker pick one to promote:
         #
+        if [ "x${NB_MASTER_PROTO}" = xtcp ]; then
+            set $@ --db-nb-create-insecure-remote=yes
+        fi
+
+        if [ "x${SB_MASTER_PROTO}" = xtcp ]; then
+            set $@ --db-sb-create-insecure-remote=yes
+        fi
         set $@ --db-nb-sync-from-addr=${INVALID_IP_ADDRESS} --db-sb-sync-from-addr=${INVALID_IP_ADDRESS}
 
     elif [ ${present_master} != ${host_name} ]; then
+        # TODO: for using LB vip, need to test for ssl.
+        if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xyes ]; then
+            if [ "x${NB_MASTER_PROTO}" = xtcp ]; then
+                set $@ --db-nb-create-insecure-remote=yes
+            fi
+
+            if [ "x${SB_MASTER_PROTO}" = xtcp ]; then
+                set $@ --db-sb-create-insecure-remote=yes
+            fi
+        fi
         # An existing master is active, connect to it
         set $@ --db-nb-sync-from-addr=${MASTER_IP} --db-sb-sync-from-addr=${MASTER_IP}
         set $@ --db-nb-sync-from-port=${NB_MASTER_PORT}
         set $@ --db-nb-sync-from-proto=${NB_MASTER_PROTO}
         set $@ --db-sb-sync-from-port=${SB_MASTER_PORT}
         set $@ --db-sb-sync-from-proto=${SB_MASTER_PROTO}
+
     fi
 
     $@ start_ovsdb
@@ -416,6 +450,11 @@ ovsdb_server_promote() {
             ;;
     esac
 
+    # Restart ovs so that new master can listen on tcp port
+    if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xno ]; then
+        ${OVN_CTL} stop_ovsdb
+        ovsdb_server_start
+    fi
     ${OVN_CTL} promote_ovnnb
     ${OVN_CTL} promote_ovnsb
 
-- 
1.9.1



More information about the dev mailing list