[ovs-dev] [PATCH v7 ovn] Add VXLAN support for non-VTEP datapath bindings

Numan Siddique numans at ovn.org
Thu Sep 17 07:14:57 UTC 2020


On Thu, Sep 17, 2020 at 8:52 AM Ihar Hrachyshka <ihrachys at redhat.com> wrote:

> Because of limited space in VXLAN VNI to pass over all three of -
> datapath id, ingress port, egress port - the implementation ignores
> ingress; and splits the remaining 24 bits of VNI into two chunks, 12
> bits each - one for datapath and one for egress port.
>
> This also bumps priority for ramp switch flows to 110 to prioritize
> them over regular non-ramp VXLAN flows.
>
> Limitations: because ingress port is not passed, ACLs that rely on it
> won't work with VXLAN; reduced number of networks and ports per
> network (max 4096 for both).
>
> NB consumers may use NB_Global options:max_tunid to determine maximum
> capacity for logical switches supported by the setup.
>
> Renamed MLF_RCV_FROM_VXLAN_BIT into MLF_RCV_FROM_RAMP_BIT to reflect
> the new use case.
>
> Added test scenarios that ping through VXLAN tunnel between two
> hypervisors added. Also max_tunid is validated.
>
> Signed-off-by: Ihar Hrachyshka <ihrachys at redhat.com>
>
>
Thank Ihar.

I did the below changes and applied this patch to master and also to the
newly created branch - branch-20.09.

******************************************************************************
diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c
index 92add9125..cfec6a2c8 100644
--- a/northd/ovn-northd.c
+++ b/northd/ovn-northd.c
@@ -1190,7 +1190,8 @@ join_datapaths(struct northd_context *ctx, struct
hmap *datapaths,
     }
 }

-static bool is_vxlan_mode(struct ovsdb_idl *ovnsb_idl)
+static bool
+is_vxlan_mode(struct ovsdb_idl *ovnsb_idl)
 {
     const struct sbrec_chassis *chassis;
     SBREC_CHASSIS_FOR_EACH (chassis, ovnsb_idl) {
@@ -1204,7 +1205,8 @@ static bool is_vxlan_mode(struct ovsdb_idl *ovnsb_idl)
 }

 static uint32_t
-get_ovn_max_dp_key_local(struct northd_context *ctx) {
+get_ovn_max_dp_key_local(struct northd_context *ctx)
+{
     if (is_vxlan_mode(ctx->ovnsb_idl)) {
         /* OVN_MAX_DP_GLOBAL_NUM doesn't apply for vxlan mode. */
         return OVN_MAX_DP_VXLAN_KEY;
diff --git a/ovn-architecture.7.xml b/ovn-architecture.7.xml
index 6d9057507..d00af36b4 100644
--- a/ovn-architecture.7.xml
+++ b/ovn-architecture.7.xml
@@ -2739,6 +2739,9 @@
     <li>
       ACLs matching against logical ingress port identifiers are not
supported.
     </li>
+    <li>
+      OVN interconnection feature is not supported.
+    </li>
   </ul>

   <p>
diff --git a/lib/ovn-util.h b/lib/ovn-util.h
index 58d41a582..d9aadcbc0 100644
--- a/lib/ovn-util.h
+++ b/lib/ovn-util.h
@@ -16,7 +16,6 @@
 #ifndef OVN_UTIL_H
 #define OVN_UTIL_H 1

-#include "lib/ovn-sb-idl.h"
 #include "lib/packets.h"
 #include "include/ovn/version.h"
******************************************************************************

Thanks
Numan

---
>
> Changes:
> - v2: run several dvr connectivity tests with vxlan tunnels.
> - v2: update ovn-architecture.7 documentation.
> - v3: added is_vxlan helper.
> - v4: reduce max tunid when vxlan is enabled in cluster.
> - v4: added options:max_tunid key for NB_Global.
> - v5: rebased.
> - v5: renamed MLF_RCV_FROM_VXLAN -> MLF_RCV_FROM_RAMP.
> - v5: simplify flows for vxlan non-ramp scenario, don't match against
>       tun_id.
> - v5: bumped priority for ramp flows to 110 to prioritize them over
>       regular non-ramp vxlan flows.
> - v6: added NEWS entry.
> - v7: return bool from is_vxlan_mode.
> - v7: document that IC is not supported when VXLAN enabled.
> - v7: move max_tunid description in ovn-nb(5) to a new "Read-Only"
>       section.
> ---
>  NEWS                         |    1 +
>  controller/physical.c        |   65 +-
>  include/ovn/logical-fields.h |   12 +-
>  lib/ovn-util.h               |    5 +
>  northd/ovn-northd.c          |   78 ++-
>  ovn-architecture.7.xml       |  110 +++-
>  ovn-nb.xml                   |   10 +
>  tests/ovn-macros.at          |    4 +-
>  tests/ovn.at                 | 1083 +++++++++++++++++-----------------
>  9 files changed, 725 insertions(+), 643 deletions(-)
>
> diff --git a/NEWS b/NEWS
> index dece5831c..ee5c2c393 100644
> --- a/NEWS
> +++ b/NEWS
> @@ -17,6 +17,7 @@ OVN v20.09.0 - xx xxx xxxx
>       this mechanism should update their code to use this new table.
>     - Added support for external ip based NAT. Now, besides the logical ip,
>       external ips will also decide if a packet will be NATed or not.
> +   - Added support for VXLAN encapsulation (not just for ramp/VTEP
> switches).
>
>  OVN v20.06.0
>  --------------------------
> diff --git a/controller/physical.c b/controller/physical.c
> index 535c77730..a7f3efd2f 100644
> --- a/controller/physical.c
> +++ b/controller/physical.c
> @@ -180,7 +180,8 @@ static void
>  put_encapsulation(enum mf_field_id mff_ovn_geneve,
>                    const struct chassis_tunnel *tun,
>                    const struct sbrec_datapath_binding *datapath,
> -                  uint16_t outport, struct ofpbuf *ofpacts)
> +                  uint16_t outport, bool is_ramp_switch,
> +                  struct ofpbuf *ofpacts)
>  {
>      if (tun->type == GENEVE) {
>          put_load(datapath->tunnel_key, MFF_TUN_ID, 0, 24, ofpacts);
> @@ -191,7 +192,12 @@ put_encapsulation(enum mf_field_id mff_ovn_geneve,
>                   MFF_TUN_ID, 0, 64, ofpacts);
>          put_move(MFF_LOG_INPORT, 0, MFF_TUN_ID, 40, 15, ofpacts);
>      } else if (tun->type == VXLAN) {
> -        put_load(datapath->tunnel_key, MFF_TUN_ID, 0, 24, ofpacts);
> +        uint64_t vni = datapath->tunnel_key;
> +        if (!is_ramp_switch) {
> +            /* Only some bits are used for regular tunnels. */
> +            vni |= (uint64_t) outport << 12;
> +        }
> +        put_load(vni, MFF_TUN_ID, 0, 24, ofpacts);
>      } else {
>          OVS_NOT_REACHED();
>      }
> @@ -323,8 +329,9 @@ put_remote_port_redirect_overlay(const struct
>          if (!rem_tun) {
>              return;
>          }
> -        put_encapsulation(mff_ovn_geneve, tun, binding->datapath,
> -                          port_key, ofpacts_p);
> +        put_encapsulation(mff_ovn_geneve, tun, binding->datapath,
> port_key,
> +                          !strcmp(binding->type, "vtep"),
> +                          ofpacts_p);
>          /* Output to tunnel. */
>          ofpact_put_OUTPUT(ofpacts_p)->port = rem_tun->ofport;
>      } else {
> @@ -360,8 +367,9 @@ put_remote_port_redirect_overlay(const struct
>              return;
>          }
>
> -        put_encapsulation(mff_ovn_geneve, tun, binding->datapath,
> -                          port_key, ofpacts_p);
> +        put_encapsulation(mff_ovn_geneve, tun, binding->datapath,
> port_key,
> +                          !strcmp(binding->type, "vtep"),
> +                          ofpacts_p);
>
>          /* Output to tunnels with active/backup */
>          struct ofpact_bundle *bundle = ofpact_put_BUNDLE(ofpacts_p);
> @@ -1370,7 +1378,7 @@ consider_mc_group(enum mf_field_id mff_ovn_geneve,
>
>              if (!prev || tun->type != prev->type) {
>                  put_encapsulation(mff_ovn_geneve, tun, mc->datapath,
> -                                  mc->tunnel_key, &remote_ofpacts);
> +                                  mc->tunnel_key, true, &remote_ofpacts);
>                  prev = tun;
>              }
>              ofpact_put_OUTPUT(&remote_ofpacts)->port = tun->ofport;
> @@ -1615,11 +1623,12 @@ physical_run(struct physical_ctx *p_ctx,
>       * Process packets that arrive from a remote hypervisor (by matching
>       * on tunnel in_port). */
>
> -    /* Add flows for Geneve and STT encapsulations.  These
> -     * encapsulations have metadata about the ingress and egress logical
> -     * ports.  We set MFF_LOG_DATAPATH, MFF_LOG_INPORT, and
> -     * MFF_LOG_OUTPORT from the tunnel key data, then resubmit to table
> -     * 33 to handle packets to the local hypervisor. */
> +    /* Add flows for Geneve, STT and VXLAN encapsulations.  Geneve and STT
> +     * encapsulations have metadata about the ingress and egress logical
> ports.
> +     * VXLAN encapsulations have metadata about the egress logical port
> only.
> +     * We set MFF_LOG_DATAPATH, MFF_LOG_INPORT, and MFF_LOG_OUTPORT from
> the
> +     * tunnel key data where possible, then resubmit to table 33 to handle
> +     * packets to the local hypervisor. */
>      HMAP_FOR_EACH (tun, hmap_node, &tunnels) {
>          struct match match = MATCH_CATCHALL_INITIALIZER;
>          match_set_in_port(&match, tun->ofport);
> @@ -1636,8 +1645,10 @@ physical_run(struct physical_ctx *p_ctx,
>              put_move(MFF_TUN_ID, 24, MFF_LOG_OUTPORT,  0, 16, &ofpacts);
>              put_move(MFF_TUN_ID,  0, MFF_LOG_DATAPATH, 0, 24, &ofpacts);
>          } else if (tun->type == VXLAN) {
> -            /* We'll handle VXLAN later. */
> -            continue;
> +            /* Add flows for non-VTEP tunnels. Split VNI into two 12-bit
> +             * sections and use them for datapath and outport IDs. */
> +            put_move(MFF_TUN_ID, 12, MFF_LOG_OUTPORT,  0, 12, &ofpacts);
> +            put_move(MFF_TUN_ID, 0, MFF_LOG_DATAPATH, 0, 12, &ofpacts);
>          } else {
>              OVS_NOT_REACHED();
>          }
> @@ -1648,11 +1659,7 @@ physical_run(struct physical_ctx *p_ctx,
>                          &ofpacts, hc_uuid);
>      }
>
> -    /* Add flows for VXLAN encapsulations.  Due to the limited amount of
> -     * metadata, we only support VXLAN for connections to gateways.  The
> -     * VNI is used to populate MFF_LOG_DATAPATH.  The gateway's logical
> -     * port is set to MFF_LOG_INPORT.  Then the packet is resubmitted to
> -     * table 16 to determine the logical egress port. */
> +    /* Handle ramp switch encapsulations. */
>      HMAP_FOR_EACH (tun, hmap_node, &tunnels) {
>          if (tun->type != VXLAN) {
>              continue;
> @@ -1660,7 +1667,9 @@ physical_run(struct physical_ctx *p_ctx,
>
>          SBREC_PORT_BINDING_TABLE_FOR_EACH (binding,
>                                             p_ctx->port_binding_table) {
> -            struct match match = MATCH_CATCHALL_INITIALIZER;
> +            if (strcmp(binding->type, "vtep")) {
> +                continue;
> +            }
>
>              if (!binding->chassis ||
>                  !encaps_tunnel_id_match(tun->chassis_id,
> @@ -1668,18 +1677,24 @@ physical_run(struct physical_ctx *p_ctx,
>                  continue;
>              }
>
> +            struct match match = MATCH_CATCHALL_INITIALIZER;
>              match_set_in_port(&match, tun->ofport);
> +            ofpbuf_clear(&ofpacts);
> +
> +            /* Add flows for ramp switches.  The VNI is used to populate
> +             * MFF_LOG_DATAPATH.  The gateway's logical port is set to
> +             * MFF_LOG_INPORT.  Then the packet is resubmitted to table 8
> +             * to determine the logical egress port. */
>              match_set_tun_id(&match,
> htonll(binding->datapath->tunnel_key));
>
> -            ofpbuf_clear(&ofpacts);
>              put_move(MFF_TUN_ID, 0,  MFF_LOG_DATAPATH, 0, 24, &ofpacts);
>              put_load(binding->tunnel_key, MFF_LOG_INPORT, 0, 15,
> &ofpacts);
> -            /* For packets received from a vxlan tunnel, set a flag to
> that
> +            /* For packets received from a ramp tunnel, set a flag to that
>               * effect. */
> -            put_load(1, MFF_LOG_FLAGS, MLF_RCV_FROM_VXLAN_BIT, 1,
> &ofpacts);
> +            put_load(1, MFF_LOG_FLAGS, MLF_RCV_FROM_RAMP_BIT, 1,
> &ofpacts);
>              put_resubmit(OFTABLE_LOG_INGRESS_PIPELINE, &ofpacts);
>
> -            ofctrl_add_flow(flow_table, OFTABLE_PHY_TO_LOG, 100,
> +            ofctrl_add_flow(flow_table, OFTABLE_PHY_TO_LOG, 110,
>                              binding->header_.uuid.parts[0],
>                              &match, &ofpacts, hc_uuid);
>          }
> @@ -1696,7 +1711,7 @@ physical_run(struct physical_ctx *p_ctx,
>      struct match match;
>      match_init_catchall(&match);
>      match_set_reg_masked(&match, MFF_LOG_FLAGS - MFF_REG0,
> -                         MLF_RCV_FROM_VXLAN, MLF_RCV_FROM_VXLAN);
> +                         MLF_RCV_FROM_RAMP, MLF_RCV_FROM_RAMP);
>
>      /* Resubmit to table 33. */
>      ofpbuf_clear(&ofpacts);
> diff --git a/include/ovn/logical-fields.h b/include/ovn/logical-fields.h
> index 61d17d14f..ac6f2f909 100644
> --- a/include/ovn/logical-fields.h
> +++ b/include/ovn/logical-fields.h
> @@ -51,7 +51,7 @@ void ovn_init_symtab(struct shash *symtab);
>  /* MFF_LOG_FLAGS_REG bit assignments */
>  enum mff_log_flags_bits {
>      MLF_ALLOW_LOOPBACK_BIT = 0,
> -    MLF_RCV_FROM_VXLAN_BIT = 1,
> +    MLF_RCV_FROM_RAMP_BIT = 1,
>      MLF_FORCE_SNAT_FOR_DNAT_BIT = 2,
>      MLF_FORCE_SNAT_FOR_LB_BIT = 3,
>      MLF_LOCAL_ONLY_BIT = 4,
> @@ -64,11 +64,11 @@ enum mff_log_flags {
>      /* Allow outputting back to inport. */
>      MLF_ALLOW_LOOPBACK = (1 << MLF_ALLOW_LOOPBACK_BIT),
>
> -    /* Indicate that a packet was received from a VXLAN tunnel to
> -     * compensate for the lack of egress port information available in
> -     * VXLAN encapsulation.  Egress port information is available for
> -     * Geneve and STT tunnel types. */
> -    MLF_RCV_FROM_VXLAN = (1 << MLF_RCV_FROM_VXLAN_BIT),
> +    /* Indicate that a packet was received from a ramp switch to
> compensate for
> +     * the lack of egress port information available in ramp switch
> +     * encapsulation.  Egress port information is available for Geneve,
> STT and
> +     * regular VXLAN tunnel types. */
> +    MLF_RCV_FROM_RAMP = (1 << MLF_RCV_FROM_RAMP_BIT),
>
>      /* Indicate that a packet needs a force SNAT in the gateway router
> when
>       * DNAT has taken place. */
> diff --git a/lib/ovn-util.h b/lib/ovn-util.h
> index 0f7b501f1..58d41a582 100644
> --- a/lib/ovn-util.h
> +++ b/lib/ovn-util.h
> @@ -16,6 +16,7 @@
>  #ifndef OVN_UTIL_H
>  #define OVN_UTIL_H 1
>
> +#include "lib/ovn-sb-idl.h"
>  #include "lib/packets.h"
>  #include "include/ovn/version.h"
>
> @@ -107,6 +108,10 @@ void ovn_conn_show(struct unixctl_conn *conn, int
> argc OVS_UNUSED,
>  #define OVN_MAX_DP_KEY_LOCAL (OVN_MAX_DP_KEY - OVN_MAX_DP_GLOBAL_NUM)
>  #define OVN_MIN_DP_KEY_GLOBAL (OVN_MAX_DP_KEY_LOCAL + 1)
>  #define OVN_MAX_DP_KEY_GLOBAL OVN_MAX_DP_KEY
> +
> +#define OVN_MAX_DP_VXLAN_KEY ((1u << 12) - 1)
> +#define OVN_MAX_DP_VXLAN_KEY_LOCAL (OVN_MAX_DP_KEY -
> OVN_MAX_DP_GLOBAL_NUM)
> +
>  struct hmap;
>  void ovn_destroy_tnlids(struct hmap *tnlids);
>  void ovn_add_tnlid(struct hmap *set, uint32_t tnlid);
> diff --git a/northd/ovn-northd.c b/northd/ovn-northd.c
> index db14909fc..92add9125 100644
> --- a/northd/ovn-northd.c
> +++ b/northd/ovn-northd.c
> @@ -64,6 +64,7 @@ struct northd_context {
>      struct ovsdb_idl *ovnsb_idl;
>      struct ovsdb_idl_txn *ovnnb_txn;
>      struct ovsdb_idl_txn *ovnsb_txn;
> +    struct ovsdb_idl_index *sbrec_chassis_by_name;
>      struct ovsdb_idl_index *sbrec_ha_chassis_grp_by_name;
>      struct ovsdb_idl_index *sbrec_mcast_group_by_name_dp;
>      struct ovsdb_idl_index *sbrec_ip_mcast_by_dp;
> @@ -1189,12 +1190,34 @@ join_datapaths(struct northd_context *ctx, struct
> hmap *datapaths,
>      }
>  }
>
> +static bool is_vxlan_mode(struct ovsdb_idl *ovnsb_idl)
> +{
> +    const struct sbrec_chassis *chassis;
> +    SBREC_CHASSIS_FOR_EACH (chassis, ovnsb_idl) {
> +        for (int i = 0; i < chassis->n_encaps; i++) {
> +            if (!strcmp(chassis->encaps[i]->type, "vxlan")) {
> +                return true;
> +            }
> +        }
> +    }
> +    return false;
> +}
> +
> +static uint32_t
> +get_ovn_max_dp_key_local(struct northd_context *ctx) {
> +    if (is_vxlan_mode(ctx->ovnsb_idl)) {
> +        /* OVN_MAX_DP_GLOBAL_NUM doesn't apply for vxlan mode. */
> +        return OVN_MAX_DP_VXLAN_KEY;
> +    }
> +    return OVN_MAX_DP_KEY - OVN_MAX_DP_GLOBAL_NUM;
> +}
> +
>  static uint32_t
> -ovn_datapath_allocate_key(struct hmap *dp_tnlids)
> +ovn_datapath_allocate_key(struct northd_context *ctx, struct hmap
> *dp_tnlids)
>  {
>      static uint32_t hint;
>      return ovn_allocate_tnlid(dp_tnlids, "datapath", OVN_MIN_DP_KEY_LOCAL,
> -                              OVN_MAX_DP_KEY_LOCAL, &hint);
> +                              get_ovn_max_dp_key_local(ctx), &hint);
>  }
>
>  /* Updates the southbound Datapath_Binding table so that it contains the
> @@ -1237,7 +1260,7 @@ build_datapaths(struct northd_context *ctx, struct
> hmap *datapaths,
>              }
>          }
>          if (!tunnel_key) {
> -            tunnel_key = ovn_datapath_allocate_key(&dp_tnlids);
> +            tunnel_key = ovn_datapath_allocate_key(ctx, &dp_tnlids);
>              if (!tunnel_key) {
>                  break;
>              }
> @@ -12110,32 +12133,34 @@ ovnnb_db_run(struct northd_context *ctx,
>          }
>      }
>
> -    if (!mac_addr_prefix || !monitor_mac) {
> -        struct smap options;
> -        smap_clone(&options, &nb->options);
> +    struct smap options;
> +    smap_clone(&options, &nb->options);
>
> -        if (!mac_addr_prefix) {
> -            eth_addr_random(&mac_prefix);
> -            memset(&mac_prefix.ea[3], 0, 3);
> +    if (!mac_addr_prefix) {
> +        eth_addr_random(&mac_prefix);
> +        memset(&mac_prefix.ea[3], 0, 3);
>
> -            smap_add_format(&options, "mac_prefix",
> -                            "%02"PRIx8":%02"PRIx8":%02"PRIx8,
> -                            mac_prefix.ea[0], mac_prefix.ea[1],
> -                            mac_prefix.ea[2]);
> -        }
> +        smap_add_format(&options, "mac_prefix",
> +                        "%02"PRIx8":%02"PRIx8":%02"PRIx8,
> +                        mac_prefix.ea[0], mac_prefix.ea[1],
> +                        mac_prefix.ea[2]);
> +    }
>
> -        if (!monitor_mac) {
> -            eth_addr_random(&svc_monitor_mac_ea);
> -            snprintf(svc_monitor_mac, sizeof svc_monitor_mac,
> -                     ETH_ADDR_FMT, ETH_ADDR_ARGS(svc_monitor_mac_ea));
> -            smap_replace(&options, "svc_monitor_mac", svc_monitor_mac);
> -        }
> +    if (!monitor_mac) {
> +        eth_addr_random(&svc_monitor_mac_ea);
> +        snprintf(svc_monitor_mac, sizeof svc_monitor_mac,
> +                 ETH_ADDR_FMT, ETH_ADDR_ARGS(svc_monitor_mac_ea));
> +        smap_replace(&options, "svc_monitor_mac", svc_monitor_mac);
> +    }
>
> -        nbrec_nb_global_verify_options(nb);
> -        nbrec_nb_global_set_options(nb, &options);
> +    char *max_tunid = xasprintf("%d", get_ovn_max_dp_key_local(ctx));
> +    smap_replace(&options, "max_tunid", max_tunid);
> +    free(max_tunid);
>
> -        smap_destroy(&options);
> -    }
> +    nbrec_nb_global_verify_options(nb);
> +    nbrec_nb_global_set_options(nb, &options);
> +
> +    smap_destroy(&options);
>
>      /* Update the probe interval. */
>      northd_probe_interval_nb = get_probe_interval(ovnnb_db, nb);
> @@ -13009,6 +13034,10 @@ main(int argc, char *argv[])
>      ovsdb_idl_add_table(ovnsb_idl_loop.idl, &sbrec_table_chassis);
>      ovsdb_idl_add_column(ovnsb_idl_loop.idl, &sbrec_chassis_col_name);
>      ovsdb_idl_add_column(ovnsb_idl_loop.idl,
> &sbrec_chassis_col_other_config);
> +    ovsdb_idl_add_column(ovnsb_idl_loop.idl, &sbrec_chassis_col_encaps);
> +
> +    ovsdb_idl_add_table(ovnsb_idl_loop.idl, &sbrec_table_encap);
> +    ovsdb_idl_add_column(ovnsb_idl_loop.idl, &sbrec_encap_col_type);
>
>      ovsdb_idl_add_table(ovnsb_idl_loop.idl, &sbrec_table_chassis_private);
>      ovsdb_idl_add_column(ovnsb_idl_loop.idl,
> @@ -13122,6 +13151,7 @@ main(int argc, char *argv[])
>                  .ovnnb_txn = ovsdb_idl_loop_run(&ovnnb_idl_loop),
>                  .ovnsb_idl = ovnsb_idl_loop.idl,
>                  .ovnsb_txn = ovsdb_idl_loop_run(&ovnsb_idl_loop),
> +                .sbrec_chassis_by_name = sbrec_chassis_by_name,
>                  .sbrec_ha_chassis_grp_by_name =
> sbrec_ha_chassis_grp_by_name,
>                  .sbrec_mcast_group_by_name_dp =
> sbrec_mcast_group_by_name_dp,
>                  .sbrec_ip_mcast_by_dp = sbrec_ip_mcast_by_dp,
> diff --git a/ovn-architecture.7.xml b/ovn-architecture.7.xml
> index b81844f89..6d9057507 100644
> --- a/ovn-architecture.7.xml
> +++ b/ovn-architecture.7.xml
> @@ -1163,11 +1163,14 @@
>
>        <p>
>          Geneve and STT tunnels pass this field as part of the tunnel key.
> -        Although VXLAN tunnels do not explicitly carry a logical input
> port,
> -        OVN only uses VXLAN to communicate with gateways that from OVN's
> +        Ramp switch VXLAN tunnels do not explicitly carry a logical input
> port,
> +        but since they are used to communicate with gateways that from
> OVN's
>          perspective consist of only a single logical port, so that OVN
> can set
>          the logical input port field to this one on ingress to the OVN
> logical
> -        pipeline.
> +        pipeline. As for regular VXLAN tunnels, they don't carry input
> port
> +        field at all. This puts additional limitations on cluster
> +        capabilities that are described in
> +        <code>Tunnel Encapsulations</code> section.
>        </p>
>      </dd>
>
> @@ -1183,15 +1186,15 @@
>        </p>
>
>        <p>
> -        Geneve and STT tunnels pass this field as part of the tunnel key.
> -        VXLAN tunnels do not transmit the logical output port field.
> -        Since VXLAN tunnels do not carry a logical output port field in
> -        the tunnel key, when a packet is received from VXLAN tunnel by
> -        an OVN hypervisor, the packet is resubmitted to table 8 to
> -        determine the output port(s);  when the packet reaches table 32,
> +        Geneve, STT and regular VXLAN tunnels pass this field as part of
> the
> +        tunnel key. Ramp switch VXLAN tunnels do not transmit the logical
> +        output port field, and since they do not carry a logical output
> port
> +        field in the tunnel key, when a packet is received from ramp
> switch
> +        VXLAN tunnel by an OVN hypervisor, the packet is resubmitted to
> table 8
> +        to determine the output port(s); when the packet reaches table 32,
>          these packets are resubmitted to table 33 for local delivery by
> -        checking a MLF_RCV_FROM_VXLAN flag, which is set when the packet
> -        arrives from a VXLAN tunnel.
> +        checking a MLF_RCV_FROM_RAMP flag, which is set when the packet
> +        arrives from a ramp tunnel.
>        </p>
>      </dd>
>
> @@ -1263,15 +1266,16 @@
>        </p>
>
>        <p>
> -        Table 0 also processes packets that arrive from other chassis.  It
> +        Table 0 also processes packets that arrive from other chassis. It
>          distinguishes them from other packets by ingress port, which is a
> -        tunnel.  As with packets just entering the OVN pipeline, the
> actions
> -        annotate these packets with logical datapath and logical ingress
> port
> -        metadata.  In addition, the actions set the logical output port
> field,
> +        tunnel. As with packets just entering the OVN pipeline, the
> actions
> +        annotate these packets with logical datapath metadata. For tunnel
> types
> +        that support it, they are also annotated with logical ingress port
> +        metadata. In addition, the actions set the logical output port
> field,
>          which is available because in OVN tunneling occurs after the
> logical
> -        output port is known.  These three pieces of information are
> obtained
> +        output port is known. These pieces of information are obtained
>          from the tunnel encapsulation metadata (see <code>Tunnel
> -        Encapsulations</code> for encoding details).  Then the actions
> resubmit
> +        Encapsulations</code> for encoding details). Then the actions
> resubmit
>          to table 33 to enter the logical egress pipeline.
>        </p>
>      </li>
> @@ -1430,12 +1434,12 @@
>
>        <ul>
>          <li>
> -          A higher-priority rule to match packets received from VXLAN
> tunnels,
> -          based on flag MLF_RCV_FROM_VXLAN, and resubmit these packets to
> table
> -          33 for local delivery.  Packets received from VXLAN tunnels
> reach
> -          here because of a lack of logical output port field in the
> tunnel key
> -          and thus these packets needed to be submitted to table 8 to
> -          determine the output port.
> +          A higher-priority rule to match packets received from ramp
> switch
> +          tunnels, based on flag MLF_RCV_FROM_RAMP, and resubmit these
> packets
> +          to table 33 for local delivery.  Packets received from ramp
> switch
> +          tunnels reach here because of a lack of logical output port
> field in
> +          the tunnel key and thus these packets needed to be submitted to
> table
> +          8 to determine the output port.
>          </li>
>          <li>
>            A higher-priority rule to match packets received from ports of
> type
> @@ -2352,6 +2356,11 @@
>      communication between the AZs are established.
>    </p>
>
> +  <p>
> +    When VXLAN tunneling is enabled in an OVN cluster, due to the limited
> +    range available for VNIs, Interconnection feature is not supported.
> +  </p>
> +
>    <h3>A day in the life of a packet crossing AZs</h3>
>    <ol>
>      <li>
> @@ -2658,9 +2667,9 @@
>    <h2>Tunnel Encapsulations</h2>
>
>    <p>
> -    OVN annotates logical network packets that it sends from one
> hypervisor to
> -    another with the following three pieces of metadata, which are
> encoded in
> -    an encapsulation-specific fashion:
> +    In general, OVN annotates logical network packets that it sends from
> one
> +    hypervisor to another with the following three pieces of metadata,
> which
> +    are encoded in an encapsulation-specific fashion:
>    </p>
>
>    <ul>
> @@ -2686,16 +2695,57 @@
>    </ul>
>
>    <p>
> -    For hypervisor-to-hypervisor traffic, OVN supports only Geneve and STT
> -    encapsulations, for the following reasons:
> +      When VXLAN is enabled on any hypervisor in a cluster, datapath and
> egress
> +      port identifier ranges are reduced to 12-bits. This is done because
> only
> +      STT and Geneve provide the large space for metadata (over 32 bits
> per
> +      packet). To accommodate for VXLAN, 24 bits available are split as
> +      follows:
>    </p>
>
>    <ul>
>      <li>
> -      Only STT and Geneve support the large amounts of metadata (over 32
> bits
> -      per packet) that OVN uses (as described above).
> +      12-bit logical datapath identifier, derived from the
> +      <code>tunnel_key</code> column in the OVN Southbound
> +      <code>Datapath_Binding</code> table.
> +    </li>
> +
> +    <li>
> +      12-bit logical egress port identifier.  IDs 0 through 32767 have
> the same
> +      meaning as for logical ingress ports.  IDs 32768 through 65535,
> +      inclusive, may be assigned to logical multicast groups (see the
> +      <code>tunnel_key</code> column in the OVN Southbound
> +      <code>Multicast_Group</code> table).
>      </li>
>
> +    <li>
> +      No logical ingress port identifier.
> +    </li>
> +  </ul>
> +
> +  <p>
> +      The limited space available for metadata when VXLAN tunnels are
> enabled
> +      in a cluster put the following functional limitations onto features
> +      available to users:
> +  </p>
> +
> +  <ul>
> +    <li>
> +      The maximum number of networks is reduced to 4096.
> +    </li>
> +    <li>
> +      The maximum number of ports per network is reduced to 4096.
> (Including
> +      multicast group ports.)
> +    </li>
> +    <li>
> +      ACLs matching against logical ingress port identifiers are not
> supported.
> +    </li>
> +  </ul>
> +
> +  <p>
> +      In addition to functional limitations described above, the following
> +      should be considered before enabling it in your cluster:
> +  </p>
> +  <ul>
>      <li>
>        STT and Geneve use randomized UDP or TCP source ports that allows
>        efficient distribution among multiple paths in environments that
> use ECMP
> diff --git a/ovn-nb.xml b/ovn-nb.xml
> index 0bfe626f1..86195af34 100644
> --- a/ovn-nb.xml
> +++ b/ovn-nb.xml
> @@ -267,6 +267,16 @@
>          OVN tunnels will be encrypted with IPsec.
>        </column>
>      </group>
> +
> +    <group title="Read-only Options">
> +      <column name="options" key="max_tunid">
> +        <p>
> +          The maximum supported tunnel ID. Depends on types of
> encapsulation
> +          enabled in the cluster.
> +        </p>
> +      </column>
> +    </group>
> +
>    </table>
>
>    <table name="Logical_Switch" title="L2 logical switch">
> diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at
> index c639c0ceb..a6719be83 100644
> --- a/tests/ovn-macros.at
> +++ b/tests/ovn-macros.at
> @@ -215,7 +215,7 @@ net_attach () {
>
>  # ovn_az_attach AZ NETWORK BRIDGE IP [MASKLEN]
>  ovn_az_attach() {
> -    local az=$1 net=$2 bridge=$3 ip=$4 masklen=${5-24}
> +    local az=$1 net=$2 bridge=$3 ip=$4 masklen=${5-24}
> encap=${6-geneve,vxlan}
>      net_attach $net $bridge || return 1
>
>      mac=`ovs-vsctl get Interface $bridge mac_in_use | sed s/\"//g`
> @@ -232,7 +232,7 @@ ovn_az_attach() {
>      ovs-vsctl \
>          -- set Open_vSwitch . external-ids:system-id=$sandbox \
>          -- set Open_vSwitch . external-ids:ovn-remote=$ovn_remote \
> -        -- set Open_vSwitch . external-ids:ovn-encap-type=geneve,vxlan \
> +        -- set Open_vSwitch . external-ids:ovn-encap-type=$encap \
>          -- set Open_vSwitch . external-ids:ovn-encap-ip=$ip \
>          -- --may-exist add-br br-int \
>          -- set bridge br-int fail-mode=secure
> other-config:disable-in-band=true \
> diff --git a/tests/ovn.at b/tests/ovn.at
> index 41fe577ff..a6f1fb58f 100644
> --- a/tests/ovn.at
> +++ b/tests/ovn.at
> @@ -17907,612 +17907,583 @@ AT_CHECK([ovn-nbctl -u $sockfile show])
>  AT_CLEANUP
>
>
> -AT_SETUP([ovn -- 2 HVs, 2 lports/HV, localnet ports, DVR N-S ARP
> handling])
> -ovn_start
> -
> -# In this test cases we create 3 switches, all connected to same
> -# physical network (through br-phys on each HV). LS1 and LS2 have
> -# 1 VIF each. Each HV has 1 VIF port. The first digit
> -# of VIF port name indicates the hypervisor it is bound to, e.g.
> -# lp23 means VIF 3 on hv2.
> -#
> -# All the switches are connected to a logical router "router".
> -#
> -# Each switch's VLAN tag and their logical switch ports are:
> -#   - ls1:
> -#       - tagged with VLAN 101
> -#       - ports: lp11
> -#   - ls2:
> -#       - tagged with VLAN 201
> -#       - ports: lp22
> -#   - ls-underlay:
> -#       - tagged with VLAN 1000
> -# Note: a localnet port is created for each switch to connect to
> -# physical network.
> -
> -for i in 1 2; do
> -    ls_name=ls$i
> -    ovn-nbctl ls-add $ls_name
> -    ln_port_name=ln$i
> -    if test $i -eq 1; then
> -        ovn-nbctl lsp-add $ls_name $ln_port_name "" 101
> -    elif test $i -eq 2; then
> -        ovn-nbctl lsp-add $ls_name $ln_port_name "" 201
> -    fi
> -    ovn-nbctl lsp-set-addresses $ln_port_name unknown
> -    ovn-nbctl lsp-set-type $ln_port_name localnet
> -    ovn-nbctl lsp-set-options $ln_port_name network_name=phys
> -done
> -
> -# lsp_to_ls LSP
> -#
> -# Prints the name of the logical switch that contains LSP.
> -lsp_to_ls () {
> -    case $1 in dnl (
> -        lp?[[11]]) echo ls1 ;; dnl (
> -        lp?[[12]]) echo ls2 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -vif_to_hv () {
> -    case $1 in dnl (
> -        vif[[1]]?) echo hv1 ;; dnl (
> -        vif[[2]]?) echo hv2 ;; dnl (
> -        vif?[[north]]?) echo hv4 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -ip_to_hex() {
> -       printf "%02x%02x%02x%02x" "$@"
> -}
> -
> -net_add n1
> -for i in 1 2; do
> -    sim_add hv$i
> -    as hv$i
> -    ovs-vsctl add-br br-phys
> -    ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> -    ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:$i$i"
> -    ovn_attach n1 br-phys 192.168.0.$i
> -
> -    ovs-vsctl add-port br-int vif$i$i -- \
> -        set Interface vif$i$i external-ids:iface-id=lp$i$i \
> -                              options:tx_pcap=hv$i/vif$i$i-tx.pcap \
> -                              options:rxq_pcap=hv$i/vif$i$i-rx.pcap \
> -                              ofport-request=$i$i
> -
> -    lsp_name=lp$i$i
> -    ls_name=$(lsp_to_ls $lsp_name)
> -
> -    ovn-nbctl lsp-add $ls_name $lsp_name
> -    ovn-nbctl lsp-set-addresses $lsp_name "f0:00:00:00:00:$i$i
> 192.168.$i.$i"
> -    ovn-nbctl lsp-set-port-security $lsp_name f0:00:00:00:00:$i$i
> -
> -    OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up $lsp_name` = xup])
> -
> -done
> -
> -ovn-nbctl ls-add ls-underlay
> -ovn-nbctl lsp-add ls-underlay ln3 "" 1000
> -ovn-nbctl lsp-set-addresses ln3 unknown
> -ovn-nbctl lsp-set-type ln3 localnet
> -ovn-nbctl lsp-set-options ln3 network_name=phys
> -
> -ovn-nbctl ls-add ls-north
> -ovn-nbctl lsp-add ls-north ln4 "" 1000
> -ovn-nbctl lsp-set-addresses ln4 unknown
> -ovn-nbctl lsp-set-type ln4 localnet
> -ovn-nbctl lsp-set-options ln4 network_name=phys
> -
> -# Add a VM on ls-north
> -ovn-nbctl lsp-add ls-north lp-north
> -ovn-nbctl lsp-set-addresses lp-north "f0:f0:00:00:00:11 172.31.0.10"
> -ovn-nbctl lsp-set-port-security lp-north f0:f0:00:00:00:11
> -
> -# Add 3rd hypervisor
> -sim_add hv3
> -as hv3 ovs-vsctl add-br br-phys
> -as hv3 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> -as hv3 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:33"
> -as hv3 ovn_attach n1 br-phys 192.168.0.3
> -
> -# Add 4th hypervisor
> -sim_add hv4
> -as hv4 ovs-vsctl add-br br-phys
> -as hv4 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> -as hv4 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:44"
> -as hv4 ovn_attach n1 br-phys 192.168.0.4
> -
> -as hv4 ovs-vsctl add-port br-int vif-north -- \
> -        set Interface vif-north external-ids:iface-id=lp-north \
> -                              options:tx_pcap=hv4/vif-north-tx.pcap \
> -                              options:rxq_pcap=hv4/vif-north-rx.pcap \
> -                              ofport-request=44
> -
> -ovn-nbctl lr-add router
> -ovn-nbctl lrp-add router router-to-ls1 00:00:01:01:02:03 192.168.1.3/24
> -ovn-nbctl <http://192.168.1.3/24-ovn-nbctl> lrp-add router router-to-ls2
> 00:00:01:01:02:05 192.168.2.3/24
> -ovn-nbctl <http://192.168.2.3/24-ovn-nbctl> lrp-add router
> router-to-underlay 00:00:01:01:02:07 172.31.0.1/24
> -
> -ovn-nbctl lsp-add ls1 ls1-to-router -- set Logical_Switch_Port
> ls1-to-router type=router \
> -          options:router-port=router-to-ls1 -- lsp-set-addresses
> ls1-to-router router
> -ovn-nbctl lsp-add ls2 ls2-to-router -- set Logical_Switch_Port
> ls2-to-router type=router \
> -          options:router-port=router-to-ls2 -- lsp-set-addresses
> ls2-to-router router
> -ovn-nbctl lsp-add ls-underlay underlay-to-router -- set
> Logical_Switch_Port \
> -                              underlay-to-router type=router \
> -                              options:router-port=router-to-underlay \
> -                              -- lsp-set-addresses underlay-to-router
> router
> -
> -
> -OVN_POPULATE_ARP
> -
> -# lsp_to_ls LSP
> -#
> -# Prints the name of the logical switch that contains LSP.
> -lsp_to_ls () {
> -    case $1 in dnl (
> -        lp?[[11]]) echo ls1 ;; dnl (
> -        lp?[[12]]) echo ls2 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -vif_to_ls () {
> -    case $1 in dnl (
> -        vif?[[11]]) echo ls1 ;; dnl (
> -        vif?[[12]]) echo ls2 ;; dnl (
> -        vif-north) echo ls-north ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -hv_to_num () {
> -    case $1 in dnl (
> -        hv1) echo 1 ;; dnl (
> -        hv2) echo 2 ;; dnl (
> -        hv3) echo 3 ;; dnl (
> -        hv4) echo 4 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -vif_to_num () {
> -    case $1 in dnl (
> -        vif22) echo 22 ;; dnl (
> -        vif21) echo 21 ;; dnl (
> -        vif11) echo 11 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -vif_to_hv () {
> -    case $1 in dnl (
> -        vif[[1]]?) echo hv1 ;; dnl (
> -        vif[[2]]?) echo hv2 ;; dnl (
> -        vif-north) echo hv4 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> -
> -vif_to_lrp () {
> -    echo router-to-`vif_to_ls $1`
> -}
> -
> -ip_to_hex() {
> -       printf "%02x%02x%02x%02x" "$@"
> -}
> -
> -# test_arp INPORT SHA SPA TPA [REPLY_HA]
> -#
> -# Causes a packet to be received on INPORT.  The packet is an ARP
> -# request with SHA, SPA, and TPA as specified.  If REPLY_HA is provided,
> then
> -# it should be the hardware address of the target to expect to receive in
> an
> -# ARP reply; otherwise no reply is expected.
> -#
> -# INPORT is an logical switch port number, e.g. 11 for vif11.
> -# SHA and REPLY_HA are each 12 hex digits.
> -# SPA and TPA are each 8 hex digits.
> -test_arp() {
> -    local inport=$1 sha=$2 spa=$3 tpa=$4 reply_ha=$5
> -    local
> request=ffffffffffff${sha}08060001080006040001${sha}${spa}ffffffffffff${tpa}
> -    hv=`vif_to_hv $inport`
> -    as $hv ovs-appctl netdev-dummy/receive $inport $request
> -
> -    if test X$reply_ha = X; then
> -        # Expect to receive the broadcast ARP on the other logical switch
> ports
> -        # if no reply is expected.
> -        local i j
> -        for i in 1 2 3; do
> -            for j in 1 2 3; do
> -                if test $i$j != $inport; then
> -                    echo $request >> $i$j.expected
> -                fi
> -            done
> -        done
> -    else
> -        # Expect to receive the reply, if any.
> -        local
> reply=${sha}${reply_ha}08060001080006040002${reply_ha}${tpa}${sha}${spa}
> -        local
> reply_vid=${sha}${reply_ha}810003e808060001080006040002${reply_ha}${tpa}${sha}${spa}
> -        echo $reply_vid >> ${inport}_vid.expected
> -        echo $reply >> $inport.expected
> -    fi
> -}
> -
> -sip=`ip_to_hex 172 31 0 10`
> -tip=`ip_to_hex 172 31 0 1`
> -
> -# Set a hypervisor as gateway chassis, for router port 172.31.0.1
> -ovn-nbctl lrp-set-gateway-chassis router-to-underlay hv3
> -ovn-nbctl --wait=sb sync
> -
> -# Dump a bunch of info helpful for debugging if there's a failure.
> -
> -echo "------ OVN dump ------"
> -ovn-nbctl show
> -ovn-sbctl show
> -ovn-sbctl list port_binding
> -ovn-sbctl list mac_binding
> -
> -echo "------ hv1 dump ------"
> -as hv1 ovs-vsctl show
> -as hv1 ovs-vsctl list Open_Vswitch
> -
> -echo "------ hv2 dump ------"
> -as hv2 ovs-vsctl show
> -as hv2 ovs-vsctl list Open_Vswitch
> -
> -echo "------ hv3 dump ------"
> -as hv3 ovs-vsctl show
> -as hv3 ovs-vsctl list Open_Vswitch
> -
> -echo "------ hv4 dump ------"
> -as hv4 ovs-vsctl show
> -as hv4 ovs-vsctl list Open_Vswitch
> -
> -OVS_WAIT_UNTIL([test x`ovn-sbctl --bare --columns chassis find
> port_binding  logical_port=cr-router-to-underlay | wc -l` = x1])
> -
> -test_arp vif-north f0f000000011 $sip $tip 000001010207
> -
> -# Confirm that vif-north gets a single ARP reply
> -OVN_CHECK_PACKETS_REMOVE_BROADCAST([hv4/vif-north-tx.pcap],
> [vif-north.expected])
> -
> -# Confirm that only redirect chassis allowed arp resolution.
> -OVN_CHECK_PACKETS_REMOVE_BROADCAST([hv3/br-phys_n1-tx.pcap],
> [vif-north_vid.expected])
> -
> -# Confirm that other OVN chassis did not generate ARP reply.
> -$PYTHON "$top_srcdir/utilities/ovs-pcap.in" hv1/br-phys_n1-tx.pcap >
> hv1/br-phys_n1-tx.packets
> -$PYTHON "$top_srcdir/utilities/ovs-pcap.in" hv2/br-phys_n1-tx.pcap >
> hv2/br-phys_n1-tx.packets
> -
> -AT_CHECK([grep 000001010207 hv1/br-phys_n1-tx.packets | wc -l], [0], [[0
> -]])
> -AT_CHECK([grep 000001010207 hv2/br-phys_n1-tx.packets | wc -l], [0], [[0
> -]])
> -
> -echo "----------- Post Traffic hv1 dump -----------"
> -as hv1 ovs-ofctl -O OpenFlow13 dump-flows br-int
> -as hv1 ovs-appctl fdb/show br-phys
> -
> -echo "----------- Post Traffic hv2 dump -----------"
> -as hv2 ovs-ofctl -O OpenFlow13 dump-flows br-int
> -as hv2 ovs-appctl fdb/show br-phys
> -
> -echo "----------- Post Traffic hv3 dump -----------"
> -as hv3 ovs-ofctl -O OpenFlow13 dump-flows br-int
> -as hv3 ovs-appctl fdb/show br-phys
> -
> -echo "----------- Post Traffic hv4 dump -----------"
> -as hv4 ovs-ofctl -O OpenFlow13 dump-flows br-int
> -as hv4 ovs-appctl fdb/show br-phys
> -
> -OVN_CLEANUP([hv1],[hv2],[hv3],[hv4])
> -
> -AT_CLEANUP
> -
> -AT_SETUP([ovn -- 2 HVs, 2 lports/HV, localnet ports, DVR N-S Ping])
> -ovn_start
> -
> -# In this test cases we create 3 switches, all connected to same
> -# physical network (through br-phys on each HV). LS1 and LS2 have
> -# 1 VIF each. Each HV has 1 VIF port. The first digit
> -# of VIF port name indicates the hypervisor it is bound to, e.g.
> -# lp23 means VIF 3 on hv2.
> -#
> -# All the switches are connected to a logical router "router".
> -#
> -# Each switch's VLAN tag and their logical switch ports are:
> -#   - ls1:
> -#       - tagged with VLAN 101
> -#       - ports: lp11
> -#   - ls2:
> -#       - tagged with VLAN 201
> -#       - ports: lp22
> -#   - ls-underlay:
> -#       - tagged with VLAN 1000
> -# Note: a localnet port is created for each switch to connect to
> -# physical network.
> -
> -for i in 1 2; do
> -    ls_name=ls$i
> -    ovn-nbctl ls-add $ls_name
> -    ln_port_name=ln$i
> -    if test $i -eq 1; then
> -        ovn-nbctl lsp-add $ls_name $ln_port_name "" 101
> -    elif test $i -eq 2; then
> -        ovn-nbctl lsp-add $ls_name $ln_port_name "" 201
> -    fi
> -    ovn-nbctl lsp-set-addresses $ln_port_name unknown
> -    ovn-nbctl lsp-set-type $ln_port_name localnet
> -    ovn-nbctl lsp-set-options $ln_port_name network_name=phys
> -done
> -
> -# lsp_to_ls LSP
> -#
> -# Prints the name of the logical switch that contains LSP.
> -lsp_to_ls () {
> -    case $1 in dnl (
> -        lp?[[11]]) echo ls1 ;; dnl (
> -        lp?[[12]]) echo ls2 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +m4_define([DVR_N_S_ARP_HANDLING],
> +  [AT_SETUP([ovn -- 2 HVs, 2 lports/HV, localnet ports, DVR N-S ARP
> handling, encap $1])
> +   set -x
> +   encap=$1
> +   ovn_start
> +
> +   # In this test cases we create 3 switches, all connected to same
> +   # physical network (through br-phys on each HV). LS1 and LS2 have
> +   # 1 VIF each. Each HV has 1 VIF port. The first digit
> +   # of VIF port name indicates the hypervisor it is bound to, e.g.
> +   # lp23 means VIF 3 on hv2.
> +   #
> +   # All the switches are connected to a logical router "router".
> +   #
> +   # Each switch's VLAN tag and their logical switch ports are:
> +   #   - ls1:
> +   #       - tagged with VLAN 101
> +   #       - ports: lp11
> +   #   - ls2:
> +   #       - tagged with VLAN 201
> +   #       - ports: lp22
> +   #   - ls-underlay:
> +   #       - tagged with VLAN 1000
> +   # Note: a localnet port is created for each switch to connect to
> +   # physical network.
> +
> +   for i in 1 2; do
> +       ls_name=ls$i
> +       ovn-nbctl ls-add $ls_name
> +       ln_port_name=ln$i
> +       if test $i -eq 1; then
> +           ovn-nbctl lsp-add $ls_name $ln_port_name "" 101
> +       elif test $i -eq 2; then
> +           ovn-nbctl lsp-add $ls_name $ln_port_name "" 201
> +       fi
> +       ovn-nbctl lsp-set-addresses $ln_port_name unknown
> +       ovn-nbctl lsp-set-type $ln_port_name localnet
> +       ovn-nbctl lsp-set-options $ln_port_name network_name=phys
> +   done
> +
> +   # lsp_to_ls LSP
> +   #
> +   # Prints the name of the logical switch that contains LSP.
> +   lsp_to_ls () {
> +       case ${1} in dnl (
> +           lp?[[11]]) echo ls1 ;; dnl (
> +           lp?[[12]]) echo ls2 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -vif_to_hv () {
> -    case $1 in dnl (
> -        vif[[1]]?) echo hv1 ;; dnl (
> -        vif[[2]]?) echo hv2 ;; dnl (
> -        vif?[[north]]?) echo hv4 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +   vif_to_hv () {
> +       case $1 in dnl (
> +           vif[[1]]?) echo hv1 ;; dnl (
> +           vif[[2]]?) echo hv2 ;; dnl (
> +           vif?[[north]]?) echo hv4 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -ip_to_hex() {
> -       printf "%02x%02x%02x%02x" "$@"
> -}
> +   net_add n1
> +   for i in 1 2; do
> +       sim_add hv$i
> +       as hv$i
> +       ovs-vsctl add-br br-phys
> +       ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> +       ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:$i$i"
> +       ovn_attach n1 br-phys 192.168.0.$i 24 $encap
> +
> +       ovs-vsctl add-port br-int vif$i$i -- \
> +           set Interface vif$i$i external-ids:iface-id=lp$i$i \
> +                                 options:tx_pcap=hv$i/vif$i$i-tx.pcap \
> +                                 options:rxq_pcap=hv$i/vif$i$i-rx.pcap \
> +                                 ofport-request=$i$i
> +
> +       lsp_name=lp$i$i
> +       ls_name=$(lsp_to_ls $lsp_name)
> +
> +       ovn-nbctl lsp-add $ls_name $lsp_name
> +       ovn-nbctl lsp-set-addresses $lsp_name "f0:00:00:00:00:$i$i
> 192.168.$i.$i"
> +       ovn-nbctl lsp-set-port-security $lsp_name f0:00:00:00:00:$i$i
> +
> +       OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up $lsp_name` = xup])
> +
> +   done
> +
> +   ovn-nbctl ls-add ls-underlay
> +   ovn-nbctl lsp-add ls-underlay ln3 "" 1000
> +   ovn-nbctl lsp-set-addresses ln3 unknown
> +   ovn-nbctl lsp-set-type ln3 localnet
> +   ovn-nbctl lsp-set-options ln3 network_name=phys
> +
> +   ovn-nbctl ls-add ls-north
> +   ovn-nbctl lsp-add ls-north ln4 "" 1000
> +   ovn-nbctl lsp-set-addresses ln4 unknown
> +   ovn-nbctl lsp-set-type ln4 localnet
> +   ovn-nbctl lsp-set-options ln4 network_name=phys
> +
> +   # Add a VM on ls-north
> +   ovn-nbctl lsp-add ls-north lp-north
> +   ovn-nbctl lsp-set-addresses lp-north "f0:f0:00:00:00:11 172.31.0.10"
> +   ovn-nbctl lsp-set-port-security lp-north f0:f0:00:00:00:11
> +
> +   # Add 3rd hypervisor
> +   sim_add hv3
> +   as hv3 ovs-vsctl add-br br-phys
> +   as hv3 ovs-vsctl set open .
> external-ids:ovn-bridge-mappings=phys:br-phys
> +   as hv3 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:33"
> +   as hv3 ovn_attach n1 br-phys 192.168.0.3 24 $encap
> +
> +   # Add 4th hypervisor
> +   sim_add hv4
> +   as hv4 ovs-vsctl add-br br-phys
> +   as hv4 ovs-vsctl set open .
> external-ids:ovn-bridge-mappings=phys:br-phys
> +   as hv4 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:44"
> +   as hv4 ovn_attach n1 br-phys 192.168.0.4 24 $encap
> +
> +   as hv4 ovs-vsctl add-port br-int vif-north -- \
> +           set Interface vif-north external-ids:iface-id=lp-north \
> +                                 options:tx_pcap=hv4/vif-north-tx.pcap \
> +                                 options:rxq_pcap=hv4/vif-north-rx.pcap \
> +                                 ofport-request=44
> +
> +   ovn-nbctl lr-add router
> +   ovn-nbctl lrp-add router router-to-ls1 00:00:01:01:02:03
> 192.168.1.3/24
> +   ovn-nbctl lrp-add router router-to-ls2 00:00:01:01:02:05
> 192.168.2.3/24
> +   ovn-nbctl lrp-add router router-to-underlay 00:00:01:01:02:07
> 172.31.0.1/24
> +
> +   ovn-nbctl lsp-add ls1 ls1-to-router -- set Logical_Switch_Port
> ls1-to-router type=router \
> +             options:router-port=router-to-ls1 -- lsp-set-addresses
> ls1-to-router router
> +   ovn-nbctl lsp-add ls2 ls2-to-router -- set Logical_Switch_Port
> ls2-to-router type=router \
> +             options:router-port=router-to-ls2 -- lsp-set-addresses
> ls2-to-router router
> +   ovn-nbctl lsp-add ls-underlay underlay-to-router -- set
> Logical_Switch_Port \
> +                                 underlay-to-router type=router \
> +                                 options:router-port=router-to-underlay \
> +                                 -- lsp-set-addresses underlay-to-router
> router
> +
> +
> +   OVN_POPULATE_ARP
> +
> +   vif_to_hv () {
> +       case ${1} in dnl (
> +           vif[[1]]?) echo hv1 ;; dnl (
> +           vif[[2]]?) echo hv2 ;; dnl (
> +           vif-north) echo hv4 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -net_add n1
> -for i in 1 2; do
> -    sim_add hv$i
> -    as hv$i
> -    ovs-vsctl add-br br-phys
> -    ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> -    ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:$i$i"
> -    ovn_attach n1 br-phys 192.168.0.$i
> +   # test_arp INPORT SHA SPA TPA [REPLY_HA]
> +   #
> +   # Causes a packet to be received on INPORT.  The packet is an ARP
> +   # request with SHA, SPA, and TPA as specified.  If REPLY_HA is
> provided, then
> +   # it should be the hardware address of the target to expect to receive
> in an
> +   # ARP reply; otherwise no reply is expected.
> +   #
> +   # INPORT is an logical switch port number, e.g. 11 for vif11.
> +   # SHA and REPLY_HA are each 12 hex digits.
> +   # SPA and TPA are each 8 hex digits.
> +   test_arp() {
> +       local inport=${1} sha=${2} spa=${3} tpa=${4} reply_ha=${5}
> +       local
> request=ffffffffffff${sha}08060001080006040001${sha}${spa}ffffffffffff${tpa}
> +       hv=`vif_to_hv $inport`
> +       as $hv ovs-appctl netdev-dummy/receive $inport $request
> +
> +       if test X$reply_ha = X; then
> +           # Expect to receive the broadcast ARP on the other logical
> switch ports
> +           # if no reply is expected.
> +           local i j
> +           for i in 1 2 3; do
> +               for j in 1 2 3; do
> +                   if test $i$j != $inport; then
> +                       echo $request >> $i$j.expected
> +                   fi
> +               done
> +           done
> +       else
> +           # Expect to receive the reply, if any.
> +           local
> reply=${sha}${reply_ha}08060001080006040002${reply_ha}${tpa}${sha}${spa}
> +           local
> reply_vid=${sha}${reply_ha}810003e808060001080006040002${reply_ha}${tpa}${sha}${spa}
> +           echo $reply_vid >> ${inport}_vid.expected
> +           echo $reply >> $inport.expected
> +       fi
> +   }
>
> -    ovs-vsctl add-port br-int vif$i$i -- \
> -        set Interface vif$i$i external-ids:iface-id=lp$i$i \
> -                              options:tx_pcap=hv$i/vif$i$i-tx.pcap \
> -                              options:rxq_pcap=hv$i/vif$i$i-rx.pcap \
> -                              ofport-request=$i$i
> +   sip=`printf "%02x%02x%02x%02x" 172 31 0 10`
> +   tip=`printf "%02x%02x%02x%02x" 172 31 0 1`
>
> -    lsp_name=lp$i$i
> -    ls_name=$(lsp_to_ls $lsp_name)
> +   # Set a hypervisor as gateway chassis, for router port 172.31.0.1
> +   ovn-nbctl lrp-set-gateway-chassis router-to-underlay hv3
> +   ovn-nbctl --wait=sb sync
>
> -    ovn-nbctl lsp-add $ls_name $lsp_name
> -    ovn-nbctl lsp-set-addresses $lsp_name "f0:00:00:00:00:$i$i
> 192.168.$i.$i"
> -    ovn-nbctl lsp-set-port-security $lsp_name f0:00:00:00:00:$i$i
> +   # Dump a bunch of info helpful for debugging if there's a failure.
>
> -    OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up $lsp_name` = xup])
> +   echo "------ OVN dump ------"
> +   ovn-nbctl show
> +   ovn-sbctl show
> +   ovn-sbctl list port_binding
> +   ovn-sbctl list mac_binding
>
> -done
> +   echo "------ hv1 dump ------"
> +   as hv1 ovs-vsctl show
> +   as hv1 ovs-vsctl list Open_Vswitch
>
> -ovn-nbctl ls-add ls-underlay
> -ovn-nbctl lsp-add ls-underlay ln3 "" 1000
> -ovn-nbctl lsp-set-addresses ln3 unknown
> -ovn-nbctl lsp-set-type ln3 localnet
> -ovn-nbctl lsp-set-options ln3 network_name=phys
> +   echo "------ hv2 dump ------"
> +   as hv2 ovs-vsctl show
> +   as hv2 ovs-vsctl list Open_Vswitch
>
> -ovn-nbctl ls-add ls-north
> -ovn-nbctl lsp-add ls-north ln4 "" 1000
> -ovn-nbctl lsp-set-addresses ln4 unknown
> -ovn-nbctl lsp-set-type ln4 localnet
> -ovn-nbctl lsp-set-options ln4 network_name=phys
> +   echo "------ hv3 dump ------"
> +   as hv3 ovs-vsctl show
> +   as hv3 ovs-vsctl list Open_Vswitch
>
> -# Add a VM on ls-north
> -ovn-nbctl lsp-add ls-north lp-north
> -ovn-nbctl lsp-set-addresses lp-north "f0:f0:00:00:00:11 172.31.0.10"
> -ovn-nbctl lsp-set-port-security lp-north f0:f0:00:00:00:11
> +   echo "------ hv4 dump ------"
> +   as hv4 ovs-vsctl show
> +   as hv4 ovs-vsctl list Open_Vswitch
>
> -# Add 3rd hypervisor
> -sim_add hv3
> -as hv3 ovs-vsctl add-br br-phys
> -as hv3 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> -as hv3 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:33"
> -as hv3 ovn_attach n1 br-phys 192.168.0.3
> +   OVS_WAIT_UNTIL([test x`ovn-sbctl --bare --columns chassis find
> port_binding  logical_port=cr-router-to-underlay | wc -l` = x1])
>
> -# Add 4th hypervisor
> -sim_add hv4
> -as hv4 ovs-vsctl add-br br-phys
> -as hv4 ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> -as hv4 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:44"
> -as hv4 ovn_attach n1 br-phys 192.168.0.4
> +   test_arp vif-north f0f000000011 $sip $tip 000001010207
>
> -as hv4 ovs-vsctl add-port br-int vif-north -- \
> -        set Interface vif-north external-ids:iface-id=lp-north \
> -                              options:tx_pcap=hv4/vif-north-tx.pcap \
> -                              options:rxq_pcap=hv4/vif-north-rx.pcap \
> -                              ofport-request=44
> +   # Confirm that vif-north gets a single ARP reply
> +   OVN_CHECK_PACKETS_REMOVE_BROADCAST([hv4/vif-north-tx.pcap],
> [vif-north.expected])
>
> -ovn-nbctl lr-add router
> -ovn-nbctl lrp-add router router-to-ls1 00:00:01:01:02:03 192.168.1.3/24
> -ovn-nbctl <http://192.168.1.3/24-ovn-nbctl> lrp-add router router-to-ls2
> 00:00:01:01:02:05 192.168.2.3/24
> -ovn-nbctl <http://192.168.2.3/24-ovn-nbctl> lrp-add router
> router-to-underlay 00:00:01:01:02:07 172.31.0.1/24
> +   # Confirm that only redirect chassis allowed arp resolution.
> +   OVN_CHECK_PACKETS_REMOVE_BROADCAST([hv3/br-phys_n1-tx.pcap],
> [vif-north_vid.expected])
>
> -ovn-nbctl lsp-add ls1 ls1-to-router -- set Logical_Switch_Port
> ls1-to-router type=router \
> -          options:router-port=router-to-ls1 -- lsp-set-addresses
> ls1-to-router router
> -ovn-nbctl lsp-add ls2 ls2-to-router -- set Logical_Switch_Port
> ls2-to-router type=router \
> -          options:router-port=router-to-ls2 -- lsp-set-addresses
> ls2-to-router router
> -ovn-nbctl lsp-add ls-underlay underlay-to-router -- set
> Logical_Switch_Port \
> -                              underlay-to-router type=router \
> -                              options:router-port=router-to-underlay \
> -                              -- lsp-set-addresses underlay-to-router
> router
> +   # Confirm that other OVN chassis did not generate ARP reply.
> +   $PYTHON "$ovs_srcdir//utilities/ovs-pcap.in" hv1/br-phys_n1-tx.pcap >
> hv1/br-phys_n1-tx.packets
> +   $PYTHON "$ovs_srcdir//utilities/ovs-pcap.in" hv2/br-phys_n1-tx.pcap >
> hv2/br-phys_n1-tx.packets
>
> -ovn-nbctl lrp-set-gateway-chassis router-to-underlay hv3
> -ovn-nbctl lrp-set-redirect-type router-to-underlay bridged
> +   AT_CHECK([grep 000001010207 hv1/br-phys_n1-tx.packets | wc -l], [0],
> [[0
> +]])
> +   AT_CHECK([grep 000001010207 hv2/br-phys_n1-tx.packets | wc -l], [0],
> [[0
> +]])
>
> -ovn-nbctl --wait=sb sync
> +   # validate max_tunid reflects the type of encapsulation used
> +   max_tunid=`ovn-nbctl get NB_Global . options:max_tunid | sed s/":"//g
> | sed s/\"//g`
> +   echo $max_tunid
> +   if [[ $encap = vxlan ]]; then
> +       max_tunid_expected=4095
> +   else
> +       max_tunid_expected=16711680
> +   fi
> +   AT_CHECK([test $max_tunid -eq $max_tunid_expected])
> +
> +   echo "----------- Post Traffic hv1 dump -----------"
> +   as hv1 ovs-ofctl -O OpenFlow13 dump-flows br-int
> +   as hv1 ovs-appctl fdb/show br-phys
> +
> +   echo "----------- Post Traffic hv2 dump -----------"
> +   as hv2 ovs-ofctl -O OpenFlow13 dump-flows br-int
> +   as hv2 ovs-appctl fdb/show br-phys
> +
> +   echo "----------- Post Traffic hv3 dump -----------"
> +   as hv3 ovs-ofctl -O OpenFlow13 dump-flows br-int
> +   as hv3 ovs-appctl fdb/show br-phys
> +
> +   echo "----------- Post Traffic hv4 dump -----------"
> +   as hv4 ovs-ofctl -O OpenFlow13 dump-flows br-int
> +   as hv4 ovs-appctl fdb/show br-phys
> +
> +   OVN_CLEANUP([hv1],[hv2],[hv3],[hv4])
> +
> +   AT_CLEANUP])
> +
> +DVR_N_S_ARP_HANDLING([geneve])
> +DVR_N_S_ARP_HANDLING([vxlan])
> +
> +m4_define([DVR_N_S_PING],
> +  [AT_SETUP([ovn -- 2 HVs, 2 lports/HV, localnet ports, DVR N-S Ping,
> encap $1])
> +   AT_KEYWORDS([$1])
> +   encap=$1
> +   ovn_start
> +
> +   # In this test cases we create 3 switches, all connected to same
> +   # physical network (through br-phys on each HV). LS1 and LS2 have
> +   # 1 VIF each. Each HV has 1 VIF port. The first digit
> +   # of VIF port name indicates the hypervisor it is bound to, e.g.
> +   # lp23 means VIF 3 on hv2.
> +   #
> +   # All the switches are connected to a logical router "router".
> +   #
> +   # Each switch's VLAN tag and their logical switch ports are:
> +   #   - ls1:
> +   #       - tagged with VLAN 101
> +   #       - ports: lp11
> +   #   - ls2:
> +   #       - tagged with VLAN 201
> +   #       - ports: lp22
> +   #   - ls-underlay:
> +   #       - tagged with VLAN 1000
> +   # Note: a localnet port is created for each switch to connect to
> +   # physical network.
> +
> +   for i in 1 2; do
> +       ls_name=ls$i
> +       ovn-nbctl ls-add $ls_name
> +       ln_port_name=ln$i
> +       if test $i -eq 1; then
> +           ovn-nbctl lsp-add $ls_name $ln_port_name "" 101
> +       elif test $i -eq 2; then
> +           ovn-nbctl lsp-add $ls_name $ln_port_name "" 201
> +       fi
> +       ovn-nbctl lsp-set-addresses $ln_port_name unknown
> +       ovn-nbctl lsp-set-type $ln_port_name localnet
> +       ovn-nbctl lsp-set-options $ln_port_name network_name=phys
> +   done
> +
> +   # lsp_to_ls LSP
> +   #
> +   # Prints the name of the logical switch that contains LSP.
> +   lsp_to_ls () {
> +       case ${1} in dnl (
> +           lp?[[11]]) echo ls1 ;; dnl (
> +           lp?[[12]]) echo ls2 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> +   vif_to_hv () {
> +       case ${1} in dnl (
> +           vif[[1]]?) echo hv1 ;; dnl (
> +           vif[[2]]?) echo hv2 ;; dnl (
> +           vif?[[north]]?) echo hv4 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -OVN_POPULATE_ARP
> +   ip_to_hex() {
> +          printf "%02x%02x%02x%02x" "$@"
> +   }
>
> -# lsp_to_ls LSP
> -#
> -# Prints the name of the logical switch that contains LSP.
> -lsp_to_ls () {
> -    case $1 in dnl (
> -        lp?[[11]]) echo ls1 ;; dnl (
> -        lp?[[12]]) echo ls2 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +   net_add n1
> +   for i in 1 2; do
> +       sim_add hv$i
> +       as hv$i
> +       ovs-vsctl add-br br-phys
> +       ovs-vsctl set open . external-ids:ovn-bridge-mappings=phys:br-phys
> +       ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:$i$i"
> +       ovn_attach n1 br-phys 192.168.0.$i 24 $encap
> +
> +       ovs-vsctl add-port br-int vif$i$i -- \
> +           set Interface vif$i$i external-ids:iface-id=lp$i$i \
> +                                 options:tx_pcap=hv$i/vif$i$i-tx.pcap \
> +                                 options:rxq_pcap=hv$i/vif$i$i-rx.pcap \
> +                                 ofport-request=$i$i
> +
> +       lsp_name=lp$i$i
> +       ls_name=$(lsp_to_ls $lsp_name)
> +
> +       ovn-nbctl lsp-add $ls_name $lsp_name
> +       ovn-nbctl lsp-set-addresses $lsp_name "f0:00:00:00:00:$i$i
> 192.168.$i.$i"
> +       ovn-nbctl lsp-set-port-security $lsp_name f0:00:00:00:00:$i$i
> +
> +       OVS_WAIT_UNTIL([test x`ovn-nbctl lsp-get-up $lsp_name` = xup])
> +
> +   done
> +
> +   ovn-nbctl ls-add ls-underlay
> +   ovn-nbctl lsp-add ls-underlay ln3 "" 1000
> +   ovn-nbctl lsp-set-addresses ln3 unknown
> +   ovn-nbctl lsp-set-type ln3 localnet
> +   ovn-nbctl lsp-set-options ln3 network_name=phys
> +
> +   ovn-nbctl ls-add ls-north
> +   ovn-nbctl lsp-add ls-north ln4 "" 1000
> +   ovn-nbctl lsp-set-addresses ln4 unknown
> +   ovn-nbctl lsp-set-type ln4 localnet
> +   ovn-nbctl lsp-set-options ln4 network_name=phys
> +
> +   # Add a VM on ls-north
> +   ovn-nbctl lsp-add ls-north lp-north
> +   ovn-nbctl lsp-set-addresses lp-north "f0:f0:00:00:00:11 172.31.0.10"
> +   ovn-nbctl lsp-set-port-security lp-north f0:f0:00:00:00:11
> +
> +   # Add 3rd hypervisor
> +   sim_add hv3
> +   as hv3 ovs-vsctl add-br br-phys
> +   as hv3 ovs-vsctl set open .
> external-ids:ovn-bridge-mappings=phys:br-phys
> +   as hv3 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:33"
> +   as hv3 ovn_attach n1 br-phys 192.168.0.3 24 $encap
> +
> +   # Add 4th hypervisor
> +   sim_add hv4
> +   as hv4 ovs-vsctl add-br br-phys
> +   as hv4 ovs-vsctl set open .
> external-ids:ovn-bridge-mappings=phys:br-phys
> +   as hv4 ovs-vsctl set open .
> external-ids:ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:44"
> +   as hv4 ovn_attach n1 br-phys 192.168.0.4 24 $encap
> +
> +   as hv4 ovs-vsctl add-port br-int vif-north -- \
> +           set Interface vif-north external-ids:iface-id=lp-north \
> +                                 options:tx_pcap=hv4/vif-north-tx.pcap \
> +                                 options:rxq_pcap=hv4/vif-north-rx.pcap \
> +                                 ofport-request=44
> +
> +   ovn-nbctl lr-add router
> +   ovn-nbctl lrp-add router router-to-ls1 00:00:01:01:02:03
> 192.168.1.3/24
> +   ovn-nbctl lrp-add router router-to-ls2 00:00:01:01:02:05
> 192.168.2.3/24
> +   ovn-nbctl lrp-add router router-to-underlay 00:00:01:01:02:07
> 172.31.0.1/24
> +
> +   ovn-nbctl lsp-add ls1 ls1-to-router -- set Logical_Switch_Port
> ls1-to-router type=router \
> +             options:router-port=router-to-ls1 -- lsp-set-addresses
> ls1-to-router router
> +   ovn-nbctl lsp-add ls2 ls2-to-router -- set Logical_Switch_Port
> ls2-to-router type=router \
> +             options:router-port=router-to-ls2 -- lsp-set-addresses
> ls2-to-router router
> +   ovn-nbctl lsp-add ls-underlay underlay-to-router -- set
> Logical_Switch_Port \
> +                                 underlay-to-router type=router \
> +                                 options:router-port=router-to-underlay \
> +                                 -- lsp-set-addresses underlay-to-router
> router
> +
> +   ovn-nbctl lrp-set-gateway-chassis router-to-underlay hv3
> +   ovn-nbctl lrp-set-redirect-type router-to-underlay bridged
> +
> +   ovn-nbctl --wait=sb sync
> +
> +
> +   OVN_POPULATE_ARP
> +
> +   # lsp_to_ls LSP
> +   #
> +   # Prints the name of the logical switch that contains LSP.
> +   lsp_to_ls () {
> +       case ${1} in dnl (
> +           lp?[[11]]) echo ls1 ;; dnl (
> +           lp?[[12]]) echo ls2 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -vif_to_ls () {
> -    case $1 in dnl (
> -        vif?[[11]]) echo ls1 ;; dnl (
> -        vif?[[12]]) echo ls2 ;; dnl (
> -        vif-north) echo ls-north ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +   vif_to_ls () {
> +       case ${1} in dnl (
> +           vif?[[11]]) echo ls1 ;; dnl (
> +           vif?[[12]]) echo ls2 ;; dnl (
> +           vif-north) echo ls-north ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -hv_to_num () {
> -    case $1 in dnl (
> -        hv1) echo 1 ;; dnl (
> -        hv2) echo 2 ;; dnl (
> -        hv3) echo 3 ;; dnl (
> -        hv4) echo 4 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +   hv_to_num () {
> +       case ${1} in dnl (
> +           hv1) echo 1 ;; dnl (
> +           hv2) echo 2 ;; dnl (
> +           hv3) echo 3 ;; dnl (
> +           hv4) echo 4 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -vif_to_num () {
> -    case $1 in dnl (
> -        vif22) echo 22 ;; dnl (
> -        vif21) echo 21 ;; dnl (
> -        vif11) echo 11 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +   vif_to_num () {
> +       case ${1} in dnl (
> +           vif22) echo 22 ;; dnl (
> +           vif21) echo 21 ;; dnl (
> +           vif11) echo 11 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -vif_to_hv () {
> -    case $1 in dnl (
> -        vif[[1]]?) echo hv1 ;; dnl (
> -        vif[[2]]?) echo hv2 ;; dnl (
> -        vif-north) echo hv4 ;; dnl (
> -        *) AT_FAIL_IF([:]) ;;
> -    esac
> -}
> +   vif_to_hv () {
> +       case ${1} in dnl (
> +           vif[[1]]?) echo hv1 ;; dnl (
> +           vif[[2]]?) echo hv2 ;; dnl (
> +           vif-north) echo hv4 ;; dnl (
> +           *) AT_FAIL_IF([:]) ;;
> +       esac
> +   }
>
> -vif_to_lrp () {
> -    echo router-to-`vif_to_ls $1`
> -}
> +   vif_to_lrp () {
> +       echo router-to-`vif_to_ls ${1}`
> +   }
>
> -ip_to_hex() {
> -       printf "%02x%02x%02x%02x" "$@"
> -}
> +   ip_to_hex() {
> +          printf "%02x%02x%02x%02x" "${@}"
> +   }
>
>
> -test_ip() {
> -        # This packet has bad checksums but logical L3 routing doesn't
> check.
> -        local inport=$1 src_mac=$2 dst_mac=$3 src_ip=$4 dst_ip=$5
> outport=$6
> -        local
> packet=${dst_mac}${src_mac}08004500001c0000000040110000${src_ip}${dst_ip}0035111100080000
> -        shift; shift; shift; shift; shift
> -        hv=`vif_to_hv $inport`
> -        as $hv ovs-appctl netdev-dummy/receive $inport $packet
> -        in_ls=`vif_to_ls $inport`
> -        for outport; do
> -            out_ls=`vif_to_ls $outport`
> -            if test $in_ls = $out_ls; then
> -                # Ports on the same logical switch receive exactly the
> same packet.
> -                echo $packet
> -            else
> -                # Routing decrements TTL and updates source and dest MAC
> -                # (and checksum).
> -                out_lrp=`vif_to_lrp $outport`
> -                # For North-South, packet will come via gateway chassis,
> i.e hv3
> -                if test $inport = vif-north; then
> -                    echo
> f0000000001100000101020308004500001c000000003f110100${src_ip}${dst_ip}0035111100080000
> >> $outport.expected
> -                fi
> -                if test $outport = vif-north; then
> -                    echo
> f0f00000001100000101020708004500001c000000003e110200${src_ip}${dst_ip}0035111100080000
> >> $outport.expected
> -                fi
> -            fi >> $outport.expected
> -        done
> -}
> +   test_ip() {
> +           # This packet has bad checksums but logical L3 routing doesn't
> check.
> +           local inport=${1} src_mac=${2} dst_mac=${3} src_ip=${4}
> dst_ip=${5} outport=${6}
> +           local
> packet=${dst_mac}${src_mac}08004500001c0000000040110000${src_ip}${dst_ip}0035111100080000
> +           shift; shift; shift; shift; shift
> +           hv=`vif_to_hv $inport`
> +           as $hv ovs-appctl netdev-dummy/receive $inport $packet
> +           in_ls=`vif_to_ls $inport`
> +           for outport; do
> +               out_ls=`vif_to_ls $outport`
> +               if test $in_ls = $out_ls; then
> +                   # Ports on the same logical switch receive exactly the
> same packet.
> +                   echo $packet
> +               else
> +                   # Routing decrements TTL and updates source and dest
> MAC
> +                   # (and checksum).
> +                   out_lrp=`vif_to_lrp $outport`
> +                   # For North-South, packet will come via gateway
> chassis, i.e hv3
> +                   if test $inport = vif-north; then
> +                       echo
> f0000000001100000101020308004500001c000000003f110100${src_ip}${dst_ip}0035111100080000
> >> $outport.expected
> +                   fi
> +                   if test $outport = vif-north; then
> +                       echo
> f0f00000001100000101020708004500001c000000003e110200${src_ip}${dst_ip}0035111100080000
> >> $outport.expected
> +                   fi
> +               fi >> $outport.expected
> +           done
> +   }
>
> -# Dump a bunch of info helpful for debugging if there's a failure.
> +   # Dump a bunch of info helpful for debugging if there's a failure.
>
> -echo "------ OVN dump ------"
> -ovn-nbctl show
> -ovn-sbctl show
> -ovn-sbctl list port_binding
> -ovn-sbctl list mac_binding
> +   echo "------ OVN dump ------"
> +   ovn-nbctl show
> +   ovn-sbctl show
> +   ovn-sbctl list port_binding
> +   ovn-sbctl list mac_binding
>
> -echo "------ hv1 dump ------"
> -as hv1 ovs-vsctl show
> -as hv1 ovs-vsctl list Open_Vswitch
> +   echo "------ hv1 dump ------"
> +   as hv1 ovs-vsctl show
> +   as hv1 ovs-vsctl list Open_Vswitch
>
> -echo "------ hv2 dump ------"
> -as hv2 ovs-vsctl show
> -as hv2 ovs-vsctl list Open_Vswitch
> +   echo "------ hv2 dump ------"
> +   as hv2 ovs-vsctl show
> +   as hv2 ovs-vsctl list Open_Vswitch
>
> -echo "------ hv3 dump ------"
> -as hv3 ovs-vsctl show
> -as hv3 ovs-vsctl list Open_Vswitch
> +   echo "------ hv3 dump ------"
> +   as hv3 ovs-vsctl show
> +   as hv3 ovs-vsctl list Open_Vswitch
>
> -echo "------ hv4 dump ------"
> -as hv4 ovs-vsctl show
> -as hv4 ovs-vsctl list Open_Vswitch
> +   echo "------ hv4 dump ------"
> +   as hv4 ovs-vsctl show
> +   as hv4 ovs-vsctl list Open_Vswitch
>
> -echo "Send traffic North to South"
> +   echo "Send traffic North to South"
>
> -sip=`ip_to_hex 172 31 0 10`
> -dip=`ip_to_hex 192 168 1 1`
> -test_ip vif-north f0f000000011 000001010207 $sip $dip vif11
> +   sip=`ip_to_hex 172 31 0 10`
> +   dip=`ip_to_hex 192 168 1 1`
> +   test_ip vif-north f0f000000011 000001010207 $sip $dip vif11
>
> -# Confirm that North to south traffic works fine.
> -OVN_CHECK_PACKETS([hv1/vif11-tx.pcap], [vif11.expected])
> +   # Confirm that North to south traffic works fine.
> +   OVN_CHECK_PACKETS([hv1/vif11-tx.pcap], [vif11.expected])
>
> -echo "Send traffic South to Nouth"
> -sip=`ip_to_hex 192 168 1 1`
> -dip=`ip_to_hex 172 31 0 10`
> -test_ip vif11 f00000000011 000001010203 $sip $dip vif-north
> +   echo "Send traffic South to Nouth"
> +   sip=`ip_to_hex 192 168 1 1`
> +   dip=`ip_to_hex 172 31 0 10`
> +   test_ip vif11 f00000000011 000001010203 $sip $dip vif-north
>
> -# Confirm that South to North traffic works fine.
> -OVN_CHECK_PACKETS_REMOVE_BROADCAST([hv4/vif-north-tx.pcap],
> [vif-north.expected])
> +   # Confirm that South to North traffic works fine.
> +   OVN_CHECK_PACKETS_REMOVE_BROADCAST([hv4/vif-north-tx.pcap],
> [vif-north.expected])
>
> -# Confirm that packets did not go out via tunnel port.
> -AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | grep table=33 | grep
> NXM_NX_TUN_METADATA0 | grep n_packets=0 | wc -l], [0], [[0
> +   # Confirm that packets did not go out via tunnel port.
> +   AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | grep table=33 | grep
> NXM_NX_TUN_METADATA0 | grep n_packets=0 | wc -l], [0], [[0
>  ]])
>
> -# Confirm that packet went out via localnet port
> -AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | grep table=65 | grep
> priority=150 | grep src=00:00:01:01:02:07 | grep n_packets=1 | wc -l], [0],
> [[1
> +   # Confirm that packet went out via localnet port
> +   AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | grep table=65 | grep
> priority=150 | grep src=00:00:01:01:02:07 | grep n_packets=1 | wc -l], [0],
> [[1
>  ]])
>
> -echo "----------- Post Traffic hv1 dump -----------"
> -as hv1 ovs-ofctl dump-flows br-int
> -as hv1 ovs-ofctl show br-phys
> -as hv1 ovs-appctl fdb/show br-phys
> +   echo "----------- Post Traffic hv1 dump -----------"
> +   as hv1 ovs-ofctl dump-flows br-int
> +   as hv1 ovs-ofctl show br-phys
> +   as hv1 ovs-appctl fdb/show br-phys
>
> -echo "----------- Post Traffic hv2 dump -----------"
> -as hv2 ovs-ofctl dump-flows br-int
> -as hv2 ovs-ofctl show br-phys
> -as hv2 ovs-appctl fdb/show br-phys
> +   echo "----------- Post Traffic hv2 dump -----------"
> +   as hv2 ovs-ofctl dump-flows br-int
> +   as hv2 ovs-ofctl show br-phys
> +   as hv2 ovs-appctl fdb/show br-phys
>
> -echo "----------- Post Traffic hv3 dump -----------"
> -as hv3 ovs-ofctl dump-flows br-int
> -as hv3 ovs-ofctl show br-phys
> -as hv3 ovs-appctl fdb/show br-phys
> +   echo "----------- Post Traffic hv3 dump -----------"
> +   as hv3 ovs-ofctl dump-flows br-int
> +   as hv3 ovs-ofctl show br-phys
> +   as hv3 ovs-appctl fdb/show br-phys
>
> -echo "----------- Post Traffic hv4 dump -----------"
> -as hv4 ovs-ofctl dump-flows br-int
> -as hv4 ovs-ofctl show br-phys
> -as hv4 ovs-appctl fdb/show br-phys
> +   echo "----------- Post Traffic hv4 dump -----------"
> +   as hv4 ovs-ofctl dump-flows br-int
> +   as hv4 ovs-ofctl show br-phys
> +   as hv4 ovs-appctl fdb/show br-phys
>
> -OVN_CLEANUP([hv1],[hv2],[hv3],[hv4])
> +   OVN_CLEANUP([hv1],[hv2],[hv3],[hv4])
>
> -AT_CLEANUP
> +   AT_CLEANUP])
> +
> +DVR_N_S_PING([geneve])
> +DVR_N_S_PING([vxlan])
>
>  AT_SETUP([ovn -- ARP lookup before learning])
>  AT_KEYWORDS([virtual ports])
> --
> 2.26.2
>
> _______________________________________________
> dev mailing list
> dev at openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
>


More information about the dev mailing list