[ovs-discuss] Problems with proxy ARP

Fri May 1 18:48:22 UTC 2020

Hi,

I'm having an issue with proxy ARP

I have a VM ca-rain06-vmovs-3 (192.16.1.6) and I want to use an
unassigned IP (192.16.1.106) in that VM's subnet to "map" to an
underlay IP 253.255.0.33, such that when I ping 192.16.1.106
pkts sent from ca-rain06-vmovs-3 to 192.16.1.106 have the dst IP
192.16.1.106 changed to 253.255.0.33 by a gateway and are sent
onto the underlay by that gateway.

The gateway is lr_vcn3, and it has the following NAT and routes
(253.255.0.1) is the default router in the underlay.

# ovn-nbctl lr-route-list lr_vcn3
IPv4 Routes
              192.16.1.106               253.255.0.1 dst-ip 
lr_vcn3_gw-ls_external_vcn3
                 0.0.0.0/0               253.255.0.1 dst-ip 
lr_vcn3_gw-ls_external_vcn3
[root at ca-rain01 ~]# ovn-nbctl lr-nat-list lr_vcn3
TYPE             EXTERNAL_IP        EXTERNAL_PORT LOGICAL_IP            
EXTERNAL_MAC         LOGICAL_PORT
dnat             253.255.0.33                        192.16.1.106
snat             253.255.2.2                         192.16.1.6

But what I see is an ARP in the overlay looking for the MAC of
192.16.1.106 and nobody responds. I would have thought that lr_vcn3
would respond as it has a route for 192.16.1.106

I tried adding proxy ARP to no avail on various ports:

ovn-nbctl set Logical_Switch_Port vcn3_subnet1-lr_vcn3 
options:arp_proxy=true
ovn-nbctl set Logical_router_port lr_vcn3-vcn3_subnet1 
options:arp_proxy=true
ovn-nbctl set Logical_Switch_Port ls_external_vcn3-lr_vcn3_gw 
options:arp_proxy=true
ovn-nbctl set Logical_router_port lr_vcn3_gw-ls_external_vcn3 
options:arp_proxy=true

I think what I need is an entry in  Ingress Table 11 for the vcn3_subnet1
Logical switch port that replies to the ARP request with the router MAC

Here is my setup, and North Bound config

Any thoughts

Thanks

# Setup gateway
#
ovn-nbctl ls-add ls_external_vcn3
ovn-nbctl lsp-add ls_external_vcn3 ln-ls_external_vcn3
ovn-nbctl lsp-set-type ln-ls_external_vcn3 localnet
ovn-nbctl lsp-set-addresses ln-ls_external_vcn3 unknown
ovn-nbctl lsp-set-options ln-ls_external_vcn3 network_name=physnet

# Create a distributed router port (40:44:00:00:10:10 253.255.31.3)
#
ovn-nbctl lrp-add lr_vcn3 lr_vcn3_gw-ls_external_vcn3 40:44:00:00:10:10 
253.255.31.3/16
ovn-nbctl lsp-add ls_external_vcn3 ls_external_vcn3-lr_vcn3_gw
ovn-nbctl lsp-set-type ls_external_vcn3-lr_vcn3_gw router
ovn-nbctl lsp-set-addresses ls_external_vcn3-lr_vcn3_gw router
ovn-nbctl lsp-set-options ls_external_vcn3-lr_vcn3_gw 
router-port=lr_vcn3_gw-ls_external_vcn3

# Tell the lr_vcn3 to use the underlay default router to get to the world
#
ovn-nbctl lr-route-add lr_vcn3 "0.0.0.0/0" 253.255.0.1 
lr_vcn3_gw-ls_external_vcn3

# Add specific route for 192.16.1.106
# We want lr_vcn3 to respond to ARPs for 192.16.1.106
# Then DNAT pkts with dst of 192.16.1.106 to have a dst of 253.255.0.33
#
ovn-nbctl lr-route-add lr_vcn3 "192.16.1.106" 253.255.0.1 
lr_vcn3_gw-ls_external_vcn3

# Add NAT rules
#
# SNAT using IP 253.255.2.2 so pkts from with ca-rain06-vmovs-3 src IP
# get an underlay IP of 253.255.2.2
ovn-nbctl lr-nat-add lr_vcn3 snat 253.255.2.2 192.16.1.6

# DNAT using  253.255.0.33
# So pkts sent to 192.16.1.106 get dst replaced with 253.255.0.33
# And pkts received from underlay with dst get dst replaced with
# 253.255.0.33
#
ovn-nbctl lr-nat-add lr_vcn3 dnat 253.255.0.33 192.16.1.106

# Schedule the distributed gateway port
#
ovn-nbctl lrp-set-gateway-chassis lr_vcn3_gw-ls_external_vcn3 ca-rain05 15
ovn-nbctl lrp-set-gateway-chassis lr_vcn3_gw-ls_external_vcn3 ca-rain06 20
ovn-nbctl lrp-set-gateway-chassis lr_vcn3_gw-ls_external_vcn3 ca-rain17 10

# Test
# From overlay VM ca-rain06-vmovs-3 (192.16.1.6)
# ping 192.16.1.106
[ca-rain06-vmovs-3 ~]# ping -c1 192.16.1.106

# tcpdump shows lr_vcn3_gw does not respond to ARP request
# Would have thought it would as it has a route for 192.16.1.106

  00:05:05.882784 52:54:00:02:55:96 > ff:ff:ff:ff:ff:ff, ethertype ARP 
(0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 
192.16.1.106 tell 192.16.1.6, length 28
     0x0000:  ffff ffff ffff 5254 0002 5596 0806 0001
     0x0010:  0800 0604 0001 5254 0002 5596 c010 0106
     0x0020:  0000 0000 0000 c010 016a
  00:00:01.061217 52:54:00:02:55:96 > ff:ff:ff:ff:ff:ff, ethertype ARP 
(0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 
192.16.1.106 tell 192.16.1.6, length 28
     0x0000:  ffff ffff ffff 5254 0002 5596 0806 0001
     0x0010:  0800 0604 0001 5254 0002 5596 c010 0106
     0x0020:  0000 0000 0000 c010 016a

# My North Bound DB:
switch 6631a7a5-018f-496c-b8d3-af17d5e997ae (ls_external_vcn3)
     port ln-ls_external_vcn3
         type: localnet
         addresses: ["unknown"]
     port ls_external_vcn3-lr_vcn3_gw
         type: router
         router-port: lr_vcn3_gw-ls_external_vcn3
switch 2f8cce6f-b41c-4a90-8785-9f56d808ac14 (ls_vcn3)
     port 284195d2-9280-4334-900e-571ecd00327a
         addresses: ["52:54:00:02:55:96 192.16.1.6"]
     port 269089c4-9464-41ec-9f63-6b3804b34b07
         addresses: ["52:54:00:30:38:35 192.16.1.5"]
     port vcn3_subnet1-lr_vcn3
         type: router
         addresses: ["40:44:00:00:00:60"]
         router-port: lr_vcn3-vcn3_subnet1
router 36c64fdf-4842-4543-95bc-9bfe731af4d3 (lr_vcn3)
     port lr_vcn3-vcn3_subnet1
         mac: "40:44:00:00:00:60"
         networks: ["192.16.1.1/24"]
     port lr_vcn3_gw-ls_external_vcn3
         mac: "40:44:00:00:10:10"
         networks: ["253.255.31.3/16"]
         gateway chassis: [ca-rain06 ca-rain05 ca-rain17]
     nat 5e179df6-15c7-403e-99a7-1e453cbc0493
         external ip: "253.255.2.2"
         logical ip: "192.16.1.6"
         type: "snat"
     nat e4dab2ac-7be0-4127-818e-d24b71c14f4b
         external ip: "253.255.0.33"
         logical ip: "192.16.1.106"
         type: "dnat"

MAC bindings
--------------
_uuid               : d82e70a0-2212-4566-a929-b7cf3c091e8c
datapath            : db21ed8c-16a5-4afb-aa51-5bbaae5bda29
ip                  : "192.16.1.6"
logical_port        : lr_vcn3-vcn3_subnet1
mac                 : "52:54:00:02:55:96"

_uuid               : fe33814e-8001-4697-ad7e-087b1f4ed98d
datapath            : db21ed8c-16a5-4afb-aa51-5bbaae5bda29
ip                  : "192.16.1.5"
logical_port        : lr_vcn3-vcn3_subnet1
mac                 : "52:54:00:30:38:35"

_uuid               : aa2f0e96-5ec7-452c-8a5f-aaec5d5b6c7c
datapath            : db21ed8c-16a5-4afb-aa51-5bbaae5bda29
ip                  : "::"
logical_port        : lr_vcn3-vcn3_subnet1
mac                 : "00:00:00:00:00:00"

# Try configure proxy ARP - But on what?
#
# vcn3_subnet1-lr_vcn3 - Logical switch port on ls_vcn3 that connects to 
Logical Router lr_vcn3
# lr_vcn3-vcn3_subnet1 - Logical router port on lr_vcn3 that connects to 
Logical switch ls_vcn3
# ls_external_vcn3-lr_vcn3_gw - LS port on ls_external_vcn3 that 
connects to the gateway
# lr_vcn3_gw-ls_external_vcn3 - LR gateway port
#

# No joy - with vcn3_subnet1-lr_vcn3
ovn-nbctl set Logical_Switch_Port vcn3_subnet1-lr_vcn3 
options:arp_proxy=true

# No joy - with lr_vcn3-vcn3_subnet1
ovn-nbctl set Logical_router_port lr_vcn3-vcn3_subnet1 
options:arp_proxy=true

# No Joy with - ls_external_vcn3-lr_vcn3_gw
ovn-nbctl set Logical_Switch_Port ls_external_vcn3-lr_vcn3_gw 
options:arp_proxy=true

# lr_vcn3_gw-ls_external_vcn3
ovn-nbctl set Logical_router_port lr_vcn3_gw-ls_external_vcn3 
options:arp_proxy=true