[ovs-discuss] ovs-vswtichd crash with DPDK on qede
Bill Kozak
bill.kozak at calian.com
Wed Oct 14 22:18:44 UTC 2020
Hello,
I'm seeing a consistent crash when trying to run an iperf test using qede interface and DPDK. Running one iperf test on its own works fine but as soon as I attempt to run two in parallel through the same interface the crash occurs.
OVS configuration is:
ovs-vsctl set Open_vSwitch . other_config:dpdk-socket-mem="4096,0"
ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true
ovs-vsctl --timeout 10 set Open_vSwitch . other_config:max-idle=30000
ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x1e
ovs-vsctl add-port br0 dpdk-p0 -- set Interface dpdk-p0 type=dpdk \
options:dpdk-devargs=0000:45:00.0 \
options:n_rxq=1 other_config:pmd-rxq-affinity="0:1" \
options:flow-ctrl-autoneg=true \
ofport=1
ovs-vsctl -- set Interface br1 mtu_request=1500
ovs-vsctl add-port br1 dpdk-p1 -- set Interface dpdk-p1 type=dpdk \
options:dpdk-devargs=0000:45:00.1 \
options:n_rxq=3 other_config:pmd-rxq-affinity="0:2,1:3,2:4" options:n_rxq_desc=2048 options:n_txq_desc=2048 \
options:flow-ctrl-autoneg=true \
mtu_request=1504 ofport=1
ip a add 192.168.190.77/25 dev br0
ip a add 192.168.190.200/25 dev br1
rm -f /dev/hugepages/*
ip link set br0 up
ip link set br1 up
ip route add default via 192.168.190.1 dev br0
ovs-vsctl set port dpdk-p1 qos=@qos -- \
--id=@qos create qos type=trtcm-policer other-config:cir=250000000 other-config:cbs=30080 other_config:eir=12500000 other-config:ebs=30080 queues:1=@queue1 queues:2=@queue2 queues:3=@queue3 -- \
--id=@queue1 create queue other-config:cir=220000000 other-config:eir=10000000 other-config:cbs=30080 other-config:ebs=30080 -- \
--id=@queue2 create queue other-config:cir=20000000 other-config:eir=220000000 other-config:cbs=30080 other-config:ebs=30080 -- \
--id=@queue3 create queue other-config:cir=10000000 other-config:eir=10000000 other-config:cbs=30080 other-config:ebs=30080
ovs-ofctl del-flows br1
ovs-ofctl add-flow br1 'table=0,priority=1,ip,ip_dst=0.0.0.201/0.0.0.255,actions=set_queue:0,resubmit(,1)'
ovs-ofctl add-flow br1 'table=0,priority=1,ip,ip_dst=0.0.0.202/0.0.0.255,actions=set_queue:1,resubmit(,1)'
ovs-ofctl add-flow br1 'table=0,priority=1,ip,ip_dst=0.0.0.203/0.0.0.255,actions=set_queue:2,resubmit(,1)'
ovs-ofctl add-flow br1 'table=0,priority=0,ip,actions=resubmit(,1)'
ovs-ofctl add-flow br1 'table=0,priority=0,arp,actions=all'
ovs-ofctl add-flow br1 'table=0,mpls,actions=pop_mpls:0x0800,local'
ovs-ofctl add-flow br1 'table=1,ip,actions=move:ip_src[0..7]->reg0[0..7],push_mpls:0x8847,move:reg0[0..7]->mpls_label[0..7],set_field:2->mpls_tc,set_field:255->mpls_ttl,1'
The crash produces the following back-trace:
Process 2800 (ovs-vswitchd) of user 0 dumped core.
Stack trace of thread 2800:
#0 0x00007f2f94f3470f raise (libc.so.6)
#1 0x00007f2f94f1eb25 abort (libc.so.6)
#2 0x00007f2f94f77897 __libc_message (libc.so.6)
#3 0x00007f2f94f7dfdc malloc_printerr (libc.so.6)
#4 0x00007f2f94f7fe44 _int_free (libc.so.6)
#5 0x000000000050881a dp_packet_delete (ovs-vswitchd)
#6 0x000000000050883e dp_packet_delete_batch (ovs-vswitchd)
#7 0x000000000050a114 dp_packet_delete_batch (ovs-vswitchd)
#8 0x000000000047d243 netdev_send (ovs-vswitchd)
#9 0x0000000000458910 dp_netdev_pmd_flush_output_on_port (ovs-vswitchd)
#10 0x0000000000458a1a dp_netdev_pmd_flush_output_packets (ovs-vswitchd)
#11 0x000000000045d705 dp_netdev_process_rxq_port (ovs-vswitchd)
#12 0x000000000045deb5 dpif_netdev_run (ovs-vswitchd)
#13 0x0000000000429e10 type_run (ovs-vswitchd)
#14 0x0000000000418b10 ofproto_type_run (ovs-vswitchd)
#15 0x000000000040c101 bridge_run__ (ovs-vswitchd)
#16 0x0000000000410f3d bridge_run (ovs-vswitchd)
#17 0x0000000000409802 main (ovs-vswitchd)
#18 0x00007f2f94f206a3 __libc_start_main (libc.so.6)
#19 0x000000000040a2de _start (ovs-vswitchd)
Stack trace of thread 2959:
#0 0x00007f2f94fedf21 __poll (libc.so.6)
#1 0x00000000004dd2e9 time_poll (ovs-vswitchd)
#2 0x00000000004cdb35 poll_block (ovs-vswitchd)
#3 0x000000000052e9b3 clean_thread_main (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 3069:
#0 0x000000000045d981 pmd_thread_main (ovs-vswitchd)
#1 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#2 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#3 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 3071:
#0 0x000000000045d5d7 rte_rdtsc (ovs-vswitchd)
#1 0x000000000045da10 pmd_thread_main (ovs-vswitchd)
#2 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#3 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#4 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 3072:
#0 0x00007ffd370139c9 n/a (linux-vdso.so.1)
#1 0x00007ffd37013bcc __vdso_clock_gettime (linux-vdso.so.1)
#2 0x00007f2f95006ea2 __clock_gettime (libc.so.6)
#3 0x00000000004dceba xclock_gettime (ovs-vswitchd)
#4 0x00000000004dcefa time_timespec__ (ovs-vswitchd)
#5 0x00000000004dd421 time_usec__ (ovs-vswitchd)
#6 0x000000000045d95f pmd_thread_ctx_time_update (ovs-vswitchd)
#7 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#8 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#9 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 3070:
#0 0x00000000004dced3 time_timespec__ (ovs-vswitchd)
#1 0x00000000004dd421 time_usec__ (ovs-vswitchd)
#2 0x000000000045d95f pmd_thread_ctx_time_update (ovs-vswitchd)
#3 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#4 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#5 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2803:
#0 0x00007f2f94fc5238 __nanosleep (libc.so.6)
#1 0x00007f2f94fc513e sleep (libc.so.6)
#2 0x00000000004e431e xsleep (ovs-vswitchd)
#3 0x0000000000506ef7 dpdk_watchdog (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2801:
#0 0x00007f2f94ff91b7 epoll_wait (libc.so.6)
#1 0x00007f2f97560a44 eal_intr_thread_main (librte_eal.so.20.0)
#2 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#3 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2804:
#0 0x00007f2f94fedf21 __poll (libc.so.6)
#1 0x00000000004dd2e9 time_poll (ovs-vswitchd)
#2 0x00000000004cdb35 poll_block (ovs-vswitchd)
#3 0x00000000004bd334 ovsrcu_postpone_thread (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2967:
#0 0x00007f2f94fedf21 __poll (libc.so.6)
#1 0x00000000004dd2e9 time_poll (ovs-vswitchd)
#2 0x00000000004cdb35 poll_block (ovs-vswitchd)
#3 0x000000000043589b udpif_upcall_handler (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2960:
#0 0x00007f2f94fedf21 __poll (libc.so.6)
#1 0x00000000004dd2e9 time_poll (ovs-vswitchd)
#2 0x00000000004cdb35 poll_block (ovs-vswitchd)
#3 0x0000000000469dce ipf_clean_thread_main (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2970:
#0 0x00007f2f94fedf21 __poll (libc.so.6)
#1 0x00000000004dd2e9 time_poll (ovs-vswitchd)
#2 0x00000000004cdb35 poll_block (ovs-vswitchd)
#3 0x000000000043589b udpif_upcall_handler (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Stack trace of thread 2976:
#0 0x00007f2f94fedf21 __poll (libc.so.6)
#1 0x00000000004dd2e9 time_poll (ovs-vswitchd)
#2 0x00000000004cdb35 poll_block (ovs-vswitchd)
#3 0x000000000043589b udpif_upcall_handler (ovs-vswitchd)
#4 0x00000000004beba0 ovsthread_wrapper (ovs-vswitchd)
#5 0x00007f2f95d9f2de start_thread (libpthread.so.0)
#6 0x00007f2f94ff8e83 __clone (libc.so.6)
Any idea as to what I might be doing wrong? Or is this a bug?
Thanks,
Bill Kozak
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.openvswitch.org/pipermail/ovs-discuss/attachments/20201014/1a1b59f1/attachment-0001.html>
More information about the discuss
mailing list