[ovs-discuss] port group reference large number of ports cause ovn-northd and ovn-nbctl deamon cpu 100% forever

贾文涛 wentao.jia at easystack.cn
Tue Jul 20 09:51:31 UTC 2021


Hi,all


ovn scale test, created 47194 logical switch ports, port group reference all the switch ports in one row. cause ovn-northd and ovn-nbctl deamon cpu 100% forever


gdb info:
()[root at ovn-northd-0 /]# date
Tue Jul 20 11:07:34 CST 2021
()[root at ovn-northd-0 /]# gdb attach 41
......
(gdb) bt
#0  0x00007f4513cedb35 in malloc () from /lib64/libc.so.6
#1  0x0000564b7cdc0188 in xmalloc__ (size=size at entry=48) at lib/util.c:137
#2  0x0000564b7cdc01ab in xmalloc (size=size at entry=48) at lib/util.c:172
#3  0x0000564b7cdaa9c2 in ovsdb_idl_get_row_arc (src=src at entry=0x564b93325d40, dst_table_class=dst_table_class at entry=0x564b7d175ad0 <nbrec_table_classes+816>, dst_uuid=0x564b895bb750) at lib/ovsdb-idl.c:2328
#4  0x0000564b7cd25eaa in nbrec_port_group_parse_ports (row_=0x564b93325d40, datum=0x564b93325ea8) at lib/ovn-nb-idl.c:33264
#5  0x0000564b7cda5b06 in ovsdb_idl_row_parse (row=row at entry=0x564b93325d40) at lib/ovsdb-idl.c:1656
#6  0x0000564b7cda77e9 in ovsdb_idl_row_reparse_backrefs (row=row at entry=0x564b9316fcb0) at lib/ovsdb-idl.c:2074
#7  0x0000564b7cda7a07 in ovsdb_idl_insert_row (row=row at entry=0x564b9316fcb0, data=0x564b8ae27f90) at lib/ovsdb-idl.c:2230
#8  0x0000564b7cda8ee2 in ovsdb_idl_process_update (ru=0x564b8af95470, table=0x564b7d423690) at lib/ovsdb-idl.c:1464
#9  ovsdb_idl_parse_update__ (du=0x564b8de0ea30, idl=0x564b7d422930) at lib/ovsdb-idl.c:1336
#10 ovsdb_idl_parse_update (update=0x564bdb13b7e8, idl=0x564b7d422930) at lib/ovsdb-idl.c:1375
#11 ovsdb_idl_run (idl=<optimized out>) at lib/ovsdb-idl.c:444
#12 0x0000564b7cdad89c in ovsdb_idl_loop_run (loop=loop at entry=0x7ffcfcff8d80) at lib/ovsdb-idl.c:4122
#13 0x0000564b7ccf9747 in main (argc=9, argv=0x7ffcfcff8f68) at northd/ovn-northd.c:14445
(gdb) frame 9
#9  ovsdb_idl_parse_update__ (du=0x564b8de0ea30, idl=0x564b7d422930) at lib/ovsdb-idl.c:1336
1336                switch (ovsdb_idl_process_update(table, ru)) {
(gdb) info local
ru = 0x564b8af95470
j = 4247
tu = 0x564b8af6fd68
table = 0x564b7d423690
i = 9
(gdb) print du->n
$1 = 10
(gdb) print du->table_updates[0]
$2 = {table_name = 0x564b907a15e0 "NB_Global", row_updates = 0x564b88cf2f90, n = 1}
(gdb) print du->table_updates[1]
$3 = {table_name = 0x564b94d13f60 "ACL", row_updates = 0x564b8af6fd90, n = 9}
(gdb) print du->table_updates[2]
$4 = {table_name = 0x564b7f9a7d10 "Logical_Switch", row_updates = 0x564b8af6fec0, n = 237}
(gdb) print du->table_updates[3]
$5 = {table_name = 0x564bda9c4c90 "HA_Chassis", row_updates = 0x564b8af71c70, n = 59}
(gdb) print du->table_updates[4]
$6 = {table_name = 0x564bdb893c60 "HA_Chassis_Group", row_updates = 0x564b90dad0f0, n = 1}
(gdb) print du->table_updates[5]
$7 = {table_name = 0x564c5ccb7b10 "Logical_Router", row_updates = 0x564b86f8ddd0, n = 1}


(gdb) print du->table_updates[6]
$9 = {table_name = 0x564bdb284750 "Port_Group", row_updates = 0x564b8f658470, n = 3}
(gdb) print du->table_updates[7]
$10 = {table_name = 0x564b8af6f920 "Connection", row_updates = 0x564b8769f0e0, n = 1}
(gdb) print du->table_updates[8]
$11 = {table_name = 0x564c38e30d10 "DHCP_Options", row_updates = 0x564b8af723e0, n = 237}
(gdb) print du->table_updates[9]
$12 = {table_name = 0x564bd8be5ed0 "Logical_Switch_Port", row_updates = 0x564b8af74190, n = 47194}
(gdb) 
$13 = {table_name = 0x564bd8be5ed0 "Logical_Switch_Port", row_updates = 0x564b8af74190, n = 47194}
(gdb) print tu->n
$14 = 47194
(gdb) print j
$15 = 4247


gdb again:
()[root at ovn-northd-0 /]# date
Tue Jul 20 11:26:25 CST 2021
()[root at ovn-northd-0 /]# 
()[root at ovn-northd-0 /]# gdb attach 41
6.6-1.el7.centos.es.x86_64 zlib-1.2.7-18.el7.centos.es.x86_64
(gdb) bt
#0  xmalloc__ (size=size at entry=48) at lib/util.c:136
#1  0x0000564b7cdc01ab in xmalloc (size=size at entry=48) at lib/util.c:172
#2  0x0000564b7cdaa9c2 in ovsdb_idl_get_row_arc (src=src at entry=0x564b7d710c40, dst_table_class=dst_table_class at entry=0x564b7d175ad0 <nbrec_table_classes+816>, dst_uuid=0x564b8dc1a8b0) at lib/ovsdb-idl.c:2328
#3  0x0000564b7cd25eaa in nbrec_port_group_parse_ports (row_=0x564b7d710c40, datum=0x564b93d9df48) at lib/ovn-nb-idl.c:33264
#4  0x0000564b7cda5b06 in ovsdb_idl_row_parse (row=row at entry=0x564b7d710c40) at lib/ovsdb-idl.c:1656
#5  0x0000564b7cda77e9 in ovsdb_idl_row_reparse_backrefs (row=row at entry=0x564b946e3470) at lib/ovsdb-idl.c:2074
#6  0x0000564b7cda7a07 in ovsdb_idl_insert_row (row=row at entry=0x564b946e3470, data=0x564bd7f1cc60) at lib/ovsdb-idl.c:2230
#7  0x0000564b7cda8ee2 in ovsdb_idl_process_update (ru=0x564b8af96f70, table=0x564b7d423690) at lib/ovsdb-idl.c:1464
#8  ovsdb_idl_parse_update__ (du=0x564b8de0ea30, idl=0x564b7d422930) at lib/ovsdb-idl.c:1336
#9  ovsdb_idl_parse_update (update=0x564bdb13b7e8, idl=0x564b7d422930) at lib/ovsdb-idl.c:1375
#10 ovsdb_idl_run (idl=<optimized out>) at lib/ovsdb-idl.c:444
#11 0x0000564b7cdad89c in ovsdb_idl_loop_run (loop=loop at entry=0x7ffcfcff8d80) at lib/ovsdb-idl.c:4122
#12 0x0000564b7ccf9747 in main (argc=9, argv=0x7ffcfcff8f68) at northd/ovn-northd.c:14445
(gdb) frame 8
#8  ovsdb_idl_parse_update__ (du=0x564b8de0ea30, idl=0x564b7d422930) at lib/ovsdb-idl.c:1336
1336                switch (ovsdb_idl_process_update(table, ru)) {
(gdb) info local
ru = 0x564b8af96f70
j = 4463
tu = 0x564b8af6fd68
table = 0x564b7d423690


ovsdb_idl_parse_update__ take long time to update logcial ports, in twice gdb,gdb inteval 19minuts,4463-4247=216 logical swich ports updated。


nbrec_port_group_parse_ports update all reference in each loop,which takes a lot of time
(gdb) print datum->n
$3 = 47194








-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.openvswitch.org/pipermail/ovs-discuss/attachments/20210720/712f6de2/attachment-0001.html>


More information about the discuss mailing list