[ovs-discuss] one patch was omitted to be pushed to branch-2.3---datapath: Fix recirc bug where skb is double freed

ZHANG Zhiming zhangzhiming at yunshan.net.cn
Sun May 24 08:46:04 UTC 2015


Hi,


Here you are. 
This bug is triggered by configuring bond_mode=banlace-tcp and lacp=active simultaneously.
The version of OVS is 2.3.1.


crash 7.0.2-6.el7
Copyright (C) 2002-2013  Red Hat, Inc.
Copyright (C) 2004, 2005, 2006, 2010  IBM Corporation
Copyright (C) 1999-2006  Hewlett-Packard Co
Copyright (C) 2005, 2006, 2011, 2012  Fujitsu Limited
Copyright (C) 2006, 2007  VA Linux Systems Japan K.K.
Copyright (C) 2005, 2011  NEC Corporation
Copyright (C) 1999, 2002, 2007  Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002  Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions.  Enter "help copying" to see the conditions.
This program has absolutely no warranty.  Enter "help warranty" for details.

GNU gdb (GDB) 7.6
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-unknown-linux-gnu"...

WARNING: kernel version inconsistency between vmlinux and dumpfile

      KERNEL: vmlinux                           
    DUMPFILE: vmcore  [PARTIAL DUMP]
        CPUS: 4
        DATE: Tue May 12 10:31:28 2015
      UPTIME: 00:02:02
LOAD AVERAGE: 0.43, 0.39, 0.16
       TASKS: 178
    NODENAME: centos125
     RELEASE: 3.10.0-123.el7.x86_64
     VERSION: #1 SMP Mon May 11 21:19:35 CST 2015
     MACHINE: x86_64  (3192 Mhz)
      MEMORY: 15.9 GB
       PANIC: "Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: ffffffffa04862ca"
         PID: 894
     COMMAND: "handler14"
        TASK: ffff8804059d2220  [THREAD_INFO: ffff8804064c4000]
         CPU: 3
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 894    TASK: ffff8804059d2220  CPU: 3   COMMAND: "handler14"
 #0 [ffff8804064c5658] machine_kexec at ffffffff8104105b
 #1 [ffff8804064c56b8] crash_kexec at ffffffff810ceec2
 #2 [ffff8804064c5788] panic at ffffffff815dab1e
 #3 [ffff8804064c5808] __stack_chk_fail at ffffffff8105dc3b
 #4 [ffff8804064c5818] execute_recirc at ffffffffa04862ca [openvswitch]
 #5 [ffff8804064c58c8] do_execute_actions at ffffffffa04871ba [openvswitch]
 #6 [ffff8804064c5968] ovs_execute_actions at ffffffffa04873f7 [openvswitch]
 #7 [ffff8804064c59a0] ovs_packet_cmd_execute at ffffffffa04897b6 [openvswitch]
 #8 [ffff8804064c59f8] genl_family_rcv_msg at ffffffff814ff268
 #9 [ffff8804064c5ac0] genl_rcv_msg at ffffffff814ff471
#10 [ffff8804064c5ae8] netlink_rcv_skb at ffffffff814fd529
#11 [ffff8804064c5b10] genl_rcv at ffffffff814fda58
#12 [ffff8804064c5b28] netlink_unicast at ffffffff814fcb4d
#13 [ffff8804064c5b70] netlink_sendmsg at ffffffff814fcf37
#14 [ffff8804064c5c08] sock_sendmsg at ffffffff814b6f30
#15 [ffff8804064c5d70] ___sys_sendmsg at ffffffff814b7369
#16 [ffff8804064c5f00] __sys_sendmsg at ffffffff814b8251
#17 [ffff8804064c5f70] sys_sendmsg at ffffffff814b82a2
#18 [ffff8804064c5f80] system_call_fastpath at ffffffff815f1619
    RIP: 00007fca637c97bd  RSP: 00007fca5bfe5748  RFLAGS: 00000206
    RAX: 000000000000002e  RBX: ffffffff815f1619  RCX: 00007fca4c0480e0
    RDX: 0000000000000000  RSI: 00007fca5bfc9d30  RDI: 0000000000000012
    RBP: 0000000000000002   R8: 0000000000000000   R9: 000000000000037e
    R10: 0000000000ab32b0  R11: 0000000000000293  R12: ffffffff814b82a2
    R13: ffff8804064c5f78  R14: 000000000000027d  R15: 0000000000aad9b0
    ORIG_RAX: 000000000000002e  CS: 0033  SS: 002b
crash> dis execute_recirc
0xffffffffa0486240 <execute_recirc>:    nopl   0x0(%rax,%rax,1)
0xffffffffa0486245 <execute_recirc+5>:  push   %rbp
0xffffffffa0486246 <execute_recirc+6>:  mov    %rsp,%rbp
0xffffffffa0486249 <execute_recirc+9>:  push   %r12
0xffffffffa048624b <execute_recirc+11>: lea    -0xa0(%rbp),%rcx
0xffffffffa0486252 <execute_recirc+18>: push   %rbx
0xffffffffa0486253 <execute_recirc+19>: mov    %rdi,%rbx
0xffffffffa0486256 <execute_recirc+22>: mov    %rbx,%rdx
0xffffffffa0486259 <execute_recirc+25>: sub    $0x90,%rsp
0xffffffffa0486260 <execute_recirc+32>: mov    0x4(%rsi),%edi
0xffffffffa0486263 <execute_recirc+35>: mov    0x30(%rbx),%rsi
0xffffffffa0486267 <execute_recirc+39>: mov    %gs:0x28,%rax
0xffffffffa0486270 <execute_recirc+48>: mov    %rax,-0x18(%rbp)
0xffffffffa0486274 <execute_recirc+52>: xor    %eax,%eax
0xffffffffa0486276 <execute_recirc+54>: callq  0xffffffffa048bc00 <ovs_flow_key_extract_recirc>
0xffffffffa048627b <execute_recirc+59>: test   %eax,%eax
0xffffffffa048627d <execute_recirc+61>: mov    %eax,%r12d
0xffffffffa0486280 <execute_recirc+64>: jne    0xffffffffa04862b8
0xffffffffa0486282 <execute_recirc+66>: lea    -0xa0(%rbp),%rsi
0xffffffffa0486289 <execute_recirc+73>: mov    $0x1,%edx
0xffffffffa048628e <execute_recirc+78>: mov    %rbx,%rdi
0xffffffffa0486291 <execute_recirc+81>: callq  0xffffffffa048a5b0 <ovs_dp_process_packet_with_key>
0xffffffffa0486296 <execute_recirc+86>: xor    %eax,%eax
0xffffffffa0486298 <execute_recirc+88>: mov    -0x18(%rbp),%rdx
0xffffffffa048629c <execute_recirc+92>: xor    %gs:0x28,%rdx
0xffffffffa04862a5 <execute_recirc+101>:        jne    0xffffffffa04862c5
0xffffffffa04862a7 <execute_recirc+103>:        add    $0x90,%rsp
0xffffffffa04862ae <execute_recirc+110>:        pop    %rbx
0xffffffffa04862af <execute_recirc+111>:        pop    %r12
0xffffffffa04862b1 <execute_recirc+113>:        pop    %rbp
0xffffffffa04862b2 <execute_recirc+114>:        retq   
0xffffffffa04862b3 <execute_recirc+115>:        nopl   0x0(%rax,%rax,1)
0xffffffffa04862b8 <execute_recirc+120>:        mov    %rbx,%rdi
0xffffffffa04862bb <execute_recirc+123>:        callq  0xffffffff814c06f0 <kfree_skb>
0xffffffffa04862c0 <execute_recirc+128>:        mov    %r12d,%eax
0xffffffffa04862c3 <execute_recirc+131>:        jmp    0xffffffffa0486298
0xffffffffa04862c5 <execute_recirc+133>:        callq  0xffffffff8105dc20 <__stack_chk_fail>
0xffffffffa04862ca <execute_recirc+138>:        nopw   0x0(%rax,%rax,1)




ZHANG Zhiming
Yunshan Networks

From: Andy Zhou
Date: 2015-05-23 04:41
To: Alex Wang
CC: zhangzhiming; discuss
Subject: Re: [ovs-discuss] one patch was omitted to be pushed to branch-2.3---datapath: Fix recirc bug where skb is double freed
Hi, Jeremy,

Sorry for the delay. I don't think this patch is required for branch
2.3. As you may have noticed, this part of code
is different on branch 2.3.  And it seems to work on my test.

Do you have the core dump from kernel crash? If yes, would you please
post the back trace?

Thanks,

Andy

On Sun, May 17, 2015 at 9:41 AM, Alex Wang <alexw at nicira.com> wrote:
> Fwd to Andy,~
>
> On Sun, May 17, 2015 at 4:29 AM, zhangzhiming <zhangzhiming at yunshan.net.cn>
> wrote:
>>
>> Hi,
>>
>> I found one patch was omitted to be pushed to branch-2.3, which leads to
>> double freed skb.
>> Could someone to confirm the patch and submit it to branch-2.3?
>> Thanks!
>>
>> Here is the patch information:
>>
>>
>> commit 867e37ba00091b3e319c4c47c1598f1ae84dd32e
>> Author: Andy Zhou <azhou at nicira.com>
>> Date:   Mon Aug 25 15:18:19 2014 -0700
>>
>>     datapath: Fix recirc bug where skb is double freed.
>>
>>     If recirc action is the last action of a action list, the SKB triggers
>>     the recirc will be freed twice. This patch fixes this bug.
>>
>>     Reported-by: Justin Pettit <jpettit at nicira.com>
>>     Signed-off-by: Andy Zhou <azhou at nicira.com>
>>
>> diff --git a/datapath/actions.c b/datapath/actions.c
>> index ad22467..7f25553 100644
>> --- a/datapath/actions.c
>> +++ b/datapath/actions.c
>> @@ -809,7 +809,16 @@ static int execute_recirc(struct datapath *dp, struct
>> sk_buff *skb,
>>                           const struct nlattr *a, int rem)
>>  {
>>         struct sw_flow_key recirc_key;
>> -       int err;
>> +
>> +       if (!is_skb_flow_key_valid(skb)) {
>> +               int err;
>> +
>> +               err = ovs_flow_key_update(skb, OVS_CB(skb)->pkt_key);
>> +               if (err)
>> +                       return err;
>> +
>> +       }
>> +       BUG_ON(!is_skb_flow_key_valid(skb));
>>
>>         if (!last_action(a, rem)) {
>>                 /* Recirc action is the not the last action
>> @@ -820,19 +829,9 @@ static int execute_recirc(struct datapath *dp, struct
>> sk_buff *skb,
>>                  * continue on with the rest of the action list. */
>>                 if (!skb)
>>                         return 0;
>> -       }
>>
>> -       if (!is_skb_flow_key_valid(skb)) {
>> -               err = ovs_flow_key_update(skb, OVS_CB(skb)->pkt_key);
>> -               if (err) {
>> -                       kfree_skb(skb);
>> -                       return err;
>> -               }
>> -       }
>> -       BUG_ON(!is_skb_flow_key_valid(skb));
>> -
>> -       if (!last_action(a, rem))
>>                 flow_key_clone(skb, &recirc_key);
>> +       }
>>
>>         flow_key_set_recirc_id(skb, nla_get_u32(a));
>>         ovs_dp_process_packet(skb, true);
>> @@ -897,6 +896,12 @@ static int do_execute_actions(struct datapath *dp,
>> struct sk_buff *skb,
>>
>>                 case OVS_ACTION_ATTR_RECIRC:
>>                         err = execute_recirc(dp, skb, a, rem);
>> +                       if (last_action(a, rem)) {
>> +                               /* If this is the last action, the skb has
>> +                                * been consumed or freed.
>> +                                * Return immediately. */
>> +                               return err;
>> +                       }
>>                         break;
>>
>>                 case OVS_ACTION_ATTR_SET:
>>
>> ________________________________
>> Jeremy Zhang
>>
>> _______________________________________________
>> discuss mailing list
>> discuss at openvswitch.org
>> http://openvswitch.org/mailman/listinfo/discuss
>>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://openvswitch.org/pipermail/ovs-discuss/attachments/20150524/350af1aa/attachment-0002.html>


More information about the discuss mailing list