[ovs-discuss] ovs-vswitchd process huge memory consumption
Ben Pfaff
blp at ovn.org
Fri Feb 22 23:23:43 UTC 2019
It's odd that two people would notice the same problem at the same time
on old branches.
Anyway, I'm attaching the scripts I have. They are rough. The second
one invokes the first one as a subprocess; it is probably the one you
should use. I might have to walk you through how to use it, or write
better documentation myself. Anyway, it should be a start.
On Wed, Feb 20, 2019 at 07:15:26PM +0400, Oleg Bondarev wrote:
> Ah, sorry, I missed "ovs-vswitchd memory consumption behavior" thread.
> So I guess I'm also interested in the scripts for analyzing the heap in a
> core dump :)
>
> Thanks,
> Oleg
>
> On Wed, Feb 20, 2019 at 7:00 PM Oleg Bondarev <obondarev at mirantis.com>
> wrote:
>
> > Hi,
> >
> > OVS 2.8.0, uptime 197 days, 44G RAM.
> > ovs-appctl memory/show reports:
> > "handlers:35 ofconns:4 ports:73 revalidators:13 rules:1099 udpif
> > keys:686"
> >
> > Similar data on other nodes of the OpenStack cluster.
> > Seems usage grows gradually over time.
> > Are there any known issues, like
> > http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-14970?
> > Please advise on the best way to debug.
> >
> > Thanks,
> > Oleg
> >
> >
> _______________________________________________
> discuss mailing list
> discuss at openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
-------------- next part --------------
#! /usr/bin/perl
use strict;
#use warnings;
use Fcntl;
my ($core, $start, $len, $vma) = @ARGV;
$start = oct($start) if $start =~ /^0/;
$len = oct($len) if $len =~ /^0/;
$vma = oct($vma) if defined($vma) && ($vma =~ /^0/);
open (CORE, '<', $core) or die "$core: open: $!\n";
my $bits = 64;
my $bytes = $bits / 8;
my $template;
if ($bits == 64) {
$template = 'QQ';
} elsif ($bits == 32) {
$template = 'LL';
} else {
die;
}
seek (CORE, $start, Fcntl::SEEK_SET) or die "$core: seek: $!\n";
my $ofs = $start;
my $n = 0;
my $n_inuse = 0;
my $last_size = 0;
my $last_ofs = 0;
my $bytes_inuse = 0;
my %hist;
my %loc;
while ($len > $bytes * 2) {
my $header;
read (CORE, $header, $bytes * 2) or die "$core: read: $!\n";
$ofs += $bytes * 2;
my ($prev_size, $size) = unpack($template, $header);
#print "$prev_size, $size\n";
die if $size < $bytes * 2;
die "$size byte block at offset $ofs" if $size > 10485760;
$n++;
if ($size & 1) {
#printf "0x%x, %d used\n", ($last_ofs - $start) + $vma, $last_size - $bytes * 2;
$n_inuse++;
my $sample = !$hist{$last_size}++ || int(rand($hist{$last_size})) < 1;
$loc{$last_size} = $last_ofs if $sample;
$bytes_inuse += $last_size;
} else {
#printf "0x%x, %u free\n", ($last_ofs - $start) + $vma, $last_size - $bytes * 2;
}
$last_size = $size & ~7;
$last_ofs = $ofs;
$len -= $size & ~7;
my $content_len = ($size & ~7) - $bytes * 2;
my $content;
read (CORE, $content, $content_len);
$ofs += $content_len;
}
print "$n blocks, $n_inuse blocks in use, $bytes_inuse bytes in use\n";
print "In use:\n";
foreach my $size (sort { $a <=> $b }(keys(%hist))) {
my $realsize = $size - 2 * $bytes;
printf "%6d bytes x %6d = %10d (0x%x)\n",
$realsize, $hist{$size}, $realsize * $hist{$size}, $loc{$size};
if ($realsize == 96) {
printf "starting at vaddr 0x%x:\n", $loc{$size} - $start + $vma;
system("hd -n $realsize -s $loc{$size} $core");
}
}
-------------- next part --------------
#! /usr/bin/perl
use strict;
#use warnings;
use Fcntl;
my ($core) = @ARGV;
# Notes:
#
# - Each heap begins with a 'heap_info' structure, which is 32 bytes long
# on 64-bit systems:
#
# typedef struct _heap_info
# {
# mstate ar_ptr; /* Arena for this heap. */
# struct _heap_info *prev; /* Previous heap. */
# size_t size; /* Current size in bytes. */
# size_t mprotect_size; /* Size in bytes that has been mprotected
# PROT_READ|PROT_WRITE. */
# /* Make sure the following data is properly aligned, particularly
# that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
# MALLOC_ALIGNMENT. */
# char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK];
# } heap_info;
#
# - heap_info.ar_ptr points to a 'struct malloc_state' (typedefed to mstate),
# which is a 2192-byte structure (on 64-bit systems):
#
# struct malloc_state
# {
# /* Serialize access. */
# __libc_lock_define (, mutex);
#
# /* Flags (formerly in max_fast). */
# int flags;
#
# /* Set if the fastbin chunks contain recently inserted free blocks. */
# /* Note this is a bool but not all targets support atomics on booleans. */
# int have_fastchunks;
#
# /* Fastbins */
# mfastbinptr fastbinsY[NFASTBINS];
#
# /* Base of the topmost chunk -- not otherwise kept in a bin */
# mchunkptr top;
#
# /* The remainder from the most recent split of a small request */
# mchunkptr last_remainder;
#
# /* Normal bins packed as described above */
# mchunkptr bins[NBINS * 2 - 2];
#
# /* Bitmap of bins */
# unsigned int binmap[BINMAPSIZE];
#
# /* Linked list */
# struct malloc_state *next;
#
# /* Linked list for free arenas. Access to this field is serialized
# by free_list_lock in arena.c. */
# struct malloc_state *next_free;
#
# /* Number of threads attached to this arena. 0 if the arena is on
# the free list. Access to this field is serialized by
# free_list_lock in arena.c. */
# INTERNAL_SIZE_T attached_threads;
#
# /* Memory allocated from the system in this arena. */
# INTERNAL_SIZE_T system_mem;
# INTERNAL_SIZE_T max_system_mem;
# };
#
# - A chunk is in the main arena if its 'A' bit is 0, in which case its arena
# is the static variable 'main_arena'.
#
# - A chunk is in a non-main arena if its 'A' bit is 1, in which case its
# heap_info can be found by clearing several low-order bits in the address,
# then the arena can be found by following ar_ptr from heap_info:
#
# #define heap_for_ptr(ptr) \
# ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1)))
# #define arena_for_chunk(ptr) \
# (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr)
#
# - The size of the block allocated for a given requested size
# (including the internal overhead) is calculated with request2size.
#
# #define request2size(req) \
# (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
# MINSIZE : \
# ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
#
# where SIZE_SZ is 8, MALLOC_ALIGN_MASK is 15, and MINSIZE is 32. In effect,
# this is MAX(32, ROUND_UP(req + 16, 16)).
#
# /* The smallest possible chunk */
# #define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
#
# /* The smallest size we can malloc is an aligned minimal chunk */
#
# #define MINSIZE \
# (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
# /* The corresponding word size. */
# #define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
#
# /* The corresponding bit mask value. */
# #define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
#
# struct malloc_chunk {
#
# INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */
# INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */
#
# struct malloc_chunk* fd; /* double links -- used only if free. */
# struct malloc_chunk* bk;
#
# /* Only used for large blocks: pointer to next larger size. */
# struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
# struct malloc_chunk* bk_nextsize;
# };
# - MALLOC_ALIGNMENT is 16.
open (OBJDUMP, "objdump -p '$core'|");
while (<OBJDUMP>) {
next unless /LOAD\s+off/;
my $hex = '(0x[0-9a-fA-F]+)';
my ($off, $vaddr) = /off\s+$hex\s+vaddr\s+$hex\s+/ or die;
$_ = <OBJDUMP>;
my ($filesz, $memsz, $flags) = /filesz\s+$hex\s+memsz\s+$hex\s+flags\s+(...)/ or die;
next if $flags ne 'rw-';
next if $filesz ne $memsz;
print "analyzing $filesz bytes starting at $off:\n";
system("dump-heap $core $off $filesz $vaddr");
system("dump-heap $core ". (hex($off) + 32) . " " . (hex($filesz) - 32) . " " . (hex($vaddr) + 32));
}
exit 0;
More information about the discuss
mailing list