[ovs-dev] [PATCH 8/9] daemon: Add support for process monitoring and restart.
Justin Pettit
jpettit at nicira.com
Fri Jan 15 22:43:22 UTC 2010
Looks good.
--Justin
On Jan 15, 2010, at 1:09 PM, Ben Pfaff wrote:
> ---
> lib/daemon.c | 90 ++++++++++++++++++++++++++++
> lib/daemon.h | 11 +++-
> lib/daemon.man | 11 ++++
> tests/automake.mk | 1 +
> tests/daemon.at | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> tests/testsuite.at | 1 +
> 6 files changed, 277 insertions(+), 2 deletions(-)
> create mode 100644 tests/daemon.at
>
> diff --git a/lib/daemon.c b/lib/daemon.c
> index 17ad26e..3e2e4e1 100644
> --- a/lib/daemon.c
> +++ b/lib/daemon.c
> @@ -25,6 +25,7 @@
> #include "fatal-signal.h"
> #include "dirs.h"
> #include "lockfile.h"
> +#include "process.h"
> #include "socket-util.h"
> #include "timeval.h"
> #include "util.h"
> @@ -47,6 +48,10 @@ static bool chdir_ = true;
> /* File descriptor used by daemonize_start() and daemonize_complete(). */
> static int daemonize_fd = -1;
>
> +/* --monitor: Should a supervisory process monitor the daemon and restart it if
> + * it dies due to an error signal? */
> +static bool monitor;
> +
> /* Returns the file name that would be used for a pidfile if 'name' were
> * provided to set_pidfile(). The caller must free the returned string. */
> char *
> @@ -117,6 +122,14 @@ get_detach(void)
> return detach;
> }
>
> +/* Sets up a following call to daemonize() to fork a supervisory process to
> + * monitor the daemon and restart it if it dies due to an error signal. */
> +void
> +daemon_set_monitor(void)
> +{
> + monitor = true;
> +}
> +
> /* If a pidfile has been configured and that pidfile already exists and is
> * locked by a running process, returns the pid of the running process.
> * Otherwise, returns 0. */
> @@ -286,6 +299,69 @@ fork_notify_startup(int fd)
> }
> }
>
> +static bool
> +should_restart(int status)
> +{
> + if (WIFSIGNALED(status)) {
> + static const int error_signals[] = {
> + SIGABRT, SIGALRM, SIGBUS, SIGFPE, SIGILL, SIGPIPE, SIGSEGV,
> + SIGXCPU, SIGXFSZ
> + };
> +
> + size_t i;
> +
> + for (i = 0; i < ARRAY_SIZE(error_signals); i++) {
> + if (error_signals[i] == WTERMSIG(status)) {
> + return true;
> + }
> + }
> + }
> + return false;
> +}
> +
> +static void
> +monitor_daemon(pid_t daemon_pid)
> +{
> + /* XXX Should limit the rate at which we restart the daemon. */
> + /* XXX Should log daemon's stderr output at startup time. */
> + const char *saved_program_name;
> +
> + saved_program_name = program_name;
> + program_name = xasprintf("monitor(%s)", program_name);
> + for (;;) {
> + int retval;
> + int status;
> +
> + do {
> + retval = waitpid(daemon_pid, &status, 0);
> + } while (retval == -1 && errno == EINTR);
> +
> + if (retval == -1) {
> + ovs_fatal(errno, "waitpid failed");
> + } else if (retval == daemon_pid) {
> + char *status_msg = process_status_msg(status);
> + if (should_restart(status)) {
> + VLOG_ERR("%s daemon died unexpectedly (%s), restarting",
> + saved_program_name, status_msg);
> + free(status_msg);
> +
> + daemon_pid = fork_and_wait_for_startup(&daemonize_fd);
> + if (!daemon_pid) {
> + break;
> + }
> + } else {
> + VLOG_INFO("%s daemon exited normally (%s), exiting",
> + saved_program_name, status_msg);
> + exit(0);
> + }
> + }
> + }
> +
> + /* Running in new daemon process. */
> + free((char *) program_name);
> + program_name = saved_program_name;
> +}
> +
> /* Close stdin, stdout, stderr. If we're started from e.g. an SSH session,
> * then this keeps up from holding that session open artificially. */
> static void
> @@ -314,6 +390,20 @@ daemonize_start(void)
> /* Running in parent process. */
> exit(0);
> }
> + /* Running in daemon or monitor process. */
> + }
> +
> + if (monitor) {
> + int saved_daemonize_fd = daemonize_fd;
> + pid_t daemon_pid;
> +
> + daemon_pid = fork_and_wait_for_startup(&daemonize_fd);
> + if (daemon_pid > 0) {
> + /* Running in monitor process. */
> + fork_notify_startup(saved_daemonize_fd);
> + close_standard_fds();
> + monitor_daemon(daemon_pid);
> + }
> /* Running in daemon process. */
> }
>
> diff --git a/lib/daemon.h b/lib/daemon.h
> index 8ded63f..1d63076 100644
> --- a/lib/daemon.h
> +++ b/lib/daemon.h
> @@ -1,5 +1,5 @@
> /*
> - * Copyright (c) 2008, 2009 Nicira Networks.
> + * Copyright (c) 2008, 2009, 2010 Nicira Networks.
> *
> * Licensed under the Apache License, Version 2.0 (the "License");
> * you may not use this file except in compliance with the License.
> @@ -26,13 +26,15 @@ enum {
> OPT_NO_CHDIR,
> OPT_OVERWRITE_PIDFILE,
> OPT_PIDFILE,
> + OPT_MONITOR
> };
>
> #define DAEMON_LONG_OPTIONS \
> {"detach", no_argument, 0, OPT_DETACH}, \
> {"no-chdir", no_argument, 0, OPT_NO_CHDIR}, \
> {"pidfile", optional_argument, 0, OPT_PIDFILE}, \
> - {"overwrite-pidfile", no_argument, 0, OPT_OVERWRITE_PIDFILE}
> + {"overwrite-pidfile", no_argument, 0, OPT_OVERWRITE_PIDFILE},\
> + {"monitor", no_argument, 0, OPT_MONITOR}
>
> #define DAEMON_OPTION_HANDLERS \
> case OPT_DETACH: \
> @@ -49,6 +51,10 @@ enum {
> \
> case OPT_OVERWRITE_PIDFILE: \
> ignore_existing_pidfile(); \
> + break; \
> + \
> + case OPT_MONITOR: \
> + daemon_set_monitor(); \
> break;
>
> char *make_pidfile_name(const char *name);
> @@ -58,6 +64,7 @@ void set_no_chdir(void);
> bool is_chdir_enabled(void);
> void set_detach(void);
> bool get_detach(void);
> +void daemon_set_monitor(void);
> void daemonize(void);
> void daemonize_start(void);
> void daemonize_complete(void);
> diff --git a/lib/daemon.man b/lib/daemon.man
> index ea0561d..c239377 100644
> --- a/lib/daemon.man
> +++ b/lib/daemon.man
> @@ -20,6 +20,17 @@ Causes \fB\*(PN\fR to detach itself from the foreground session and
> run as a background process.
>
> .TP
> +\fB--monitor\fR
> +Creates an additional process to monitor the \fB\*(PN\fR daemon. If
> +the daemon dies due to a signal that indicates a programming error
> +(e.g. \fBSIGSEGV\fR, \fBSIGABRT\fR), then the monitor process starts a
> +new copy of it. If the daemon die or exits for another reason, the
> +monitor process exits.
> +.IP
> +This option is normally used with \fB--detach\fR, but it also
> +functions without it.
> +
> +.TP
> \fB--no-chdir\fR
> By default, when \fB--detach\fR is specified, \fB\*(PN\fR
> changes its current working directory to the root directory after it
> diff --git a/tests/automake.mk b/tests/automake.mk
> index 7a73a84..dc677eb 100644
> --- a/tests/automake.mk
> +++ b/tests/automake.mk
> @@ -9,6 +9,7 @@ TESTSUITE_AT = \
> tests/ovsdb-macros.at \
> tests/lcov-pre.at \
> tests/library.at \
> + tests/daemon.at \
> tests/vconn.at \
> tests/dir_name.at \
> tests/aes128.at \
> diff --git a/tests/daemon.at b/tests/daemon.at
> new file mode 100644
> index 0000000..1062229
> --- /dev/null
> +++ b/tests/daemon.at
> @@ -0,0 +1,165 @@
> +AT_BANNER([daemon unit tests])
> +
> +AT_SETUP([daemon])
> +OVSDB_INIT([db])
> +AT_CAPTURE_FILE([pid])
> +AT_CAPTURE_FILE([expected])
> +# Start the daemon and wait for the pidfile to get created
> +# and that its contents are the correct pid.
> +AT_CHECK([ovsdb-server --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db& echo $! > expected], [0], [ignore], [ignore])
> +OVS_WAIT_UNTIL([test -s pid], [kill `cat expected`])
> +AT_CHECK(
> + [pid=`cat pid` && expected=`cat expected` && test "$pid" = "$expected"],
> + [0], [], [], [kill `cat expected`])
> +AT_CHECK([kill -0 `cat pid`], [0], [], [], [kill `cat expected`])
> +# Kill the daemon and make sure that the pidfile gets deleted.
> +kill `cat expected`
> +OVS_WAIT_WHILE([kill -0 `cat expected`])
> +AT_CHECK([test ! -e pid])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --monitor])
> +OVSDB_INIT([db])
> +AT_CAPTURE_FILE([pid])
> +AT_CAPTURE_FILE([parent])
> +AT_CAPTURE_FILE([parentpid])
> +AT_CAPTURE_FILE([newpid])
> +# Start the daemon and wait for the pidfile to get created.
> +AT_CHECK([ovsdb-server --monitor --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db& echo $! > parent], [0], [ignore], [ignore])
> +OVS_WAIT_UNTIL([test -s pid], [kill `cat parent`])
> +# Check that the pidfile names a running process,
> +# and that the parent process of that process is our child process.
> +AT_CHECK([kill -0 `cat pid`], [0], [], [], [kill `cat parent`])
> +AT_CHECK([ps -o ppid= -p `cat pid` > parentpid],
> + [0], [], [], [kill `cat parent`])
> +AT_CHECK(
> + [parentpid=`cat parentpid` &&
> + parent=`cat parent` &&
> + test $parentpid = $parent],
> + [0], [], [], [kill `cat parent`])
> +# Kill the daemon process, making it look like a segfault,
> +# and wait for a new child process to get spawned.
> +AT_CHECK([cp pid oldpid], [0], [], [], [kill `cat parent`])
> +AT_CHECK([kill -SEGV `cat pid`], [0], [], [ignore], [kill `cat parent`])
> +OVS_WAIT_WHILE([kill -0 `cat oldpid`], [kill `cat parent`])
> +OVS_WAIT_UNTIL([test -s pid && test `cat pid` != `cat oldpid`],
> + [kill `cat parent`])
> +AT_CHECK([cp pid newpid], [0], [], [], [kill `cat parent`])
> +# Check that the pidfile names a running process,
> +# and that the parent process of that process is our child process.
> +AT_CHECK([ps -o ppid= -p `cat pid` > parentpid],
> + [0], [], [], [kill `cat parent`])
> +AT_CHECK(
> + [parentpid=`cat parentpid` &&
> + parent=`cat parent` &&
> + test $parentpid = $parent],
> + [0], [], [], [kill `cat parent`])
> +# Kill the daemon process with SIGTERM, and wait for the daemon
> +# and the monitor processes to go away and the pidfile to get deleted.
> +AT_CHECK([kill `cat pid`], [0], [], [ignore], [kill `cat parent`])
> +OVS_WAIT_WHILE([kill -0 `cat parent` || kill -0 `cat newpid` || test -e pid],
> + [kill `cat parent`])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --detach])
> +AT_CAPTURE_FILE([pid])
> +OVSDB_INIT([db])
> +# Start the daemon and make sure that the pidfile exists immediately.
> +# We don't wait for the pidfile to get created because the daemon is
> +# supposed to do so before the parent exits.
> +AT_CHECK([ovsdb-server --detach --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db], [0], [ignore], [ignore])
> +AT_CHECK([test -s pid])
> +AT_CHECK([kill -0 `cat pid`])
> +# Kill the daemon and make sure that the pidfile gets deleted.
> +cp pid saved-pid
> +kill `cat pid`
> +OVS_WAIT_WHILE([kill -0 `cat saved-pid`])
> +AT_CHECK([test ! -e pid])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --detach --monitor])
> +m4_define([CHECK],
> + [AT_CHECK([$1], [$2], [$3], [$4], [kill `cat daemon monitor`])])
> +OVSDB_INIT([db])
> +AT_CAPTURE_FILE([daemon])
> +AT_CAPTURE_FILE([olddaemon])
> +AT_CAPTURE_FILE([newdaemon])
> +AT_CAPTURE_FILE([monitor])
> +AT_CAPTURE_FILE([newmonitor])
> +AT_CAPTURE_FILE([init])
> +# Start the daemon and make sure that the pidfile exists immediately.
> +# We don't wait for the pidfile to get created because the daemon is
> +# supposed to do so before the parent exits.
> +AT_CHECK([ovsdb-server --detach --pidfile=$PWD/daemon --monitor --remote=punix:socket --unixctl=$PWD/unixctl db], [0], [ignore], [ignore])
> +AT_CHECK([test -s daemon])
> +# Check that the pidfile names a running process,
> +# and that the parent process of that process is a running process,
> +# and that the parent process of that process is init.
> +CHECK([kill -0 `cat daemon`])
> +CHECK([ps -o ppid= -p `cat daemon` > monitor])
> +CHECK([kill -0 `cat monitor`])
> +CHECK([ps -o ppid= -p `cat monitor` > init])
> +CHECK([test `cat init` = 1])
> +# Kill the daemon process, making it look like a segfault,
> +# and wait for a new daemon process to get spawned.
> +CHECK([cp daemon olddaemon])
> +CHECK([kill -SEGV `cat daemon`], [0], [ignore], [ignore])
> +OVS_WAIT_WHILE([kill -0 `cat olddaemon`], [kill `cat olddaemon daemon`])
> +OVS_WAIT_UNTIL([test -s daemon && test `cat daemon` != `cat olddaemon`],
> + [kill `cat olddaemon daemon`])
> +CHECK([cp daemon newdaemon])
> +# Check that the pidfile names a running process,
> +# and that the parent process of that process is our child process.
> +CHECK([kill -0 `cat daemon`])
> +CHECK([diff olddaemon newdaemon], [1], [ignore])
> +CHECK([ps -o ppid= -p `cat daemon` > newmonitor])
> +CHECK([diff monitor newmonitor])
> +CHECK([kill -0 `cat newmonitor`])
> +CHECK([ps -o ppid= -p `cat newmonitor` > init])
> +CHECK([test `cat init` = 1])
> +# Kill the daemon process with SIGTERM, and wait for the daemon
> +# and the monitor processes to go away and the pidfile to get deleted.
> +CHECK([kill `cat daemon`], [0], [], [ignore])
> +OVS_WAIT_WHILE(
> + [kill -0 `cat monitor` || kill -0 `cat newdaemon` || test -e daemon],
> + [kill `cat monitor newdaemon`])
> +m4_undefine([CHECK])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --detach startup errors])
> +AT_CAPTURE_FILE([pid])
> +OVSDB_INIT([db])
> +OVS_CHECK_LCOV([ovsdb-server --detach --pidfile=$PWD/pid --unixctl=$PWD/nonexistent/unixctl db], [1], [], [stderr])
> +AT_CHECK([grep 'ovsdb-server: could not initialize control socket' stderr],
> + [0], [ignore], [])
> +AT_CHECK([test ! -s pid])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --detach --monitor startup errors])
> +AT_CAPTURE_FILE([pid])
> +OVSDB_INIT([db])
> +OVS_CHECK_LCOV([ovsdb-server --detach --pidfile=$PWD/pid --monitor --unixctl=$PWD/nonexistent/unixctl db], [1], [], [stderr])
> +AT_CHECK([grep 'ovsdb-server: could not initialize control socket' stderr],
> + [0], [ignore], [])
> +AT_CHECK([test ! -s pid])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --detach closes standard fds])
> +AT_CAPTURE_FILE([pid])
> +OVSDB_INIT([db])
> +AT_CHECK([(yes; echo $? > status) | ovsdb-server --detach --pidfile=$PWD/pid --unixctl=$PWD/unixctl db], [0], [], [stderr])
> +AT_CHECK([kill `cat pid`])
> +AT_CHECK([test -s status])
> +AT_CHECK([kill -l `cat status`], [0], [PIPE
> +])
> +AT_CLEANUP
> +
> +AT_SETUP([daemon --detach --monitor closes standard fds])
> +AT_CAPTURE_FILE([pid])
> +OVSDB_INIT([db])
> +AT_CHECK([(yes; echo $? > status) | ovsdb-server --detach --monitor --pidfile=$PWD/pid --unixctl=$PWD/unixctl db], [0], [], [stderr])
> +AT_CHECK([kill `cat pid`])
> +AT_CHECK([test -s status])
> +AT_CHECK([kill -l `cat status`], [0], [PIPE
> +])
> +AT_CLEANUP
> diff --git a/tests/testsuite.at b/tests/testsuite.at
> index 6ec1a80..93d7e6e 100644
> --- a/tests/testsuite.at
> +++ b/tests/testsuite.at
> @@ -39,6 +39,7 @@ m4_include([tests/ovsdb-macros.at])
> m4_include([tests/lcov-pre.at])
>
> m4_include([tests/library.at])
> +m4_include([tests/daemon.at])
> m4_include([tests/vconn.at])
> m4_include([tests/dir_name.at])
> m4_include([tests/aes128.at])
> --
> 1.6.3.3
>
>
> _______________________________________________
> dev mailing list
> dev at openvswitch.org
> http://openvswitch.org/mailman/listinfo/dev_openvswitch.org
More information about the dev
mailing list