core: rework crash handling

This introduces a new systemd.crash_reboot=1 kernel command line option
that triggers a reboot after crashing.

This also cleans up crash VT handling. Specifically, it cleans up the
configuration setting, to be between 1..63 or a boolean. This is to
replace the previous logic where "-1" meant disabled. We continue to
accept that setting, but only document the boolean syntax instead.

This also brings the documentation of the default settings in sync with
what actually happens.

The CrashChVT= configuration file setting is renamed to CrashChangeVT=,
following our usual logic of not abbreviating unnecessarily. The old
setting stays support for compat reasons.

Fixes #1300

man/kernel-command-line.xml

@@ -79,8 +79,9 @@
         <term><varname>systemd.unit=</varname></term>
         <term><varname>rd.systemd.unit=</varname></term>
         <term><varname>systemd.dump_core=</varname></term>
-        <term><varname>systemd.crash_shell=</varname></term>
         <term><varname>systemd.crash_chvt=</varname></term>
+        <term><varname>systemd.crash_shell=</varname></term>
+        <term><varname>systemd.crash_reboot=</varname></term>
         <term><varname>systemd.confirm_spawn=</varname></term>
         <term><varname>systemd.show_status=</varname></term>
         <term><varname>systemd.log_target=</varname></term>

man/systemd-system.conf.xml

@@ -90,9 +90,10 @@
         <term><varname>LogColor=</varname></term>
         <term><varname>LogLocation=</varname></term>
         <term><varname>DumpCore=yes</varname></term>
+        <term><varname>CrashChangeVT=no</varname></term>
         <term><varname>CrashShell=no</varname></term>
+        <term><varname>CrashReboot=no</varname></term>
         <term><varname>ShowStatus=yes</varname></term>
-        <term><varname>CrashChVT=1</varname></term>
         <term><varname>DefaultStandardOutput=journal</varname></term>
         <term><varname>DefaultStandardError=inherit</varname></term>
 

man/systemd.xml

@@ -1,4 +1,4 @@
-<?xml version='1.0'?> <!--*-nxml-*-->
+<?xml version='1.0'?> <!--*- Mode: nxml; nxml-child-indent: 2; indent-tabs-mode: nil -*-->
 <!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
   "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
 
@@ -131,17 +131,48 @@
       <varlistentry>
         <term><option>--dump-core</option></term>
 
-        <listitem><para>Dump core on crash. This switch has no effect
+        <listitem><para>Enable core dumping on crash. This switch has
-        when run as user instance.</para></listitem>
+        no effect when running as user instance. This setting may also
+        be enabled during boot on the kernel command line via the
+        <varname>systemd.dump_core=</varname> option, see
+        below.</para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><option>--crash-vt=</option><replaceable>VT</replaceable></term>
+
+        <listitem><para>Switch to a specific virtual console (VT) on
+        crash. Takes a positive integer in the range 1..63, or a
+        boolean argument. If an integer is passed, selects which VT to
+        switch to. If <constant>yes</constant>, the VT kernel messages
+        are written to is selected. If <constant>no</constant>, no VT
+        switch is attempted. This switch has no effect when running as
+        user instance. This setting may also be enabled during boot,
+        on the kernel command line via the
+        <varname>systemd.crash_vt=</varname> option, see
+        below.</para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>--crash-shell</option></term>
 
-        <listitem><para>Run shell on
+        <listitem><para>Run a shell on crash. This switch has no
-        crash. This switch has no effect when
+        effect when running as user instance. This setting may also be
-        run as user
+        enabled during boot, on the kernel command line via the
-        instance.</para></listitem>
+        <varname>systemd.crash_shell=</varname> option, see
+        below.</para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><option>--crash-reboot</option></term>
+
+        <listitem><para>Automatically reboot the system on crash. This
+        switch has no effect when running as user instance. This
+        setting may also be enabled during boot, on the kernel command
+        line via the <varname>systemd.crash_reboot=</varname> option,
+        see below.</para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>--confirm-spawn</option></term>
 
@@ -854,50 +885,67 @@
         <term><varname>systemd.dump_core=</varname></term>
 
         <listitem><para>Takes a boolean argument. If
-        <option>true</option>, systemd dumps core when it crashes.
+        <option>yes</option>, the systemd manager (PID 1) dumps core
-        Otherwise, no core dump is created. Defaults to
+        when it crashes.  Otherwise, no core dump is created. Defaults
-        <option>true</option>.</para></listitem>
+        to <option>yes</option>.</para></listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><varname>systemd.crash_chvt=</varname></term>
+
+        <listitem><para>Takes a positive integer, or a boolean
+        argument. If a positive integer (in the range 1..63) is
+        specified the system manager (PID 1) will activate the specified
+        virtual terminal (VT) when it crashes. Defaults to
+        <constant>no</constant>, meaning that no such switch is
+        attempted. If set to <constant>yes</constant> the VT the
+        kernel messages are written to is selected.</para></listitem>
       </varlistentry>
 
       <varlistentry>
         <term><varname>systemd.crash_shell=</varname></term>
 
         <listitem><para>Takes a boolean argument. If
-        <option>true</option>, systemd spawns a shell when it crashes.
+        <option>yes</option>, the system manager (PID 1) spawns a
-        Otherwise, no shell is spawned. Defaults to
+        shell when it crashes, after a 10s delay. Otherwise, no shell
-        <option>false</option>, for security reasons, as the shell is
+        is spawned. Defaults to <option>no</option>, for security
-        not protected by any password
+        reasons, as the shell is not protected by password
         authentication.</para></listitem>
       </varlistentry>
 
       <varlistentry>
-        <term><varname>systemd.crash_chvt=</varname></term>
+        <term><varname>systemd.crash_reboot=</varname></term>
 
-        <listitem><para>Takes an integer argument. If positive systemd
+        <listitem><para>Takes a boolean argument. If
-        activates the specified virtual terminal when it crashes.
+        <option>yes</option>, the system manager (PID 1) will reboot
-        Defaults to <constant>-1</constant>.</para></listitem>
+        the machine automatically when it crashes, after a 10s delay.
+        Otherwise, the system will hang indefinitely. Defaults to
+        <option>no</option>, in order to avoid a reboot loop. If
+        combined with <varname>systemd.crash_shell=</varname>, it is
+        first attempted to invoke a shell, and if this is not
+        successful the system is rebooted.</para></listitem>
       </varlistentry>
 
       <varlistentry>
         <term><varname>systemd.confirm_spawn=</varname></term>
 
         <listitem><para>Takes a boolean argument. If
-        <option>true</option>, asks for confirmation when spawning
+        <option>yes</option>, the system manager (PID 1) asks for
-        processes. Defaults to
+        confirmation when spawning processes. Defaults to
-        <option>false</option>.</para></listitem>
+        <option>no</option>.</para></listitem>
       </varlistentry>
 
       <varlistentry>
         <term><varname>systemd.show_status=</varname></term>
 
         <listitem><para>Takes a boolean argument or the constant
-        <constant>auto</constant>. If <option>true</option>, shows
+        <constant>auto</constant>. If <option>yes</option>, the
-        terse service status updates on the console during bootup.
+        systemd manager (PID 1) shows terse service status updates on
-        <constant>auto</constant> behaves like <option>false</option>
+        the console during bootup.  <constant>auto</constant> behaves
-        until a service fails or there is a significant delay in boot.
+        like <option>false</option> until a service fails or there is
-        Defaults to <option>true</option>, unless
+        a significant delay in boot.  Defaults to
-        <option>quiet</option> is passed as kernel command line option
+        <option>yes</option>, unless <option>quiet</option> is passed
-        in which case it defaults to
+        as kernel command line option in which case it defaults to
         <constant>auto</constant>.</para></listitem>
       </varlistentry>
 

src/basic/terminal-util.c

@@ -48,7 +48,7 @@ int chvt(int vt) {
         if (fd < 0)
                 return -errno;
 
-        if (vt < 0) {
+        if (vt <= 0) {
                 int tiocl[2] = {
                         TIOCL_GETKMSGREDIRECT,
                         0

src/core/main.c

@@ -27,6 +27,7 @@
 #include <string.h>
 #include <sys/mount.h>
 #include <sys/prctl.h>
+#include <sys/reboot.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #ifdef HAVE_SECCOMP
@@ -87,8 +88,9 @@ static enum {
 static char *arg_default_unit = NULL;
 static ManagerRunningAs arg_running_as = _MANAGER_RUNNING_AS_INVALID;
 static bool arg_dump_core = true;
-static bool arg_crash_shell = false;
 static int arg_crash_chvt = -1;
+static bool arg_crash_shell = false;
+static bool arg_crash_reboot = false;
 static bool arg_confirm_spawn = false;
 static ShowStatus arg_show_status = _SHOW_STATUS_UNSET;
 static bool arg_switched_root = false;
@@ -123,6 +125,21 @@ static void pager_open_if_enabled(void) {
         pager_open(false);
 }
 
+noreturn static void freeze_or_reboot(void) {
+
+        if (arg_crash_reboot) {
+                log_notice("Rebooting in 10s...");
+                (void) sleep(10);
+
+                log_notice("Rebooting now...");
+                (void) reboot(RB_AUTOBOOT);
+                log_emergency_errno(errno, "Failed to reboot: %m");
+        }
+
+        log_emergency("Freezing execution.");
+        freeze();
+}
+
 noreturn static void crash(int sig) {
 
         if (getpid() != 1)
@@ -188,7 +205,7 @@ noreturn static void crash(int sig) {
                 }
         }
 
-        if (arg_crash_chvt)
+        if (arg_crash_chvt >= 0)
                 (void) chvt(arg_crash_chvt);
 
         if (arg_crash_shell) {
@@ -198,7 +215,7 @@ noreturn static void crash(int sig) {
                 };
                 pid_t pid;
 
-                log_info("Executing crash shell in 10s...");
+                log_notice("Executing crash shell in 10s...");
                 (void) sleep(10);
 
                 /* Let the kernel reap children for us */
@@ -208,17 +225,20 @@ noreturn static void crash(int sig) {
                 if (pid < 0)
                         log_emergency_errno(errno, "Failed to fork off crash shell: %m");
                 else if (pid == 0) {
+                        (void) setsid();
                         (void) make_console_stdio();
                         (void) execle("/bin/sh", "/bin/sh", NULL, environ);
 
                         log_emergency_errno(errno, "execle() failed: %m");
+                        freeze_or_reboot();
                         _exit(EXIT_FAILURE);
-                } else
+                } else {
-                        log_info("Successfully spawned crash shell as PID "PID_FMT".", pid);
+                        log_info("Spawned crash shell as PID "PID_FMT".", pid);
+                        freeze();
+                }
         }
 
-        log_emergency("Freezing execution.");
+        freeze_or_reboot();
-        freeze();
 }
 
 static void install_crash_handler(void) {
@@ -252,6 +272,24 @@ static int console_setup(void) {
         return 0;
 }
 
+static int parse_crash_chvt(const char *value) {
+        int b;
+
+        if (safe_atoi(value, &arg_crash_chvt) >= 0)
+                return 0;
+
+        b = parse_boolean(value);
+        if (b < 0)
+                return b;
+
+        if (b > 0)
+                arg_crash_chvt = 0; /* switch to where kmsg goes */
+        else
+                arg_crash_chvt = -1; /* turn off switching */
+
+        return 0;
+}
+
 static int parse_proc_cmdline_item(const char *key, const char *value) {
 
         static const char * const rlmap[] = {
@@ -290,6 +328,11 @@ static int parse_proc_cmdline_item(const char *key, const char *value) {
                 else
                         arg_dump_core = r;
 
+        } else if (streq(key, "systemd.crash_chvt") && value) {
+
+                if (parse_crash_chvt(value) < 0)
+                        log_warning("Failed to parse crash chvt switch %s. Ignoring.", value);
+
         } else if (streq(key, "systemd.crash_shell") && value) {
 
                 r = parse_boolean(value);
@@ -298,12 +341,13 @@ static int parse_proc_cmdline_item(const char *key, const char *value) {
                 else
                         arg_crash_shell = r;
 
-        } else if (streq(key, "systemd.crash_chvt") && value) {
+        } else if (streq(key, "systemd.crash_reboot") && value) {
 
-                if (safe_atoi(value, &r) < 0)
+                r = parse_boolean(value);
-                        log_warning("Failed to parse crash chvt switch %s. Ignoring.", value);
+                if (r < 0)
+                        log_warning("Failed to parse crash reboot switch %s. Ignoring.", value);
                 else
-                        arg_crash_chvt = r;
+                        arg_crash_reboot = r;
 
         } else if (streq(key, "systemd.confirm_spawn") && value) {
 
@@ -461,6 +505,34 @@ static int config_parse_show_status(
         return 0;
 }
 
+static int config_parse_crash_chvt(
+                const char* unit,
+                const char *filename,
+                unsigned line,
+                const char *section,
+                unsigned section_line,
+                const char *lvalue,
+                int ltype,
+                const char *rvalue,
+                void *data,
+                void *userdata) {
+
+        int r;
+
+        assert(filename);
+        assert(lvalue);
+        assert(rvalue);
+        assert(data);
+
+        r = parse_crash_chvt(rvalue);
+        if (r < 0) {
+                log_syntax(unit, LOG_ERR, filename, line, r, "Failed to parse CrashChangeVT= setting, ignoring: %s", rvalue);
+                return 0;
+        }
+
+        return 0;
+}
+
 static int config_parse_join_controllers(const char *unit,
                                          const char *filename,
                                          unsigned line,
@@ -571,9 +643,11 @@ static int parse_config_file(void) {
                 { "Manager", "LogColor",                  config_parse_color,            0, NULL                                   },
                 { "Manager", "LogLocation",               config_parse_location,         0, NULL                                   },
                 { "Manager", "DumpCore",                  config_parse_bool,             0, &arg_dump_core                         },
+                { "Manager", "CrashChVT", /* legacy */    config_parse_crash_chvt,       0, NULL                                   },
+                { "Manager", "CrashChangeVT",             config_parse_crash_chvt,       0, NULL                                   },
                 { "Manager", "CrashShell",                config_parse_bool,             0, &arg_crash_shell                       },
+                { "Manager", "CrashReboot",               config_parse_bool,             0, &arg_crash_reboot                      },
                 { "Manager", "ShowStatus",                config_parse_show_status,      0, &arg_show_status                       },
-                { "Manager", "CrashChVT",                 config_parse_int,              0, &arg_crash_chvt                        },
                 { "Manager", "CPUAffinity",               config_parse_cpu_affinity2,    0, NULL                                   },
                 { "Manager", "JoinControllers",           config_parse_join_controllers, 0, &arg_join_controllers                  },
                 { "Manager", "RuntimeWatchdogSec",        config_parse_sec,              0, &arg_runtime_watchdog                  },
@@ -661,7 +735,9 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_VERSION,
                 ARG_DUMP_CONFIGURATION_ITEMS,
                 ARG_DUMP_CORE,
+                ARG_CRASH_CHVT,
                 ARG_CRASH_SHELL,
+                ARG_CRASH_REBOOT,
                 ARG_CONFIRM_SPAWN,
                 ARG_SHOW_STATUS,
                 ARG_DESERIALIZE,
@@ -684,7 +760,9 @@ static int parse_argv(int argc, char *argv[]) {
                 { "version",                  no_argument,       NULL, ARG_VERSION                  },
                 { "dump-configuration-items", no_argument,       NULL, ARG_DUMP_CONFIGURATION_ITEMS },
                 { "dump-core",                optional_argument, NULL, ARG_DUMP_CORE                },
+                { "crash-chvt",               required_argument, NULL, ARG_CRASH_CHVT               },
                 { "crash-shell",              optional_argument, NULL, ARG_CRASH_SHELL              },
+                { "crash-reboot",             optional_argument, NULL, ARG_CRASH_REBOOT             },
                 { "confirm-spawn",            optional_argument, NULL, ARG_CONFIRM_SPAWN            },
                 { "show-status",              optional_argument, NULL, ARG_SHOW_STATUS              },
                 { "deserialize",              required_argument, NULL, ARG_DESERIALIZE              },
@@ -802,21 +880,42 @@ static int parse_argv(int argc, char *argv[]) {
                         break;
 
                 case ARG_DUMP_CORE:
-                        r = optarg ? parse_boolean(optarg) : 1;
+                        if (!optarg)
-                        if (r < 0) {
+                                arg_dump_core = true;
-                                log_error("Failed to parse dump core boolean %s.", optarg);
+                        else {
-                                return r;
+                                r = parse_boolean(optarg);
-                        }
+                                if (r < 0)
+                                        return log_error_errno(r, "Failed to parse dump core boolean: %s", optarg);
                                 arg_dump_core = r;
+                        }
+                        break;
+
+                case ARG_CRASH_CHVT:
+                        r = parse_crash_chvt(optarg);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to parse crash virtual terminal index: %s", optarg);
                         break;
 
                 case ARG_CRASH_SHELL:
-                        r = optarg ? parse_boolean(optarg) : 1;
+                        if (!optarg)
-                        if (r < 0) {
+                                arg_crash_shell = true;
-                                log_error("Failed to parse crash shell boolean %s.", optarg);
+                        else {
-                                return r;
+                                r = parse_boolean(optarg);
-                        }
+                                if (r < 0)
+                                        return log_error_errno(r, "Failed to parse crash shell boolean: %s", optarg);
                                 arg_crash_shell = r;
+                        }
+                        break;
+
+                case ARG_CRASH_REBOOT:
+                        if (!optarg)
+                                arg_crash_reboot = true;
+                        else {
+                                r = parse_boolean(optarg);
+                                if (r < 0)
+                                        return log_error_errno(r, "Failed to parse crash shell boolean: %s", optarg);
+                                arg_crash_reboot = r;
+                        }
                         break;
 
                 case ARG_CONFIRM_SPAWN:
@@ -846,17 +945,16 @@ static int parse_argv(int argc, char *argv[]) {
                         r = safe_atoi(optarg, &fd);
                         if (r < 0 || fd < 0) {
                                 log_error("Failed to parse deserialize option %s.", optarg);
-                                return r < 0 ? r : -EINVAL;
+                                return -EINVAL;
                         }
 
-                        fd_cloexec(fd, true);
+                        (void) fd_cloexec(fd, true);
 
                         f = fdopen(fd, "r");
                         if (!f)
                                 return log_error_errno(errno, "Failed to open serialization fd: %m");
 
                         safe_fclose(arg_serialization);
-
                         arg_serialization = f;
 
                         break;
@@ -916,14 +1014,16 @@ static int help(void) {
                "     --unit=UNIT                 Set default unit\n"
                "     --system                    Run a system instance, even if PID != 1\n"
                "     --user                      Run a user instance\n"
-               "     --dump-core[=0|1]           Dump core on crash\n"
+               "     --dump-core[=BOOL]          Dump core on crash\n"
-               "     --crash-shell[=0|1]         Run shell on crash\n"
+               "     --crash-vt=NR               Change to specified VT on crash\n"
-               "     --confirm-spawn[=0|1]       Ask for confirmation when spawning processes\n"
+               "     --crash-reboot[=BOOL]       Reboot on crash\n"
-               "     --show-status[=0|1]         Show status updates on the console during bootup\n"
+               "     --crash-shell[=BOOL]        Run shell on crash\n"
+               "     --confirm-spawn[=BOOL]      Ask for confirmation when spawning processes\n"
+               "     --show-status[=BOOL]        Show status updates on the console during bootup\n"
                "     --log-target=TARGET         Set log target (console, journal, kmsg, journal-or-kmsg, null)\n"
                "     --log-level=LEVEL           Set log level (debug, info, notice, warning, err, crit, alert, emerg)\n"
-               "     --log-color[=0|1]           Highlight important log messages\n"
+               "     --log-color[=BOOL]          Highlight important log messages\n"
-               "     --log-location[=0|1]        Include code location in log messages\n"
+               "     --log-location[=BOOL]       Include code location in log messages\n"
                "     --default-standard-output=  Set default standard output for services\n"
                "     --default-standard-error=   Set default standard error output for services\n",
                program_invocation_short_name);
@@ -1032,7 +1132,7 @@ static void test_mtab(void) {
         log_error("/etc/mtab is not a symlink or not pointing to /proc/self/mounts. "
                   "This is not supported anymore. "
                   "Please make sure to replace this file by a symlink to avoid incorrect or misleading mount(8) output.");
-        freeze();
+        freeze_or_reboot();
 }
 
 static void test_usr(void) {
@@ -1964,7 +2064,7 @@ finish:
                         manager_status_printf(NULL, STATUS_TYPE_EMERGENCY,
                                               ANSI_HIGHLIGHT_RED "!!!!!!" ANSI_NORMAL,
                                               "%s, freezing.", error_message);
-                freeze();
+                freeze_or_reboot();
         }
 
         return retval;

src/core/shutdown.c

@@ -430,6 +430,5 @@ int main(int argc, char *argv[]) {
 
   error:
         log_emergency_errno(r, "Critical error while doing system shutdown: %m");
-
         freeze();
 }

src/core/system.conf

@@ -17,9 +17,10 @@
 #LogColor=yes
 #LogLocation=no
 #DumpCore=yes
-#CrashShell=no
 #ShowStatus=yes
-#CrashChVT=1
+#CrashChangeVT=no
+#CrashShell=no
+#CrashReboot=no
 #CPUAffinity=1 2
 #JoinControllers=cpu,cpuacct net_cls,net_prio
 #RuntimeWatchdogSec=0