watchdog: add setting to configure pretimeout governor

man/kernel-command-line.xml

@@ -440,6 +440,16 @@
         </listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><varname>systemd.watchdog_pretimeout_governor=</varname></term>
+
+        <listitem>
+          <para>Overrides the watchdog pre-timeout settings otherwise configured with
+          <varname>RuntimeWatchdogPreGovernor=</varname>. Takes a string value. For details, see
+          <citerefentry><refentrytitle>systemd-system.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
+        </listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><varname>systemd.cpu_affinity=</varname></term>
 

man/org.freedesktop.systemd1.xml

@@ -405,6 +405,9 @@ node /org/freedesktop/systemd1 {
       readwrite t RuntimeWatchdogPreUSec = ...;
       @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
       @org.freedesktop.systemd1.Privileged("true")
+      readwrite s RuntimeWatchdogPreGovernor = '...';
+      @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
+      @org.freedesktop.systemd1.Privileged("true")
       readwrite t RebootWatchdogUSec = ...;
       @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
       @org.freedesktop.systemd1.Privileged("true")
@@ -655,6 +658,8 @@ node /org/freedesktop/systemd1 {
 
     <!--property RuntimeWatchdogPreUSec is not documented!-->
 
+    <!--property RuntimeWatchdogPreGovernor is not documented!-->
+
     <!--property RebootWatchdogUSec is not documented!-->
 
     <!--property KExecWatchdogUSec is not documented!-->
@@ -1059,6 +1064,8 @@ node /org/freedesktop/systemd1 {
 
     <variablelist class="dbus-property" generated="True" extra-ref="RuntimeWatchdogPreUSec"/>
 
+    <variablelist class="dbus-property" generated="True" extra-ref="RuntimeWatchdogPreGovernor"/>
+
     <variablelist class="dbus-property" generated="True" extra-ref="RebootWatchdogUSec"/>
 
     <variablelist class="dbus-property" generated="True" extra-ref="KExecWatchdogUSec"/>

man/systemd-system.conf.xml

@@ -186,20 +186,13 @@
         notification generated by the watchdog before the watchdog reset might
         occur in the event the watchdog has not been serviced. This notification
         is handled by the kernel and can be configured to take an action (i.e.
-        generate a kernel panic) using the
+        generate a kernel panic) using <varname>RuntimeWatchdogPreGovernor=</varname>.
-        <filename>/sys/class/watchdog/watchdog0/pretimeout_governor</filename>
+        Not all watchdog hardware or drivers support generating a pre-timeout and
-        sysfs file for your watchdog device. The available actions (or
+        depending on the state of the system, the kernel may be unable to take the
-        governors) are listed in the
+        configured action before the watchdog reboot. The watchdog will be configured
-        <filename>/sys/class/watchdog/watchdog0/pretimeout_available_governors</filename>
+        to generate the pre-timeout event at the amount of time specified by
-        sysfs file for your watchdog device. The default action for the
+        <varname>RuntimeWatchdogPreSec=</varname> before the runtime watchdog timeout
-        pre-timeout event is to log a kernel message but that can be changed in
+        (set by <varname>RuntimeWatchdogSec=</varname>). For example, if the we have
-        the kernel's configuration. Not all watchdog hardware or drivers support
-        generating a pre-timeout and depending on the state of the system, the
-        kernel may be unable to take the configured action before the watchdog
-        reboot. The watchdog will be configured to generate the pre-timeout event
-        at the amount of time specified by <varname>RuntimeWatchdogPreSec=</varname>
-        before the runtime watchdog timeout (set by
-        <varname>RuntimeWatchdogSec=</varname>). For example, if the we have
         <varname>RuntimeWatchdogSec=30</varname> and
         <varname>RuntimeWatchdogPreSec=10</varname>, then the pre-timeout event
         will occur if the watchdog has not pinged for 20s (10s before the
@@ -211,6 +204,23 @@
         kernel if the setting is greater than the actual watchdog timeout.</para></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><varname>RuntimeWatchdogPreGovernor=</varname></term>
+
+        <listitem><para>Configure the action taken by the hardware watchdog device
+        when the pre-timeout expires. The default action for the pre-timeout event
+        depends on the kernel configuration, but it is usually to log a kernel
+        message. For a list of valid actions available for a given watchdog device,
+        check the content of the
+        <filename>/sys/class/watchdog/watchdog<replaceable>X</replaceable>/pretimeout_available_governors</filename>
+        file. Typically, available governor types are <varname>noop</varname> and <varname>panic</varname>.
+        Availability, names and functionality might vary depending on the specific device driver
+        in use. If the <filename>pretimeout_available_governors</filename> sysfs file is empty,
+        the governor might be built as a kernel module and might need to be manually loaded
+        (e.g. <varname>pretimeout_noop.ko</varname>), or the watchdog device might not support
+        pre-timeouts.</para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><varname>WatchdogDevice=</varname></term>
 

src/core/dbus-manager.c

@@ -283,6 +283,24 @@ static int property_get_pretimeout_watchdog(
         return sd_bus_message_append(reply, "t", manager_get_watchdog(m, WATCHDOG_PRETIMEOUT));
 }
 
+static int property_get_pretimeout_watchdog_governor(
+                sd_bus *bus,
+                const char *path,
+                const char *interface,
+                const char *property,
+                sd_bus_message *reply,
+                void *userdata,
+                sd_bus_error *error) {
+
+        Manager *m = userdata;
+
+        assert(m);
+        assert(bus);
+        assert(reply);
+
+        return sd_bus_message_append(reply, "s", m->watchdog_pretimeout_governor);
+}
+
 static int property_get_reboot_watchdog(
                 sd_bus *bus,
                 const char *path,
@@ -360,6 +378,30 @@ static int property_set_pretimeout_watchdog(
         return property_set_watchdog(userdata, WATCHDOG_PRETIMEOUT, value);
 }
 
+static int property_set_pretimeout_watchdog_governor(
+                sd_bus *bus,
+                const char *path,
+                const char *interface,
+                const char *property,
+                sd_bus_message *value,
+                void *userdata,
+                sd_bus_error *error) {
+
+        Manager *m = userdata;
+        char *governor;
+        int r;
+
+        assert(m);
+
+        r = sd_bus_message_read(value, "s", &governor);
+        if (r < 0)
+                return r;
+        if (!string_is_safe(governor))
+                return -EINVAL;
+
+        return manager_override_watchdog_pretimeout_governor(m, governor);
+}
+
 static int property_set_reboot_watchdog(
                 sd_bus *bus,
                 const char *path,
@@ -2727,6 +2769,7 @@ const sd_bus_vtable bus_manager_vtable[] = {
         SD_BUS_PROPERTY("DefaultStandardError", "s", bus_property_get_exec_output, offsetof(Manager, default_std_error), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_WRITABLE_PROPERTY("RuntimeWatchdogUSec", "t", property_get_runtime_watchdog, property_set_runtime_watchdog, 0, 0),
         SD_BUS_WRITABLE_PROPERTY("RuntimeWatchdogPreUSec", "t", property_get_pretimeout_watchdog, property_set_pretimeout_watchdog, 0, 0),
+        SD_BUS_WRITABLE_PROPERTY("RuntimeWatchdogPreGovernor", "s", property_get_pretimeout_watchdog_governor, property_set_pretimeout_watchdog_governor, 0, 0),
         SD_BUS_WRITABLE_PROPERTY("RebootWatchdogUSec", "t", property_get_reboot_watchdog, property_set_reboot_watchdog, 0, 0),
         /* The following item is an obsolete alias */
         SD_BUS_WRITABLE_PROPERTY("ShutdownWatchdogUSec", "t", property_get_reboot_watchdog, property_set_reboot_watchdog, 0, SD_BUS_VTABLE_HIDDEN),

src/core/main.c

@@ -140,6 +140,7 @@ static usec_t arg_reboot_watchdog;
 static usec_t arg_kexec_watchdog;
 static usec_t arg_pretimeout_watchdog;
 static char *arg_early_core_pattern;
+static char *arg_watchdog_pretimeout_governor;
 static char *arg_watchdog_device;
 static char **arg_default_environment;
 static char **arg_manager_environment;
@@ -575,6 +576,20 @@ static int parse_proc_cmdline_item(const char *key, const char *value, void *dat
                         }
                 }
 
+        } else if (proc_cmdline_key_streq(key, "systemd.watchdog_pretimeout_governor")) {
+
+                if (proc_cmdline_value_missing(key, value) || isempty(value)) {
+                        arg_watchdog_pretimeout_governor = mfree(arg_watchdog_pretimeout_governor);
+                        return 0;
+                }
+
+                if (!string_is_safe(value)) {
+                        log_warning("Watchdog pretimeout governor '%s' is not valid, ignoring.", value);
+                        return 0;
+                }
+
+                return free_and_strdup_warn(&arg_watchdog_pretimeout_governor, value);
+
         } else if (proc_cmdline_key_streq(key, "systemd.clock_usec")) {
 
                 if (proc_cmdline_value_missing(key, value))
@@ -732,6 +747,7 @@ static int parse_config_file(void) {
                 { "Manager", "ShutdownWatchdogSec",          config_parse_watchdog_sec,          0, &arg_reboot_watchdog                   }, /* obsolete alias */
                 { "Manager", "KExecWatchdogSec",             config_parse_watchdog_sec,          0, &arg_kexec_watchdog                    },
                 { "Manager", "WatchdogDevice",               config_parse_path,                  0, &arg_watchdog_device                   },
+                { "Manager", "RuntimeWatchdogPreGovernor",   config_parse_safe_string,           0, &arg_watchdog_pretimeout_governor      },
                 { "Manager", "CapabilityBoundingSet",        config_parse_capability_set,        0, &arg_capability_bounding_set           },
                 { "Manager", "NoNewPrivileges",              config_parse_bool,                  0, &arg_no_new_privs                      },
 #if HAVE_SECCOMP
@@ -856,6 +872,7 @@ static void set_manager_defaults(Manager *m) {
 }
 
 static void set_manager_settings(Manager *m) {
+        int r;
 
         assert(m);
 
@@ -871,6 +888,9 @@ static void set_manager_settings(Manager *m) {
         manager_set_watchdog(m, WATCHDOG_REBOOT, arg_reboot_watchdog);
         manager_set_watchdog(m, WATCHDOG_KEXEC, arg_kexec_watchdog);
         manager_set_watchdog(m, WATCHDOG_PRETIMEOUT, arg_pretimeout_watchdog);
+        r = manager_set_watchdog_pretimeout_governor(m, arg_watchdog_pretimeout_governor);
+        if (r < 0)
+                log_warning_errno(r, "Failed to set watchdog pretimeout governor to '%s', ignoring: %m", arg_watchdog_pretimeout_governor);
 
         manager_set_show_status(m, arg_show_status, "commandline");
         m->status_unit_format = arg_status_unit_format;
@@ -1618,6 +1638,7 @@ static int become_shutdown(
          * shutdown binary to repeatedly ping it.
          * Disable the pretimeout watchdog, as we do not support it from the shutdown binary. */
         (void) watchdog_setup_pretimeout(0);
+        (void) watchdog_setup_pretimeout_governor(NULL);
         r = watchdog_setup(watchdog_timer);
         watchdog_close(r < 0);
 
@@ -2473,6 +2494,7 @@ static void reset_arguments(void) {
         arg_pretimeout_watchdog = 0;
         arg_early_core_pattern = NULL;
         arg_watchdog_device = NULL;
+        arg_watchdog_pretimeout_governor = mfree(arg_watchdog_pretimeout_governor);
 
         arg_default_environment = strv_free(arg_default_environment);
         arg_manager_environment = strv_free(arg_manager_environment);

src/core/manager-serialize.c

@@ -119,6 +119,7 @@ int manager_serialize(
         (void) serialize_usec(f, "reboot-watchdog-overridden", m->watchdog_overridden[WATCHDOG_REBOOT]);
         (void) serialize_usec(f, "kexec-watchdog-overridden", m->watchdog_overridden[WATCHDOG_KEXEC]);
         (void) serialize_usec(f, "pretimeout-watchdog-overridden", m->watchdog_overridden[WATCHDOG_PRETIMEOUT]);
+        (void) serialize_item(f, "pretimeout-watchdog-governor-overridden", m->watchdog_pretimeout_governor_overridden);
 
         for (ManagerTimestamp q = 0; q < _MANAGER_TIMESTAMP_MAX; q++) {
                 _cleanup_free_ char *joined = NULL;
@@ -464,6 +465,11 @@ int manager_deserialize(Manager *m, FILE *f, FDSet *fds) {
                         else
                                 manager_override_watchdog(m, WATCHDOG_PRETIMEOUT, t);
 
+                } else if ((val = startswith(l, "pretimeout-watchdog-governor-overridden="))) {
+                        r = free_and_strdup(&m->watchdog_pretimeout_governor_overridden, val);
+                        if (r < 0)
+                                return r;
+
                 } else if (startswith(l, "env=")) {
                         r = deserialize_environment(l + 4, &m->client_environment);
                         if (r < 0)

src/core/manager.c

@@ -1542,6 +1542,9 @@ Manager* manager_free(Manager *m) {
                 m->prefix[dt] = mfree(m->prefix[dt]);
         free(m->received_credentials);
 
+        free(m->watchdog_pretimeout_governor);
+        free(m->watchdog_pretimeout_governor_overridden);
+
 #if BPF_FRAMEWORK
         lsm_bpf_destroy(m->restrict_fs);
 #endif
@@ -3263,6 +3266,52 @@ void manager_override_watchdog(Manager *m, WatchdogType t, usec_t timeout) {
         m->watchdog_overridden[t] = timeout;
 }
 
+int manager_set_watchdog_pretimeout_governor(Manager *m, const char *governor) {
+        _cleanup_free_ char *p = NULL;
+        int r;
+
+        assert(m);
+
+        if (MANAGER_IS_USER(m))
+                return 0;
+
+        if (streq_ptr(m->watchdog_pretimeout_governor, governor))
+                return 0;
+
+        p = strdup(governor);
+        if (!p)
+                return -ENOMEM;
+
+        r = watchdog_setup_pretimeout_governor(governor);
+        if (r < 0)
+                return r;
+
+        return free_and_replace(m->watchdog_pretimeout_governor, p);
+}
+
+int manager_override_watchdog_pretimeout_governor(Manager *m, const char *governor) {
+        _cleanup_free_ char *p = NULL;
+        int r;
+
+        assert(m);
+
+        if (MANAGER_IS_USER(m))
+                return 0;
+
+        if (streq_ptr(m->watchdog_pretimeout_governor_overridden, governor))
+                return 0;
+
+        p = strdup(governor);
+        if (!p)
+                return -ENOMEM;
+
+        r = watchdog_setup_pretimeout_governor(governor);
+        if (r < 0)
+                return r;
+
+        return free_and_replace(m->watchdog_pretimeout_governor_overridden, p);
+}
+
 int manager_reload(Manager *m) {
         _unused_ _cleanup_(manager_reloading_stopp) Manager *reloading = NULL;
         _cleanup_fdset_free_ FDSet *fds = NULL;

src/core/manager.h

@@ -248,6 +248,8 @@ struct Manager {
 
         usec_t watchdog[_WATCHDOG_TYPE_MAX];
         usec_t watchdog_overridden[_WATCHDOG_TYPE_MAX];
+        char *watchdog_pretimeout_governor;
+        char *watchdog_pretimeout_governor_overridden;
 
         dual_timestamp timestamps[_MANAGER_TIMESTAMP_MAX];
 
@@ -575,6 +577,8 @@ ManagerTimestamp manager_timestamp_initrd_mangle(ManagerTimestamp s);
 usec_t manager_get_watchdog(Manager *m, WatchdogType t);
 void manager_set_watchdog(Manager *m, WatchdogType t, usec_t timeout);
 void manager_override_watchdog(Manager *m, WatchdogType t, usec_t timeout);
+int manager_set_watchdog_pretimeout_governor(Manager *m, const char *governor);
+int manager_override_watchdog_pretimeout_governor(Manager *m, const char *governor);
 
 const char* oom_policy_to_string(OOMPolicy i) _const_;
 OOMPolicy oom_policy_from_string(const char *s) _pure_;

src/core/system.conf.in

@@ -31,6 +31,7 @@
 #NUMAMask=
 #RuntimeWatchdogSec=off
 #RuntimeWatchdogPreSec=off
+#RuntimeWatchdogPreGovernor=
 #RebootWatchdogSec=10min
 #KExecWatchdogSec=off
 #WatchdogDevice=

src/shared/watchdog.c

@@ -22,6 +22,7 @@ static usec_t watchdog_timeout; /* 0 → close device and USEC_INFINITY → don'
 static usec_t watchdog_pretimeout; /* 0 → disable pretimeout and USEC_INFINITY → don't change pretimeout */
 static usec_t watchdog_last_ping = USEC_INFINITY;
 static bool watchdog_supports_pretimeout = false; /* Depends on kernel state that might change at runtime */
+static char *watchdog_pretimeout_governor = NULL;
 
 /* Starting from kernel version 4.5, the maximum allowable watchdog timeout is
  * UINT_MAX/1000U seconds (since internal calculations are done in milliseconds
@@ -76,6 +77,28 @@ static int get_pretimeout_governor(char **ret_gov) {
         return 0;
 }
 
+static int set_pretimeout_governor(const char *governor) {
+        _cleanup_free_ char *sys_fn = NULL;
+        int r;
+
+        if (isempty(governor))
+                return 0; /* Nothing to do */
+
+        r = get_watchdog_sysfs_path("pretimeout_governor", &sys_fn);
+        if (r < 0)
+                return r;
+
+        log_info("Watchdog: setting pretimeout_governor to '%s' via '%s'", governor, sys_fn);
+
+        r = write_string_file(sys_fn,
+                              governor,
+                              WRITE_STRING_FILE_DISABLE_BUFFER | WRITE_STRING_FILE_VERIFY_ON_FAILURE | WRITE_STRING_FILE_VERIFY_IGNORE_NEWLINE);
+        if (r < 0)
+                return log_error_errno(r, "Failed to set pretimeout_governor to '%s': %m", governor);
+
+        return r;
+}
+
 static int watchdog_set_enable(bool enable) {
         int flags = enable ? WDIOS_ENABLECARD : WDIOS_DISABLECARD;
 
@@ -194,6 +217,9 @@ static int update_pretimeout(void) {
          * might have been unloaded. */
         watchdog_supports_pretimeout = false;
 
+        /* Update the pretimeout governor as well */
+        (void) set_pretimeout_governor(watchdog_pretimeout_governor);
+
         r = get_pretimeout_governor(&governor);
         if (r < 0)
                 return log_warning_errno(r, "Watchdog: failed to read pretimeout governor: %m");
@@ -362,6 +388,13 @@ int watchdog_setup_pretimeout(usec_t timeout) {
         return update_pretimeout();
 }
 
+int watchdog_setup_pretimeout_governor(const char *governor) {
+        if (free_and_strdup(&watchdog_pretimeout_governor, governor) < 0)
+                return -ENOMEM;
+
+        return set_pretimeout_governor(watchdog_pretimeout_governor);
+}
+
 static usec_t calc_timeout(void) {
         /* Calculate the effective timeout which accounts for the watchdog
          * pretimeout if configured and supported. */

src/shared/watchdog.h

@@ -9,6 +9,7 @@
 int watchdog_set_device(const char *path);
 int watchdog_setup(usec_t timeout);
 int watchdog_setup_pretimeout(usec_t usec);
+int watchdog_setup_pretimeout_governor(const char *governor);
 int watchdog_ping(void);
 void watchdog_close(bool disarm);
 usec_t watchdog_runtime_wait(void);

test/fuzz/fuzz-unit-file/directives-all.service

@@ -738,6 +738,7 @@ LogLocation=
 LogTarget=
 RuntimeWatchdogSec=
 RuntimeWatchdogPreSec=
+RuntimeWatchdogPreGovernor=
 ShowStatus=
 RebootWatchdogSec=
 ShutdownWatchdogSec=