journald: allow restarting journald without losing stream connections

Making use of the fd storage capability of the previous commit, allow
restarting journald by serilizing stream state to /run, and pushing open
fds to PID 1.

src/journal/journald-server.c

@@ -1182,6 +1182,10 @@ int server_process_datagram(sd_event_source *es, int fd, uint32_t revents, void
                         log_error_errno(errno, "recvmsg() failed: %m");
                         return -errno;
                 }
+                if (n == 0) {
+                        log_error("Got EOF on socket.");
+                        return -ECONNRESET;
+                }
 
                 for (cmsg = CMSG_FIRSTHDR(&msghdr); cmsg; cmsg = CMSG_NXTHDR(&msghdr, cmsg)) {
 
@@ -1462,6 +1466,7 @@ static int server_open_hostname(Server *s) {
 }
 
 int server_init(Server *s) {
+        _cleanup_fdset_free_ FDSet *fds = NULL;
         int n, r, fd;
 
         assert(s);
@@ -1558,17 +1563,28 @@ int server_init(Server *s) {
                         s->audit_fd = fd;
 
                 } else {
-                        log_warning("Unknown socket passed as file descriptor %d, ignoring.", fd);
 
-                        /* Let's close the fd, better be safe than
-                           sorry. The fd might reference some resource
-                           that we really want to release if we don't
-                           make use of it. */
+                        if (!fds) {
+                                fds = fdset_new();
+                                if (!fds)
+                                        return log_oom();
+                        }
 
-                        safe_close(fd);
+                        r = fdset_put(fds, fd);
+                        if (r < 0)
+                                return log_oom();
                 }
         }
 
+        r = server_open_stdout_socket(s, fds);
+        if (r < 0)
+                return r;
+
+        if (fdset_size(fds) > 0) {
+                log_warning("%u unknown file descriptors passed, closing.", fdset_size(fds));
+                fds = fdset_free(fds);
+        }
+
         r = server_open_syslog_socket(s);
         if (r < 0)
                 return r;
@@ -1577,10 +1593,6 @@ int server_init(Server *s) {
         if (r < 0)
                 return r;
 
-        r = server_open_stdout_socket(s);
-        if (r < 0)
-                return r;
-
         r = server_open_dev_kmsg(s);
         if (r < 0)
                 return r;

src/journal/journald-stream.c

@@ -28,8 +28,11 @@
 #endif
 
 #include "sd-event.h"
+#include "sd-daemon.h"
 #include "socket-util.h"
 #include "selinux-util.h"
+#include "mkdir.h"
+#include "fileio.h"
 #include "journald-server.h"
 #include "journald-stream.h"
 #include "journald-syslog.h"
@@ -69,14 +72,153 @@ struct StdoutStream {
         bool forward_to_kmsg:1;
         bool forward_to_console:1;
 
+        bool fdstore:1;
+
         char buffer[LINE_MAX+1];
         size_t length;
 
         sd_event_source *event_source;
 
+        char *state_file;
+
         LIST_FIELDS(StdoutStream, stdout_stream);
 };
 
+void stdout_stream_free(StdoutStream *s) {
+        if (!s)
+                return;
+
+        if (s->server) {
+                assert(s->server->n_stdout_streams > 0);
+                s->server->n_stdout_streams --;
+                LIST_REMOVE(stdout_stream, s->server->stdout_streams, s);
+        }
+
+        if (s->event_source) {
+                sd_event_source_set_enabled(s->event_source, SD_EVENT_OFF);
+                s->event_source = sd_event_source_unref(s->event_source);
+        }
+
+        safe_close(s->fd);
+
+#ifdef HAVE_SELINUX
+        if (s->security_context)
+                freecon(s->security_context);
+#endif
+
+        free(s->identifier);
+        free(s->unit_id);
+        free(s->state_file);
+
+        free(s);
+}
+
+DEFINE_TRIVIAL_CLEANUP_FUNC(StdoutStream*, stdout_stream_free);
+
+static void stdout_stream_destroy(StdoutStream *s) {
+        if (!s)
+                return;
+
+        if (s->state_file)
+                unlink(s->state_file);
+
+        stdout_stream_free(s);
+}
+
+static int stdout_stream_save(StdoutStream *s) {
+        _cleanup_free_ char *temp_path = NULL;
+        _cleanup_fclose_ FILE *f = NULL;
+        int r;
+
+        assert(s);
+
+        if (s->state != STDOUT_STREAM_RUNNING)
+                return 0;
+
+        if (!s->state_file) {
+                struct stat st;
+
+                r = fstat(s->fd, &st);
+                if (r < 0)
+                        return log_warning_errno(errno, "Failed to stat connected stream: %m");
+
+                /* We use device and inode numbers as identifier for the stream */
+                if (asprintf(&s->state_file, "/run/systemd/journal/streams/%lu:%lu", (unsigned long) st.st_dev, (unsigned long) st.st_ino) < 0)
+                        return log_oom();
+        }
+
+        mkdir_p("/run/systemd/journal/streams", 0755);
+
+        r = fopen_temporary(s->state_file, &f, &temp_path);
+        if (r < 0)
+                goto finish;
+
+        fprintf(f,
+                "# This is private data. Do not parse\n"
+                "PRIORITY=%i\n"
+                "LEVEL_PREFIX=%i\n"
+                "FORWARD_TO_SYSLOG=%i\n"
+                "FORWARD_TO_KMSG=%i\n"
+                "FORWARD_TO_CONSOLE=%i\n",
+                s->priority,
+                s->level_prefix,
+                s->forward_to_syslog,
+                s->forward_to_kmsg,
+                s->forward_to_console);
+
+        if (!isempty(s->identifier)) {
+                _cleanup_free_ char *escaped;
+
+                escaped = cescape(s->identifier);
+                if (!escaped) {
+                        r = -ENOMEM;
+                        goto finish;
+                }
+
+                fprintf(f, "IDENTIFIER=%s\n", escaped);
+        }
+
+        if (!isempty(s->unit_id)) {
+                _cleanup_free_ char *escaped;
+
+                escaped = cescape(s->unit_id);
+                if (!escaped) {
+                        r = -ENOMEM;
+                        goto finish;
+                }
+
+                fprintf(f, "UNIT=%s\n", escaped);
+        }
+
+        r = fflush_and_check(f);
+        if (r < 0)
+                goto finish;
+
+        if (rename(temp_path, s->state_file) < 0) {
+                r = -errno;
+                goto finish;
+        }
+
+        free(temp_path);
+        temp_path = NULL;
+
+        /* Store the connection fd in PID 1, so that we get it passed
+         * in again on next start */
+        if (!s->fdstore) {
+                sd_pid_notify_with_fds(0, false, "FDSTORE=1", &s->fd, 1);
+                s->fdstore = true;
+        }
+
+finish:
+        if (temp_path)
+                unlink(temp_path);
+
+        if (r < 0)
+                log_error_errno(r, "Failed to save stream data %s: %m", s->state_file);
+
+        return r;
+}
+
 static int stdout_stream_log(StdoutStream *s, const char *p) {
         struct iovec iovec[N_IOVEC_META_FIELDS + 5];
         int priority;
@@ -229,6 +371,9 @@ static int stdout_stream_line(StdoutStream *s, char *p) {
 
                 s->forward_to_console = !!r;
                 s->state = STDOUT_STREAM_RUNNING;
+
+                /* Try to save the stream, so that journald can be restarted and we can recover */
+                (void) stdout_stream_save(s);
                 return 0;
 
         case STDOUT_STREAM_RUNNING:
@@ -323,40 +468,63 @@ static int stdout_stream_process(sd_event_source *es, int fd, uint32_t revents,
         return 1;
 
 terminate:
-        stdout_stream_free(s);
+        stdout_stream_destroy(s);
         return 0;
 }
 
-void stdout_stream_free(StdoutStream *s) {
+static int stdout_stream_install(Server *s, int fd, StdoutStream **ret) {
+        _cleanup_(stdout_stream_freep) StdoutStream *stream = NULL;
+        int r;
+
         assert(s);
+        assert(fd >= 0);
 
-        if (s->server) {
-                assert(s->server->n_stdout_streams > 0);
-                s->server->n_stdout_streams --;
-                LIST_REMOVE(stdout_stream, s->server->stdout_streams, s);
-        }
+        stream = new0(StdoutStream, 1);
+        if (!stream)
+                return log_oom();
 
-        if (s->event_source) {
-                sd_event_source_set_enabled(s->event_source, SD_EVENT_OFF);
-                s->event_source = sd_event_source_unref(s->event_source);
-        }
+        stream->fd = -1;
+        stream->priority = LOG_INFO;
 
-        safe_close(s->fd);
+        r = getpeercred(fd, &stream->ucred);
+        if (r < 0)
+                return log_error_errno(r, "Failed to determine peer credentials: %m");
 
 #ifdef HAVE_SELINUX
-        if (s->security_context)
-                freecon(s->security_context);
+        if (mac_selinux_use()) {
+                if (getpeercon(fd, &stream->security_context) < 0 && errno != ENOPROTOOPT)
+                        log_error_errno(errno, "Failed to determine peer security context: %m");
+        }
 #endif
 
-        free(s->identifier);
-        free(s->unit_id);
-        free(s);
+        (void) shutdown(fd, SHUT_WR);
+
+        r = sd_event_add_io(s->event, &stream->event_source, fd, EPOLLIN, stdout_stream_process, stream);
+        if (r < 0)
+                return log_error_errno(r, "Failed to add stream to event loop: %m");
+
+        r = sd_event_source_set_priority(stream->event_source, SD_EVENT_PRIORITY_NORMAL+5);
+        if (r < 0)
+                return log_error_errno(r, "Failed to adjust stdout event source priority: %m");
+
+        stream->fd = fd;
+
+        stream->server = s;
+        LIST_PREPEND(stdout_stream, s->stdout_streams, stream);
+        s->n_stdout_streams ++;
+
+        if (ret)
+                *ret = stream;
+
+        stream = NULL;
+
+        return 0;
 }
 
 static int stdout_stream_new(sd_event_source *es, int listen_fd, uint32_t revents, void *userdata) {
+        _cleanup_close_ int fd = -1;
         Server *s = userdata;
-        StdoutStream *stream;
-        int fd, r;
+        int r;
 
         assert(s);
 
@@ -376,60 +544,163 @@ static int stdout_stream_new(sd_event_source *es, int listen_fd, uint32_t revent
 
         if (s->n_stdout_streams >= STDOUT_STREAMS_MAX) {
                 log_warning("Too many stdout streams, refusing connection.");
-                safe_close(fd);
                 return 0;
         }
 
-        stream = new0(StdoutStream, 1);
-        if (!stream) {
-                safe_close(fd);
-                return log_oom();
+        r = stdout_stream_install(s, fd, NULL);
+        if (r < 0)
+                return r;
+
+        fd = -1;
+        return 0;
+}
+
+static int stdout_stream_load(StdoutStream *stream, const char *fname) {
+        _cleanup_free_ char
+                *priority = NULL,
+                *level_prefix = NULL,
+                *forward_to_syslog = NULL,
+                *forward_to_kmsg = NULL,
+                *forward_to_console = NULL;
+        int r;
+
+        assert(stream);
+        assert(fname);
+
+        if (!stream->state_file) {
+                stream->state_file = strappend("/run/systemd/journal/streams/", fname);
+                if (!stream->state_file)
+                        return log_oom();
         }
 
-        stream->fd = fd;
+        r = parse_env_file(stream->state_file, NEWLINE,
+                           "PRIORITY", &priority,
+                           "LEVEL_PREFIX", &level_prefix,
+                           "FORWARD_TO_SYSLOG", &forward_to_syslog,
+                           "FORWARD_TO_KMSG", &forward_to_kmsg,
+                           "FORWARD_TO_CONSOLE", &forward_to_console,
+                           "IDENTIFIER", &stream->identifier,
+                           "UNIT", &stream->unit_id,
+                           NULL);
+        if (r < 0)
+                return log_error_errno(r, "Failed to read: %s", stream->state_file);
 
-        r = getpeercred(fd, &stream->ucred);
-        if (r < 0) {
-                log_error_errno(errno, "Failed to determine peer credentials: %m");
-                goto fail;
+        if (priority) {
+                int p;
+
+                p = log_level_from_string(priority);
+                if (p >= 0)
+                        stream->priority = p;
         }
 
-#ifdef HAVE_SELINUX
-        if (mac_selinux_use()) {
-                if (getpeercon(fd, &stream->security_context) < 0 && errno != ENOPROTOOPT)
-                        log_error_errno(errno, "Failed to determine peer security context: %m");
-        }
-#endif
-
-        if (shutdown(fd, SHUT_WR) < 0) {
-                log_error_errno(errno, "Failed to shutdown writing side of socket: %m");
-                goto fail;
+        if (level_prefix) {
+                r = parse_boolean(level_prefix);
+                if (r >= 0)
+                        stream->level_prefix = r;
         }
 
-        r = sd_event_add_io(s->event, &stream->event_source, fd, EPOLLIN, stdout_stream_process, stream);
-        if (r < 0) {
-                log_error_errno(r, "Failed to add stream to event loop: %m");
-                goto fail;
+        if (forward_to_syslog) {
+                r = parse_boolean(forward_to_syslog);
+                if (r >= 0)
+                        stream->forward_to_syslog = r;
         }
 
-        r = sd_event_source_set_priority(stream->event_source, SD_EVENT_PRIORITY_NORMAL+5);
-        if (r < 0) {
-                log_error_errno(r, "Failed to adjust stdout event source priority: %m");
-                goto fail;
+        if (forward_to_kmsg) {
+                r = parse_boolean(forward_to_kmsg);
+                if (r >= 0)
+                        stream->forward_to_kmsg = r;
         }
 
-        stream->server = s;
-        LIST_PREPEND(stdout_stream, s->stdout_streams, stream);
-        s->n_stdout_streams ++;
+        if (forward_to_console) {
+                r = parse_boolean(forward_to_console);
+                if (r >= 0)
+                        stream->forward_to_console = r;
+        }
+
+        return 0;
+}
+
+static int stdout_stream_restore(Server *s, const char *fname, int fd) {
+        StdoutStream *stream;
+        int r;
+
+        assert(s);
+        assert(fname);
+        assert(fd >= 0);
+
+        if (s->n_stdout_streams >= STDOUT_STREAMS_MAX) {
+                log_warning("Too many stdout streams, refusing restoring of stream.");
+                return -ENOBUFS;
+        }
+
+        r = stdout_stream_install(s, fd, &stream);
+        if (r < 0)
+                return r;
+
+        stream->state = STDOUT_STREAM_RUNNING;
+        stream->fdstore = true;
+
+        /* Ignore all parsing errors */
+        (void) stdout_stream_load(stream, fname);
+
+        return 0;
+}
+
+static int server_restore_streams(Server *s, FDSet *fds) {
+        _cleanup_closedir_ DIR *d = NULL;
+        struct dirent *de;
+        int r;
+
+        d = opendir("/run/systemd/journal/streams");
+        if (!d) {
+                if (errno == ENOENT)
+                        return 0;
+
+                return log_warning_errno(errno, "Failed to enumerate /run/systemd/journal/streams: %m");
+        }
+
+        FOREACH_DIRENT(de, d, goto fail) {
+                unsigned long st_dev, st_ino;
+                bool found = false;
+                Iterator i;
+                int fd;
+
+                if (sscanf(de->d_name, "%lu:%lu", &st_dev, &st_ino) != 2)
+                        continue;
+
+                FDSET_FOREACH(fd, fds, i) {
+                        struct stat st;
+
+                        if (fstat(fd, &st) < 0)
+                                return log_error_errno(errno, "Failed to stat %s: %m", de->d_name);
+
+                        if (S_ISSOCK(st.st_mode) && st.st_dev == st_dev && st.st_ino == st_ino) {
+                                found = true;
+                                break;
+                        }
+                }
+
+                if (!found) {
+                        /* No file descriptor? Then let's delete the state file */
+                        log_debug("Cannot restore stream file %s", de->d_name);
+                        unlinkat(dirfd(d), de->d_name, 0);
+                        continue;
+                }
+
+                fdset_remove(fds, fd);
+
+                r = stdout_stream_restore(s, de->d_name, fd);
+                if (r < 0)
+                        safe_close(fd);
+        }
 
         return 0;
 
 fail:
-        stdout_stream_free(stream);
-        return 0;
+        return log_error_errno(errno, "Failed to read streams directory: %m");
 }
 
-int server_open_stdout_socket(Server *s) {
+int server_open_stdout_socket(Server *s, FDSet *fds) {
         int r;
 
         assert(s);
@@ -465,5 +736,8 @@ int server_open_stdout_socket(Server *s) {
         if (r < 0)
                 return log_error_errno(r, "Failed to adjust priority of stdout server event source: %m");
 
+        /* Try to restore streams, but don't bother if this fails */
+        (void) server_restore_streams(s, fds);
+
         return 0;
 }

src/journal/journald-stream.h

@@ -21,8 +21,9 @@
   along with systemd; If not, see <http://www.gnu.org/licenses/>.
 ***/
 
+#include "fdset.h"
 #include "journald-server.h"
 
-int server_open_stdout_socket(Server *s);
+int server_open_stdout_socket(Server *s, FDSet *fds);
 
 void stdout_stream_free(StdoutStream *s);

units/systemd-journald.service.in

@@ -23,6 +23,7 @@ NotifyAccess=all
 StandardOutput=null
 CapabilityBoundingSet=CAP_SYS_ADMIN CAP_DAC_OVERRIDE CAP_SYS_PTRACE CAP_SYSLOG CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_CHOWN CAP_DAC_READ_SEARCH CAP_FOWNER CAP_SETUID CAP_SETGID CAP_MAC_OVERRIDE
 WatchdogSec=1min
+FileDescriptorStoreMax=1024
 
 # Increase the default a bit in order to allow many simultaneous
 # services being run since we keep one fd open per service.