journal: Add --convert= command to journalctl

--convert writes the journal files read by journalctl to the given
location. The location should be specified as a full journal file
path (e.g. /a/b/c/converted.journal). The directory specifies where
the converted journal files will be stored. The filename specifies
the naming convention the converted journal files will follow.

man/journalctl.xml

@@ -854,6 +854,17 @@
         cryptographic theory it is based on.</para></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>--convert=</option></term>
+
+        <listitem><para>Converts the specified journal files to the latest supported journal format. Takes
+        the path to store the converted journal files. The path should include the filename to be used for
+        the converted files, with the <literal>.journal</literal> extension (e.g.
+        <filename>/a/b/c/converted.journal</filename> will store the journal files in the
+        <filename>/a/b/c</filename> directory using <filename>converted.journal</filename> as the filename).
+        </para></listitem>
+      </varlistentry>
+
       <xi:include href="standard-options.xml" xpointer="help" />
       <xi:include href="standard-options.xml" xpointer="version" />
     </variablelist>

meson.build

@@ -2293,11 +2293,13 @@ public_programs += executable(
         install : true)
 
 if get_option('link-journalctl-shared')
-        journalctl_link_with = [libshared]
+        journalctl_link_with = [libshared,
+                                libjournal_core]
 else
         journalctl_link_with = [libsystemd_static,
                                 libshared_static,
-                                libbasic_gcrypt]
+                                libbasic_gcrypt,
+                                libjournal_core]
 endif
 
 public_programs += executable(

src/journal/journalctl.c

@@ -44,6 +44,7 @@
 #include "locale-util.h"
 #include "log.h"
 #include "logs-show.h"
+#include "managed-journal-file.h"
 #include "memory-util.h"
 #include "mkdir.h"
 #include "mount-util.h"
@@ -128,6 +129,7 @@ static uint64_t arg_vacuum_size = 0;
 static uint64_t arg_vacuum_n_files = 0;
 static usec_t arg_vacuum_time = 0;
 static char **arg_output_fields = NULL;
+static const char *arg_convert = NULL;
 static const char *arg_pattern = NULL;
 static pcre2_code *arg_compiled_pattern = NULL;
 static PatternCompileCase arg_case = PATTERN_COMPILE_CASE_AUTO;
@@ -162,6 +164,7 @@ static enum {
         ACTION_ROTATE_AND_VACUUM,
         ACTION_LIST_FIELDS,
         ACTION_LIST_FIELD_NAMES,
+        ACTION_CONVERT,
 } arg_action = ACTION_SHOW;
 
 typedef struct BootId {
@@ -387,6 +390,7 @@ static int help(void) {
                "     --dump-catalog          Show entries in the message catalog\n"
                "     --update-catalog        Update the message catalog database\n"
                "     --setup-keys            Generate a new FSS key pair\n"
+               "     --convert=PATH          Convert the journal to the latest journal format\n"
                "\nSee the %2$s for details.\n",
                program_invocation_short_name,
                link,
@@ -441,6 +445,7 @@ static int parse_argv(int argc, char *argv[]) {
                 ARG_NO_HOSTNAME,
                 ARG_OUTPUT_FIELDS,
                 ARG_NAMESPACE,
+                ARG_CONVERT,
         };
 
         static const struct option options[] = {
@@ -508,6 +513,7 @@ static int parse_argv(int argc, char *argv[]) {
                 { "no-hostname",          no_argument,       NULL, ARG_NO_HOSTNAME          },
                 { "output-fields",        required_argument, NULL, ARG_OUTPUT_FIELDS        },
                 { "namespace",            required_argument, NULL, ARG_NAMESPACE            },
+                { "convert",              required_argument, NULL, ARG_CONVERT              },
                 {}
         };
 
@@ -1034,6 +1040,11 @@ static int parse_argv(int argc, char *argv[]) {
                         break;
                 }
 
+                case ARG_CONVERT:
+                        arg_action = ACTION_CONVERT;
+                        arg_convert = optarg;
+                        break;
+
                 case '?':
                         return -EINVAL;
 
@@ -2093,6 +2104,52 @@ static int wait_for_change(sd_journal *j, int poll_fd) {
         return 0;
 }
 
+static int journal_convert(sd_journal *j) {
+        _cleanup_(managed_journal_file_closep) ManagedJournalFile *to = NULL;
+        _cleanup_(mmap_cache_unrefp) MMapCache *mmap = NULL;
+        int r;
+
+        assert(arg_convert);
+
+        mmap = mmap_cache_new();
+        if (!mmap)
+                return -ENOMEM;
+
+        r = managed_journal_file_open(-1, arg_convert, O_RDWR | O_CREAT, JOURNAL_COMPRESS, 0640, UINT64_MAX,
+                                      &(JournalMetrics) { -1, -1, -1, -1, -1, -1 }, mmap, NULL, NULL, &to);
+        if (r < 0)
+                return log_error_errno(r, "Failed to open journal: %m");
+
+        SD_JOURNAL_FOREACH(j) {
+                Object *o;
+                JournalFile *from;
+
+                from = j->current_file;
+                assert(from && from->current_offset > 0);
+
+                r = journal_file_move_to_object(from, OBJECT_ENTRY, from->current_offset, &o);
+                if (r < 0)
+                        return log_error_errno(r, "Can't read entry: %m");
+
+                r = journal_file_copy_entry(from, to->file, o, from->current_offset);
+                if (r >= 0)
+                        continue;
+
+                if (!journal_shall_try_append_again(to->file, r))
+                        return log_error_errno(r, "Can't write entry: %m");
+
+                r = managed_journal_file_rotate(&to, mmap, JOURNAL_COMPRESS, UINT64_MAX, NULL);
+                if (r < 0)
+                        return r;
+
+                r = journal_file_copy_entry(from, to->file, o, from->current_offset);
+                if (r < 0)
+                        return log_error_errno(r, "Can't write entry: %m");
+        }
+
+        return 0;
+}
+
 int main(int argc, char *argv[]) {
         _cleanup_(loop_device_unrefp) LoopDevice *loop_device = NULL;
         _cleanup_(umount_and_rmdir_and_freep) char *unlink_dir = NULL;
@@ -2203,6 +2260,7 @@ int main(int argc, char *argv[]) {
         case ACTION_ROTATE_AND_VACUUM:
         case ACTION_LIST_FIELDS:
         case ACTION_LIST_FIELD_NAMES:
+        case ACTION_CONVERT:
                 /* These ones require access to the journal files, continue below. */
                 break;
 
@@ -2357,6 +2415,10 @@ int main(int argc, char *argv[]) {
         case ACTION_LIST_FIELDS:
                 break;
 
+        case ACTION_CONVERT:
+                r = journal_convert(j);
+                goto finish;
+
         default:
                 assert_not_reached();
         }

src/journal/journald-server.c

@@ -30,6 +30,7 @@
 #include "io-util.h"
 #include "journal-authenticate.h"
 #include "journal-internal.h"
+#include "journal-util.h"
 #include "journal-vacuum.h"
 #include "journald-audit.h"
 #include "journald-context.h"
@@ -769,55 +770,6 @@ static void server_cache_hostname(Server *s) {
         free_and_replace(s->hostname_field, x);
 }
 
-static bool shall_try_append_again(JournalFile *f, int r) {
-        switch (r) {
-
-        case -E2BIG:           /* Hit configured limit          */
-        case -EFBIG:           /* Hit fs limit                  */
-        case -EDQUOT:          /* Quota limit hit               */
-        case -ENOSPC:          /* Disk full                     */
-                log_debug("%s: Allocation limit reached, rotating.", f->path);
-                return true;
-
-        case -EIO:             /* I/O error of some kind (mmap) */
-                log_warning("%s: IO error, rotating.", f->path);
-                return true;
-
-        case -EHOSTDOWN:       /* Other machine                 */
-                log_info("%s: Journal file from other machine, rotating.", f->path);
-                return true;
-
-        case -EBUSY:           /* Unclean shutdown              */
-                log_info("%s: Unclean shutdown, rotating.", f->path);
-                return true;
-
-        case -EPROTONOSUPPORT: /* Unsupported feature           */
-                log_info("%s: Unsupported feature, rotating.", f->path);
-                return true;
-
-        case -EBADMSG:         /* Corrupted                     */
-        case -ENODATA:         /* Truncated                     */
-        case -ESHUTDOWN:       /* Already archived              */
-                log_warning("%s: Journal file corrupted, rotating.", f->path);
-                return true;
-
-        case -EIDRM:           /* Journal file has been deleted */
-                log_warning("%s: Journal file has been deleted, rotating.", f->path);
-                return true;
-
-        case -ETXTBSY:         /* Journal file is from the future */
-                log_warning("%s: Journal file is from the future, rotating.", f->path);
-                return true;
-
-        case -EAFNOSUPPORT:
-                log_warning("%s: underlying file system does not support memory mapping or another required file system feature.", f->path);
-                return false;
-
-        default:
-                return false;
-        }
-}
-
 static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n, int priority) {
         bool vacuumed = false, rotate = false;
         struct dual_timestamp ts;
@@ -872,7 +824,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n
                 return;
         }
 
-        if (vacuumed || !shall_try_append_again(f->file, r)) {
+        if (vacuumed || !journal_shall_try_append_again(f->file, r)) {
                 log_ratelimit_full_errno(LOG_ERR, r, "Failed to write entry (%zu items, %zu bytes), ignoring: %m", n, IOVEC_TOTAL_SIZE(iovec, n));
                 return;
         }
@@ -1202,7 +1154,7 @@ int server_flush_to_var(Server *s, bool require_flag_file) {
                 if (r >= 0)
                         continue;
 
-                if (!shall_try_append_again(s->system_journal->file, r)) {
+                if (!journal_shall_try_append_again(s->system_journal->file, r)) {
                         log_error_errno(r, "Can't write entry: %m");
                         goto finish;
                 }

src/shared/journal-util.c

@@ -136,3 +136,52 @@ int journal_access_check_and_warn(sd_journal *j, bool quiet, bool want_other_use
 
         return r;
 }
+
+bool journal_shall_try_append_again(JournalFile *f, int r) {
+        switch (r) {
+
+        case -E2BIG:           /* Hit configured limit          */
+        case -EFBIG:           /* Hit fs limit                  */
+        case -EDQUOT:          /* Quota limit hit               */
+        case -ENOSPC:          /* Disk full                     */
+                log_debug("%s: Allocation limit reached, rotating.", f->path);
+                return true;
+
+        case -EIO:             /* I/O error of some kind (mmap) */
+                log_warning("%s: IO error, rotating.", f->path);
+                return true;
+
+        case -EHOSTDOWN:       /* Other machine                 */
+                log_info("%s: Journal file from other machine, rotating.", f->path);
+                return true;
+
+        case -EBUSY:           /* Unclean shutdown              */
+                log_info("%s: Unclean shutdown, rotating.", f->path);
+                return true;
+
+        case -EPROTONOSUPPORT: /* Unsupported feature           */
+                log_info("%s: Unsupported feature, rotating.", f->path);
+                return true;
+
+        case -EBADMSG:         /* Corrupted                     */
+        case -ENODATA:         /* Truncated                     */
+        case -ESHUTDOWN:       /* Already archived              */
+                log_warning("%s: Journal file corrupted, rotating.", f->path);
+                return true;
+
+        case -EIDRM:           /* Journal file has been deleted */
+                log_warning("%s: Journal file has been deleted, rotating.", f->path);
+                return true;
+
+        case -ETXTBSY:         /* Journal file is from the future */
+                log_warning("%s: Journal file is from the future, rotating.", f->path);
+                return true;
+
+        case -EAFNOSUPPORT:
+                log_warning("%s: underlying file system does not support memory mapping or another required file system feature.", f->path);
+                return false;
+
+        default:
+                return false;
+        }
+}

src/shared/journal-util.h

@@ -6,5 +6,9 @@
 
 #include "sd-journal.h"
 
+#include "journal-internal.h"
+
 int journal_access_blocked(sd_journal *j);
 int journal_access_check_and_warn(sd_journal *j, bool quiet, bool want_other_users);
+
+bool journal_shall_try_append_again(JournalFile *f, int r);