diff --git a/Makefile-man.am b/Makefile-man.am index 12900daa96..28b5fb6adb 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -72,6 +72,7 @@ MANPAGES += \ man/sd_id128_to_string.3 \ man/sd_is_fifo.3 \ man/sd_journal_add_match.3 \ + man/sd_journal_enumerate_fields.3 \ man/sd_journal_get_catalog.3 \ man/sd_journal_get_cursor.3 \ man/sd_journal_get_cutoff_realtime_usec.3 \ @@ -237,6 +238,7 @@ MANPAGES_ALIAS += \ man/SD_JOURNAL_FOREACH.3 \ man/SD_JOURNAL_FOREACH_BACKWARDS.3 \ man/SD_JOURNAL_FOREACH_DATA.3 \ + man/SD_JOURNAL_FOREACH_FIELD.3 \ man/SD_JOURNAL_FOREACH_UNIQUE.3 \ man/SD_JOURNAL_INVALIDATE.3 \ man/SD_JOURNAL_LOCAL_ONLY.3 \ @@ -397,6 +399,7 @@ MANPAGES_ALIAS += \ man/sd_journal_process.3 \ man/sd_journal_reliable_fd.3 \ man/sd_journal_restart_data.3 \ + man/sd_journal_restart_fields.3 \ man/sd_journal_restart_unique.3 \ man/sd_journal_seek_cursor.3 \ man/sd_journal_seek_monotonic_usec.3 \ @@ -565,6 +568,7 @@ man/SD_JOURNAL_CURRENT_USER.3: man/sd_journal_open.3 man/SD_JOURNAL_FOREACH.3: man/sd_journal_next.3 man/SD_JOURNAL_FOREACH_BACKWARDS.3: man/sd_journal_next.3 man/SD_JOURNAL_FOREACH_DATA.3: man/sd_journal_get_data.3 +man/SD_JOURNAL_FOREACH_FIELD.3: man/sd_journal_enumerate_fields.3 man/SD_JOURNAL_FOREACH_UNIQUE.3: man/sd_journal_query_unique.3 man/SD_JOURNAL_INVALIDATE.3: man/sd_journal_get_fd.3 man/SD_JOURNAL_LOCAL_ONLY.3: man/sd_journal_open.3 @@ -725,6 +729,7 @@ man/sd_journal_printv.3: man/sd_journal_print.3 man/sd_journal_process.3: man/sd_journal_get_fd.3 man/sd_journal_reliable_fd.3: man/sd_journal_get_fd.3 man/sd_journal_restart_data.3: man/sd_journal_get_data.3 +man/sd_journal_restart_fields.3: man/sd_journal_enumerate_fields.3 man/sd_journal_restart_unique.3: man/sd_journal_query_unique.3 man/sd_journal_seek_cursor.3: man/sd_journal_seek_head.3 man/sd_journal_seek_monotonic_usec.3: man/sd_journal_seek_head.3 @@ -1017,6 +1022,9 @@ man/SD_JOURNAL_FOREACH_BACKWARDS.html: man/sd_journal_next.html man/SD_JOURNAL_FOREACH_DATA.html: man/sd_journal_get_data.html $(html-alias) +man/SD_JOURNAL_FOREACH_FIELD.html: man/sd_journal_enumerate_fields.html + $(html-alias) + man/SD_JOURNAL_FOREACH_UNIQUE.html: man/sd_journal_query_unique.html $(html-alias) @@ -1497,6 +1505,9 @@ man/sd_journal_reliable_fd.html: man/sd_journal_get_fd.html man/sd_journal_restart_data.html: man/sd_journal_get_data.html $(html-alias) +man/sd_journal_restart_fields.html: man/sd_journal_enumerate_fields.html + $(html-alias) + man/sd_journal_restart_unique.html: man/sd_journal_query_unique.html $(html-alias) @@ -2534,6 +2545,7 @@ EXTRA_DIST += \ man/sd_id128_to_string.xml \ man/sd_is_fifo.xml \ man/sd_journal_add_match.xml \ + man/sd_journal_enumerate_fields.xml \ man/sd_journal_get_catalog.xml \ man/sd_journal_get_cursor.xml \ man/sd_journal_get_cutoff_realtime_usec.xml \ diff --git a/man/journalctl.xml b/man/journalctl.xml index b57afb6ebf..5af98c67cb 100644 --- a/man/journalctl.xml +++ b/man/journalctl.xml @@ -571,6 +571,13 @@ field can take in all entries of the journal. + + + + + Print all field names currently used in all entries of the journal. + + diff --git a/man/sd-journal.xml b/man/sd-journal.xml index a1185d372b..09747a480c 100644 --- a/man/sd-journal.xml +++ b/man/sd-journal.xml @@ -77,6 +77,8 @@ sd_journal_get_realtime_usec3, sd_journal_add_match3, sd_journal_seek_head3, + sd_journal_query_enumerate3, + sd_journal_enumerate_fields3, sd_journal_get_cursor3, sd_journal_get_cutoff_realtime_usec3, sd_journal_get_cutoff_monotonic_usec3, @@ -111,6 +113,8 @@ sd_journal_get_realtime_usec3, sd_journal_add_match3, sd_journal_seek_head3, + sd_journal_query_enumerate3, + sd_journal_enumerate_fields3, sd_journal_get_cursor3, sd_journal_get_cutoff_realtime_usec3, sd_journal_get_cutoff_monotonic_usec3, diff --git a/man/sd_journal_enumerate_fields.xml b/man/sd_journal_enumerate_fields.xml new file mode 100644 index 0000000000..fa5884106b --- /dev/null +++ b/man/sd_journal_enumerate_fields.xml @@ -0,0 +1,161 @@ + + + + + + + + + sd_journal_enumerate_fields + systemd + + + + Developer + Lennart + Poettering + lennart@poettering.net + + + + + + sd_journal_enumerate_fields + 3 + + + + sd_journal_enumerate_fields + sd_journal_restart_fields + SD_JOURNAL_FOREACH_FIELD + Read used field names from the journal + + + + + #include <systemd/sd-journal.h> + + + int sd_journal_enumerate_fields + sd_journal *j + const char **field + + + + void sd_journal_restart_fields + sd_journal *j + + + + SD_JOURNAL_FOREACH_FIELD + sd_journal *j + const char *field + + + + + + + Description + + sd_journal_enumerate_fields() may be used to iterate through all field names used in the + opened journal files. On each invocation the next field name is returned. The order of the returned field names is + not defined. It takes two arguments: the journal context object, plus a pointer to a constant string pointer where + the field name is stored in. The returned data is in a read-only memory map and is only valid until the next + invocation of sd_journal_enumerate_fields(). Note that this call is subject to the data field + size threshold as controlled by sd_journal_set_data_threshold(). + + sd_journal_restart_fields() resets the field name enumeration index to the beginning of + the list. The next invocation of sd_journal_enumerate_fields() will return the first field + name again. + + The SD_JOURNAL_FOREACH_FIELD() macro may be used as a handy wrapper around + sd_journal_restart_fields() and sd_journal_enumerate_fields(). + + These functions currently are not influenced by matches set with sd_journal_add_match() + but this might change in a later version of this software. + + To retrieve the possible values a specific field can take use + sd_journal_query_unique3. + + + + Return Value + + sd_journal_enumerate_fields() returns a + positive integer if the next field name has been read, 0 when no + more field names are known, or a negative errno-style error code. + sd_journal_restart_fields() returns + nothing. + + + + Notes + + The sd_journal_enumerate_fields() and sd_journal_restart_fields() + interfaces are available as a shared library, which can be compiled and linked to with the + libsystemd pkg-config1 file. + + + + Examples + + Use the SD_JOURNAL_FOREACH_FIELD macro to iterate through all field names in use in the + current journal. + + #include <stdio.h> +#include <string.h> +#include <systemd/sd-journal.h> + +int main(int argc, char *argv[]) { + sd_journal *j; + const char *field; + int r; + + r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY); + if (r < 0) { + fprintf(stderr, "Failed to open journal: %s\n", strerror(-r)); + return 1; + } + SD_JOURNAL_FOREACH_FIELD(j, field) + printf("%s\n", field); + sd_journal_close(j); + return 0; +} + + + + + See Also + + + systemd1, + systemd.journal-fields7, + sd-journal3, + sd_journal_open3, + sd_journal_query_unique3, + sd_journal_get_data3, + sd_journal_add_match3 + + + + diff --git a/man/sd_journal_query_unique.xml b/man/sd_journal_query_unique.xml index ac0e5f633f..dbff55c105 100644 --- a/man/sd_journal_query_unique.xml +++ b/man/sd_journal_query_unique.xml @@ -128,6 +128,11 @@ Note that these functions currently are not influenced by matches set with sd_journal_add_match() but this might change in a later version of this software. + + To enumerate all field names currently in use (and thus all suitable field parameters for + sd_journal_query_unique()), use the + sd_journal_enumerate_fields3 + call. @@ -167,25 +172,25 @@ #include <systemd/sd-journal.h> int main(int argc, char *argv[]) { - sd_journal *j; - const void *d; - size_t l; - int r; + sd_journal *j; + const void *d; + size_t l; + int r; - r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY); - if (r < 0) { - fprintf(stderr, "Failed to open journal: %s\n", strerror(-r)); - return 1; - } - r = sd_journal_query_unique(j, "_SYSTEMD_UNIT"); - if (r < 0) { - fprintf(stderr, "Failed to query journal: %s\n", strerror(-r)); - return 1; - } - SD_JOURNAL_FOREACH_UNIQUE(j, d, l) - printf("%.*s\n", (int) l, (const char*) d); - sd_journal_close(j); - return 0; + r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY); + if (r < 0) { + fprintf(stderr, "Failed to open journal: %s\n", strerror(-r)); + return 1; + } + r = sd_journal_query_unique(j, "_SYSTEMD_UNIT"); + if (r < 0) { + fprintf(stderr, "Failed to query journal: %s\n", strerror(-r)); + return 1; + } + SD_JOURNAL_FOREACH_UNIQUE(j, d, l) + printf("%.*s\n", (int) l, (const char*) d); + sd_journal_close(j); + return 0; } @@ -198,6 +203,7 @@ int main(int argc, char *argv[]) { systemd.journal-fields7, sd-journal3, sd_journal_open3, + sd_journal_enumerate_fields3, sd_journal_get_data3, sd_journal_add_match3 diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h index fa5ca11636..a55d1bcc47 100644 --- a/src/journal/journal-internal.h +++ b/src/journal/journal-internal.h @@ -103,18 +103,27 @@ struct sd_journal { unsigned current_invalidate_counter, last_invalidate_counter; usec_t last_process_usec; + /* Iterating through unique fields and their data values */ char *unique_field; JournalFile *unique_file; uint64_t unique_offset; + /* Iterating through known fields */ + JournalFile *fields_file; + uint64_t fields_offset; + uint64_t fields_hash_table_index; + char *fields_buffer; + size_t fields_buffer_allocated; + int flags; - bool on_network; - bool no_new_files; - bool unique_file_lost; /* File we were iterating over got - removed, and there were no more - files, so sd_j_enumerate_unique - will return a value equal to 0. */ + bool on_network:1; + bool no_new_files:1; + bool unique_file_lost:1; /* File we were iterating over got + removed, and there were no more + files, so sd_j_enumerate_unique + will return a value equal to 0. */ + bool fields_file_lost:1; bool has_runtime_files:1; bool has_persistent_files:1; diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c index cf359d20ca..38fb43970e 100644 --- a/src/journal/journalctl.c +++ b/src/journal/journalctl.c @@ -138,6 +138,8 @@ static enum { ACTION_SYNC, ACTION_ROTATE, ACTION_VACUUM, + ACTION_LIST_FIELDS, + ACTION_LIST_FIELD_NAMES, } arg_action = ACTION_SHOW; typedef struct BootId { @@ -320,6 +322,7 @@ static void help(void) { "\nCommands:\n" " -h --help Show this help text\n" " --version Show package version\n" + " -N --fields List all field names currently used\n" " -F --field=FIELD List all values that a specified field takes\n" " --disk-usage Show total disk usage of all journal files\n" " --vacuum-size=BYTES Reduce disk usage below specified size\n" @@ -416,6 +419,7 @@ static int parse_argv(int argc, char *argv[]) { { "unit", required_argument, NULL, 'u' }, { "user-unit", required_argument, NULL, ARG_USER_UNIT }, { "field", required_argument, NULL, 'F' }, + { "fields", no_argument, NULL, 'N' }, { "catalog", no_argument, NULL, 'x' }, { "list-catalog", no_argument, NULL, ARG_LIST_CATALOG }, { "dump-catalog", no_argument, NULL, ARG_DUMP_CATALOG }, @@ -437,7 +441,7 @@ static int parse_argv(int argc, char *argv[]) { assert(argc >= 0); assert(argv); - while ((c = getopt_long(argc, argv, "hefo:aln::qmb::kD:p:c:S:U:t:u:F:xrM:", options, NULL)) >= 0) + while ((c = getopt_long(argc, argv, "hefo:aln::qmb::kD:p:c:S:U:t:u:NF:xrM:", options, NULL)) >= 0) switch (c) { @@ -774,9 +778,14 @@ static int parse_argv(int argc, char *argv[]) { break; case 'F': + arg_action = ACTION_LIST_FIELDS; arg_field = optarg; break; + case 'N': + arg_action = ACTION_LIST_FIELD_NAMES; + break; + case 'x': arg_catalog = true; break; @@ -2081,6 +2090,8 @@ int main(int argc, char *argv[]) { case ACTION_DISK_USAGE: case ACTION_LIST_BOOTS: case ACTION_VACUUM: + case ACTION_LIST_FIELDS: + case ACTION_LIST_FIELD_NAMES: /* These ones require access to the journal files, continue below. */ break; @@ -2163,7 +2174,20 @@ int main(int argc, char *argv[]) { goto finish; } + case ACTION_LIST_FIELD_NAMES: { + const char *field; + + SD_JOURNAL_FOREACH_FIELD(j, field) { + printf("%s\n", field); + n_shown ++; + } + + r = 0; + goto finish; + } + case ACTION_SHOW: + case ACTION_LIST_FIELDS: break; default: @@ -2217,10 +2241,12 @@ int main(int argc, char *argv[]) { log_debug("Journal filter: %s", filter); } - if (arg_field) { + if (arg_action == ACTION_LIST_FIELDS) { const void *data; size_t size; + assert(arg_field); + r = sd_journal_set_data_threshold(j, 0); if (r < 0) { log_error_errno(r, "Failed to unset data size threshold: %m"); diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index 74a5e262f8..7a3aaf0cab 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -1338,6 +1338,13 @@ static void remove_file_real(sd_journal *j, JournalFile *f) { j->unique_file_lost = true; } + if (j->fields_file == f) { + j->fields_file = ordered_hashmap_next(j->files, j->fields_file->path); + j->fields_offset = 0; + if (!j->fields_file) + j->fields_file_lost = true; + } + journal_file_close(f); j->current_invalidate_counter ++; @@ -1806,6 +1813,7 @@ _public_ void sd_journal_close(sd_journal *j) { free(j->path); free(j->prefix); free(j->unique_field); + free(j->fields_buffer); free(j); } @@ -2512,24 +2520,20 @@ _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_ * traversed files. */ found = false; ORDERED_HASHMAP_FOREACH(of, j->files, i) { - Object *oo; - uint64_t op; - if (of == j->unique_file) break; - /* Skip this file it didn't have any fields - * indexed */ - if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) && - le64toh(of->header->n_fields) <= 0) + /* Skip this file it didn't have any fields indexed */ + if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) && le64toh(of->header->n_fields) <= 0) continue; - r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op); + r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), NULL, NULL); if (r < 0) return r; - - if (r > 0) + if (r > 0) { found = true; + break; + } } if (found) @@ -2552,6 +2556,154 @@ _public_ void sd_journal_restart_unique(sd_journal *j) { j->unique_file_lost = false; } +_public_ int sd_journal_enumerate_fields(sd_journal *j, const char **field) { + int r; + + assert_return(j, -EINVAL); + assert_return(!journal_pid_changed(j), -ECHILD); + assert_return(field, -EINVAL); + + if (!j->fields_file) { + if (j->fields_file_lost) + return 0; + + j->fields_file = ordered_hashmap_first(j->files); + if (!j->fields_file) + return 0; + + j->fields_hash_table_index = 0; + j->fields_offset = 0; + } + + for (;;) { + JournalFile *f, *of; + Iterator i; + uint64_t m; + Object *o; + size_t sz; + bool found; + + f = j->fields_file; + + if (j->fields_offset == 0) { + bool eof = false; + + /* We are not yet positioned at any field. Let's pick the first one */ + r = journal_file_map_field_hash_table(f); + if (r < 0) + return r; + + m = le64toh(f->header->field_hash_table_size) / sizeof(HashItem); + for (;;) { + if (j->fields_hash_table_index >= m) { + /* Reached the end of the hash table, go to the next file. */ + eof = true; + break; + } + + j->fields_offset = le64toh(f->field_hash_table[j->fields_hash_table_index].head_hash_offset); + + if (j->fields_offset != 0) + break; + + /* Empty hash table bucket, go to next one */ + j->fields_hash_table_index++; + } + + if (eof) { + /* Proceed with next file */ + j->fields_file = ordered_hashmap_next(j->files, f->path); + if (!j->fields_file) { + *field = NULL; + return 0; + } + + j->fields_offset = 0; + j->fields_hash_table_index = 0; + continue; + } + + } else { + /* We are already positioned at a field. If so, let's figure out the next field from it */ + + r = journal_file_move_to_object(f, OBJECT_FIELD, j->fields_offset, &o); + if (r < 0) + return r; + + j->fields_offset = le64toh(o->field.next_hash_offset); + if (j->fields_offset == 0) { + /* Reached the end of the hash table chain */ + j->fields_hash_table_index++; + continue; + } + } + + /* We use OBJECT_UNUSED here, so that the iteator below doesn't remove our mmap window */ + r = journal_file_move_to_object(f, OBJECT_UNUSED, j->fields_offset, &o); + if (r < 0) + return r; + + /* Because we used OBJECT_UNUSED above, we need to do our type check manually */ + if (o->object.type != OBJECT_FIELD) { + log_debug("%s:offset " OFSfmt ": object has type %i, expected %i", f->path, j->fields_offset, o->object.type, OBJECT_FIELD); + return -EBADMSG; + } + + sz = le64toh(o->object.size) - offsetof(Object, field.payload); + + /* Let's see if we already returned this field name before. */ + found = false; + ORDERED_HASHMAP_FOREACH(of, j->files, i) { + if (of == f) + break; + + /* Skip this file it didn't have any fields indexed */ + if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) && le64toh(of->header->n_fields) <= 0) + continue; + + r = journal_file_find_field_object_with_hash(of, o->field.payload, sz, le64toh(o->field.hash), NULL, NULL); + if (r < 0) + return r; + if (r > 0) { + found = true; + break; + } + } + + if (found) + continue; + + /* Check if this is really a valid string containing no NUL byte */ + if (memchr(o->field.payload, 0, sz)) + return -EBADMSG; + + if (sz > j->data_threshold) + sz = j->data_threshold; + + if (!GREEDY_REALLOC(j->fields_buffer, j->fields_buffer_allocated, sz + 1)) + return -ENOMEM; + + memcpy(j->fields_buffer, o->field.payload, sz); + j->fields_buffer[sz] = 0; + + if (!field_is_valid(j->fields_buffer)) + return -EBADMSG; + + *field = j->fields_buffer; + return 1; + } +} + +_public_ void sd_journal_restart_fields(sd_journal *j) { + if (!j) + return; + + j->fields_file = NULL; + j->fields_hash_table_index = 0; + j->fields_offset = 0; + j->fields_file_lost = false; +} + _public_ int sd_journal_reliable_fd(sd_journal *j) { assert_return(j, -EINVAL); assert_return(!journal_pid_changed(j), -ECHILD); diff --git a/src/libsystemd/libsystemd.sym b/src/libsystemd/libsystemd.sym index 043ff13e6f..4ab637b686 100644 --- a/src/libsystemd/libsystemd.sym +++ b/src/libsystemd/libsystemd.sym @@ -481,3 +481,11 @@ global: sd_bus_path_encode_many; sd_listen_fds_with_names; } LIBSYSTEMD_226; + +LIBSYSTEMD_229 { +global: + sd_journal_has_runtime_files; + sd_journal_has_persistent_files; + sd_journal_enumerate_fields; + sd_journal_restart_fields; +} LIBSYSTEMD_227; diff --git a/src/systemd/sd-journal.h b/src/systemd/sd-journal.h index 7f16c69ce5..caf322f062 100644 --- a/src/systemd/sd-journal.h +++ b/src/systemd/sd-journal.h @@ -129,6 +129,9 @@ int sd_journal_query_unique(sd_journal *j, const char *field); int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l); void sd_journal_restart_unique(sd_journal *j); +int sd_journal_enumerate_fields(sd_journal *j, const char **field); +void sd_journal_restart_fields(sd_journal *j); + int sd_journal_get_fd(sd_journal *j); int sd_journal_get_events(sd_journal *j); int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec); @@ -142,22 +145,28 @@ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **text); int sd_journal_has_runtime_files(sd_journal *j); int sd_journal_has_persistent_files(sd_journal *j); -/* the inverse condition avoids ambiguity of danling 'else' after the macro */ +/* The inverse condition avoids ambiguity of dangling 'else' after the macro */ #define SD_JOURNAL_FOREACH(j) \ if (sd_journal_seek_head(j) < 0) { } \ else while (sd_journal_next(j) > 0) -/* the inverse condition avoids ambiguity of danling 'else' after the macro */ +/* The inverse condition avoids ambiguity of dangling 'else' after the macro */ #define SD_JOURNAL_FOREACH_BACKWARDS(j) \ if (sd_journal_seek_tail(j) < 0) { } \ else while (sd_journal_previous(j) > 0) +/* Iterate through the data fields of the current journal entry */ #define SD_JOURNAL_FOREACH_DATA(j, data, l) \ for (sd_journal_restart_data(j); sd_journal_enumerate_data((j), &(data), &(l)) > 0; ) +/* Iterate through the all known values of a specific field */ #define SD_JOURNAL_FOREACH_UNIQUE(j, data, l) \ for (sd_journal_restart_unique(j); sd_journal_enumerate_unique((j), &(data), &(l)) > 0; ) +/* Iterate through all known field names */ +#define SD_JOURNAL_FOREACH_FIELD(j, field) \ + for (sd_journal_restart_fields(j); sd_journal_enumerate_fields((j), &(field)) > 0; ) + _SD_DEFINE_POINTER_CLEANUP_FUNC(sd_journal, sd_journal_close); _SD_END_DECLARATIONS;