diff --git a/TODO b/TODO index f1593a2c02..099ce36f01 100644 --- a/TODO +++ b/TODO @@ -1029,7 +1029,7 @@ Features: - journal: add a setgid "systemd-journal" utility to invoke from libsystemd-journal, which passes fds via STDOUT and does PK access - journactl: support negative filtering, i.e. FOOBAR!="waldo", and !FOOBAR for events without FOOBAR. - - journal: store timestamp of journal_file_set_offline() int he header, + - journal: store timestamp of journal_file_set_offline() in the header, so it is possible to display when the file was last synced. - journal-send.c, log.c: when the log socket is clogged, and we drop, count this and write a message about this when it gets unclogged again. - journal: find a way to allow dropping history early, based on priority, other rules @@ -1071,6 +1071,7 @@ Features: them via machined, and also watch containers coming and going. Benefit: nspawn --ephemeral would start working nicely with the journal. - assign MESSAGE_ID to log messages about failed services + - check if loop in decompress_blob_xz() is necessary * add a test if all entries in the catalog are properly formatted. (Adding dashes in a catalog entry currently results in the catalog entry diff --git a/man/sd_journal_get_data.xml b/man/sd_journal_get_data.xml index 0a0030e300..209f5deaa1 100644 --- a/man/sd_journal_get_data.xml +++ b/man/sd_journal_get_data.xml @@ -18,6 +18,7 @@ sd_journal_get_data sd_journal_enumerate_data + sd_journal_enumerate_available_data sd_journal_restart_data SD_JOURNAL_FOREACH_DATA sd_journal_set_data_threshold @@ -44,6 +45,13 @@ size_t *length + + int sd_journal_enumerate_available_data + sd_journal *j + const void **data + size_t *length + + void sd_journal_restart_data sd_journal *j @@ -73,24 +81,18 @@ Description - sd_journal_get_data() gets the data - object associated with a specific field from the current journal - entry. It takes four arguments: the journal context object, a - string with the field name to request, plus a pair of pointers to - pointer/size variables where the data object and its size shall be - stored in. The field name should be an entry field name. - Well-known field names are listed in - systemd.journal-fields7. - The returned data is in a read-only memory map and is only valid - until the next invocation of - sd_journal_get_data() or - sd_journal_enumerate_data(), or the read - pointer is altered. Note that the data returned will be prefixed - with the field name and '='. Also note that, by default, data fields - larger than 64K might get truncated to 64K. This threshold may be - changed and turned off with - sd_journal_set_data_threshold() (see - below). + sd_journal_get_data() gets the data object associated with a specific field + from the current journal entry. It takes four arguments: the journal context object, a string with the + field name to request, plus a pair of pointers to pointer/size variables where the data object and its + size shall be stored in. The field name should be an entry field name. Well-known field names are listed in + systemd.journal-fields7, + but any field can be specified. The returned data is in a read-only memory map and is only valid until + the next invocation of sd_journal_get_data(), + sd_journal_enumerate_data(), + sd_journal_enumerate_available_data(), or when the read pointer is altered. Note + that the data returned will be prefixed with the field name and =. Also note that, by + default, data fields larger than 64K might get truncated to 64K. This threshold may be changed and turned + off with sd_journal_set_data_threshold() (see below). sd_journal_enumerate_data() may be used to iterate through all fields of the current entry. On each @@ -99,15 +101,18 @@ format as with sd_journal_get_data() and also follows the same life-time semantics. + sd_journal_enumerate_available_data() is similar to + sd_journal_enumerate_data(), but silently skips any fields which may be valid, but + are too large or not supported by current implementation. + sd_journal_restart_data() resets the data enumeration index to the beginning of the entry. The next invocation of sd_journal_enumerate_data() will return the first field of the entry again. - Note that the SD_JOURNAL_FOREACH_DATA() - macro may be used as a handy wrapper around - sd_journal_restart_data() and - sd_journal_enumerate_data(). + Note that the SD_JOURNAL_FOREACH_DATA() macro may be used as a handy wrapper + around sd_journal_restart_data() and + sd_journal_enumerate_available_data(). Note that these functions will not work before sd_journal_next3 @@ -139,18 +144,88 @@ Return Value - sd_journal_get_data() returns 0 on - success or a negative errno-style error code. If the current entry - does not include the specified field, -ENOENT is returned. If - sd_journal_next3 - has not been called at least once, -EADDRNOTAVAIL is returned. - sd_journal_enumerate_data() returns a - positive integer if the next field has been read, 0 when no more - fields are known, or a negative errno-style error code. - sd_journal_restart_data() returns nothing. - sd_journal_set_data_threshold() and - sd_journal_get_threshold() return 0 on - success or a negative errno-style error code. + sd_journal_get_data() returns 0 on success or a negative errno-style error + code. sd_journal_enumerate_data() and + sd_journal_enumerate_available_data() return a positive integer if the next field + has been read, 0 when no more fields remain, or a negative errno-style error code. + sd_journal_restart_data() doesn't return anything. + sd_journal_set_data_threshold() and sd_journal_get_threshold() + return 0 on success or a negative errno-style error code. + + + Errors + + Returned errors may indicate the following problems: + + + + -EINVAL + + One of the required parameters is NULL or invalid. + + + + + -ECHILD + + The journal object was created in a different process. + + + + -EADDRNOTAVAIL + + The read pointer is not positioned at a valid entry; + sd_journal_next3 + or a related call has not been called at least once. + + + + -ENOENT + + The current entry does not include the specified field. + + + + + -ENOMEM + + Memory allocation failed. + + + + -ENOBUFS + + A compressed entry is too large. + + + + -E2BIG + + The data field is too large for this computer architecture (e.g. above 4 GB on a + 32-bit architecture). + + + + -EPROTONOSUPPORT + + The journal is compressed with an unsupported method or the journal uses an + unsupported feature. + + + + -EBADMSG + + The journal is corrupted (possibly just the entry being iterated over). + + + + + -EIO + + An I/O error was reported by the kernel. + + + diff --git a/man/sd_journal_query_unique.xml b/man/sd_journal_query_unique.xml index 1bf83968dd..88beaa6460 100644 --- a/man/sd_journal_query_unique.xml +++ b/man/sd_journal_query_unique.xml @@ -18,6 +18,7 @@ sd_journal_query_unique sd_journal_enumerate_unique + sd_journal_enumerate_available_unique sd_journal_restart_unique SD_JOURNAL_FOREACH_UNIQUE Read unique data fields from the journal @@ -33,6 +34,13 @@ const char *field + + int sd_journal_enumerate_available_unique + sd_journal *j + const void **data + size_t *length + + int sd_journal_enumerate_unique sd_journal *j @@ -58,33 +66,31 @@ Description - sd_journal_query_unique() queries the - journal for all unique values the specified field can take. It - takes two arguments: the journal to query and the field name to - look for. Well-known field names are listed on - systemd.journal-fields7. - Field names must be specified without a trailing '='. After this - function has been executed successfully the field values may be - queried using sd_journal_enumerate_unique(). - Invoking this call a second time will change the field name being - queried and reset the enumeration index to the first field value - that matches. + sd_journal_query_unique() queries the journal for all unique values the + specified field can take. It takes two arguments: the journal to query and the field name to look + for. Well-known field names are listed on + systemd.journal-fields7, + but any field can be specified. Field names must be specified without a trailing + =. After this function has been executed successfully the field values may be queried + using sd_journal_enumerate_unique() and + sd_journal_enumerate_available_unique(). Invoking one of those calls will change the + field name being queried and reset the enumeration index to the first field value that matches. - sd_journal_enumerate_unique() may be - used to iterate through all data fields which match the previously - selected field name as set with - sd_journal_query_unique(). On each invocation - the next field data matching the field name is returned. The order - of the returned data fields is not defined. It takes three - arguments: the journal context object, plus a pair of pointers to - pointer/size variables where the data object and its size shall be - stored in. The returned data is in a read-only memory map and is - only valid until the next invocation of - sd_journal_enumerate_unique(). Note that the - data returned will be prefixed with the field name and '='. Note - that this call is subject to the data field size threshold as - controlled by - sd_journal_set_data_threshold(). + sd_journal_enumerate_unique() may be used to iterate through all data fields + which match the previously selected field name as set with + sd_journal_query_unique(). On each invocation the next field data matching the field + name is returned. The order of the returned data fields is not defined. It takes three arguments: the + journal object, plus a pair of pointers to pointer/size variables where the data object and its size + shall be stored. The returned data is in a read-only memory map and is only valid until the next + invocation of sd_journal_enumerate_unique(). Note that the data returned will be + prefixed with the field name and =. Note that this call is subject to the data field + size threshold as controlled by sd_journal_set_data_threshold() and only the initial + part of the field up to the threshold is returned. An error is returned for fields which cannot be + retrieved. See the error list below for details. + + sd_journal_enumerate_available_unique() is similar to + sd_journal_enumerate_unique(), but silently skips any fields which may be valid, but + are too large or not supported by current implementation. sd_journal_restart_unique() resets the data enumeration index to the beginning of the list. The next @@ -92,11 +98,9 @@ will return the first field data matching the field name again. - Note that the - SD_JOURNAL_FOREACH_UNIQUE() macro may be used - as a handy wrapper around - sd_journal_restart_unique() and - sd_journal_enumerate_unique(). + Note that the SD_JOURNAL_FOREACH_UNIQUE() macro may be used as a handy wrapper + around sd_journal_restart_unique() and + sd_journal_enumerate_available_unique(). Note that these functions currently are not influenced by matches set with sd_journal_add_match() but @@ -111,13 +115,29 @@ Return Value - sd_journal_query_unique() returns 0 on - success or a negative errno-style error code. - sd_journal_enumerate_unique() returns a - positive integer if the next field data has been read, 0 when no - more fields are known, or a negative errno-style error code. - sd_journal_restart_unique() returns - nothing. + sd_journal_query_unique() returns 0 on success or a negative errno-style error + code. sd_journal_enumerate_unique() and and + sd_journal_query_available_unique() return a positive integer if the next field data + has been read, 0 when no more fields remain, or a negative errno-style error code. + sd_journal_restart_unique() doesn't return anything. + + + Errors + + Returned errors may indicate the following problems: + + + + + + + + + + + + + @@ -131,10 +151,9 @@ Examples - Use the SD_JOURNAL_FOREACH_UNIQUE macro - to iterate through all values a field of the journal can take. The - following example lists all unit names referenced in the - journal: + Use the SD_JOURNAL_FOREACH_UNIQUE macro to iterate through all values a field + of the journal can take (and which can be accessed on the given architecture and are not compressed with + an unsupported mechanism). The following example lists all unit names referenced in the journal: diff --git a/src/home/homectl-pkcs11.c b/src/home/homectl-pkcs11.c index 830aafaab1..f4253ed7bf 100644 --- a/src/home/homectl-pkcs11.c +++ b/src/home/homectl-pkcs11.c @@ -16,12 +16,12 @@ struct pkcs11_callback_data { X509 *cert; }; +#if HAVE_P11KIT static void pkcs11_callback_data_release(struct pkcs11_callback_data *data) { erase_and_free(data->pin_used); X509_free(data->cert); } -#if HAVE_P11KIT static int pkcs11_callback( CK_FUNCTION_LIST *m, CK_SESSION_HANDLE session, diff --git a/src/journal/compress.c b/src/journal/compress.c index e6ce64fc65..a59c2b7a88 100644 --- a/src/journal/compress.c +++ b/src/journal/compress.c @@ -259,10 +259,10 @@ int decompress_blob_lz4(const void *src, uint64_t src_size, int decompress_blob_zstd( const void *src, uint64_t src_size, - void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max) { + void **dst, size_t *dst_alloc_size, size_t *dst_size, size_t dst_max) { #if HAVE_ZSTD - size_t space; + uint64_t size; assert(src); assert(src_size > 0); @@ -271,38 +271,40 @@ int decompress_blob_zstd( assert(dst_size); assert(*dst_alloc_size == 0 || *dst); - if (src_size > SIZE_MAX/2) /* Overflow? */ - return -ENOBUFS; - space = src_size * 2; - if (dst_max > 0 && space > dst_max) - space = dst_max; + size = ZSTD_getFrameContentSize(src, src_size); + if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN)) + return -EBADMSG; - if (!greedy_realloc(dst, dst_alloc_size, space, 1)) + if (dst_max > 0 && size > dst_max) + size = dst_max; + if (size > SIZE_MAX) + return -E2BIG; + + if (!(greedy_realloc(dst, dst_alloc_size, MAX(ZSTD_DStreamOutSize(), size), 1))) return -ENOMEM; - for (;;) { - size_t k; + _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx(); + if (!dctx) + return -ENOMEM; - k = ZSTD_decompress(*dst, *dst_alloc_size, src, src_size); - if (!ZSTD_isError(k)) { - *dst_size = k; - return 0; - } - if (ZSTD_getErrorCode(k) != ZSTD_error_dstSize_tooSmall) - return zstd_ret_to_errno(k); + ZSTD_inBuffer input = { + .src = src, + .size = src_size, + }; + ZSTD_outBuffer output = { + .dst = *dst, + .size = *dst_alloc_size, + }; - if (dst_max > 0 && space >= dst_max) /* Already at max? */ - return -ENOBUFS; - if (space > SIZE_MAX / 2) /* Overflow? */ - return -ENOBUFS; - - space *= 2; - if (dst_max > 0 && space > dst_max) - space = dst_max; - - if (!greedy_realloc(dst, dst_alloc_size, space, 1)) - return -ENOMEM; + size_t k = ZSTD_decompressStream(dctx, &output, &input); + if (ZSTD_isError(k)) { + log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k)); + return zstd_ret_to_errno(k); } + assert(output.pos >= size); + + *dst_size = size; + return 0; #else return -EPROTONOSUPPORT; #endif @@ -326,7 +328,7 @@ int decompress_blob( src, src_size, dst, dst_alloc_size, dst_size, dst_max); else - return -EBADMSG; + return -EPROTONOSUPPORT; } int decompress_startswith_xz(const void *src, uint64_t src_size, @@ -456,9 +458,6 @@ int decompress_startswith_zstd( const void *prefix, size_t prefix_len, uint8_t extra) { #if HAVE_ZSTD - _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL; - size_t k; - assert(src); assert(src_size > 0); assert(buffer); @@ -466,7 +465,14 @@ int decompress_startswith_zstd( assert(prefix); assert(*buffer_size == 0 || *buffer); - dctx = ZSTD_createDCtx(); + uint64_t size = ZSTD_getFrameContentSize(src, src_size); + if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN)) + return -EBADMSG; + + if (size < prefix_len + 1) + return 0; /* Decompressed text too short to match the prefix and extra */ + + _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = ZSTD_createDCtx(); if (!dctx) return -ENOMEM; @@ -481,30 +487,17 @@ int decompress_startswith_zstd( .dst = *buffer, .size = *buffer_size, }; + size_t k; - for (;;) { - k = ZSTD_decompressStream(dctx, &output, &input); - if (ZSTD_isError(k)) { - log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k)); - return zstd_ret_to_errno(k); - } - - if (output.pos >= prefix_len + 1) - return memcmp(*buffer, prefix, prefix_len) == 0 && - ((const uint8_t*) *buffer)[prefix_len] == extra; - - if (input.pos >= input.size) - return 0; - - if (*buffer_size > SIZE_MAX/2) - return -ENOBUFS; - - if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1))) - return -ENOMEM; - - output.dst = *buffer; - output.size = *buffer_size; + k = ZSTD_decompressStream(dctx, &output, &input); + if (ZSTD_isError(k)) { + log_debug("ZSTD decoder failed: %s", ZSTD_getErrorName(k)); + return zstd_ret_to_errno(k); } + assert(output.pos >= prefix_len + 1); + + return memcmp(*buffer, prefix, prefix_len) == 0 && + ((const uint8_t*) *buffer)[prefix_len] == extra; #else return -EPROTONOSUPPORT; #endif diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h index a649acf634..d87b0a11e5 100644 --- a/src/journal/journal-internal.h +++ b/src/journal/journal-internal.h @@ -127,3 +127,12 @@ void journal_print_header(sd_journal *j); #define JOURNAL_FOREACH_DATA_RETVAL(j, data, l, retval) \ for (sd_journal_restart_data(j); ((retval) = sd_journal_enumerate_data((j), &(data), &(l))) > 0; ) + +/* All errors that we might encounter while extracting a field that are not real errors, + * but only mean that the field is too large or we don't support the compression. */ +static inline bool JOURNAL_ERRNO_IS_UNAVAILABLE_FIELD(int r) { + return IN_SET(abs(r), + ENOBUFS, /* Field or decompressed field too large */ + E2BIG, /* Field too large for pointer width */ + EPROTONOSUPPORT); /* Unsupported compression */ +} diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index 853dd0c28b..6fb0abb419 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -2462,6 +2462,19 @@ _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t return 1; } +_public_ int sd_journal_enumerate_available_data(sd_journal *j, const void **data, size_t *size) { + for (;;) { + int r; + + r = sd_journal_enumerate_data(j, data, size); + if (r >= 0) + return r; + if (!JOURNAL_ERRNO_IS_UNAVAILABLE_FIELD(r)) + return r; + j->current_field++; /* Try with the next field */ + } +} + _public_ void sd_journal_restart_data(sd_journal *j) { if (!j) return; @@ -3002,6 +3015,20 @@ _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_ } } +_public_ int sd_journal_enumerate_available_unique(sd_journal *j, const void **data, size_t *size) { + for (;;) { + int r; + + r = sd_journal_enumerate_unique(j, data, size); + if (r >= 0) + return r; + if (!JOURNAL_ERRNO_IS_UNAVAILABLE_FIELD(r)) + return r; + /* Try with the next field. sd_journal_enumerate_unique() modifies state, so on the next try + * we will access the next field. */ + } +} + _public_ void sd_journal_restart_unique(sd_journal *j) { if (!j) return; diff --git a/src/journal/test-compress.c b/src/journal/test-compress.c index 0990f7604d..f50fb0acea 100644 --- a/src/journal/test-compress.c +++ b/src/journal/test-compress.c @@ -232,6 +232,8 @@ static void test_lz4_decompress_partial(void) { int r; _cleanup_free_ char *huge = NULL; + log_debug("/* %s */", __func__); + assert_se(huge = malloc(HUGE_SIZE)); memcpy(huge, "HUGE=", STRLEN("HUGE=")); memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1); diff --git a/src/libsystemd/libsystemd.sym b/src/libsystemd/libsystemd.sym index 6f00985f5d..1e654b49ea 100644 --- a/src/libsystemd/libsystemd.sym +++ b/src/libsystemd/libsystemd.sym @@ -717,4 +717,7 @@ global: sd_path_lookup_strv; sd_notify_barrier; + + sd_journal_enumerate_available_data; + sd_journal_enumerate_available_unique; } LIBSYSTEMD_245; diff --git a/src/systemd/sd-journal.h b/src/systemd/sd-journal.h index a232992f03..d220f21aa2 100644 --- a/src/systemd/sd-journal.h +++ b/src/systemd/sd-journal.h @@ -105,6 +105,7 @@ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz); int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *l); int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *l); +int sd_journal_enumerate_available_data(sd_journal *j, const void **data, size_t *l); void sd_journal_restart_data(sd_journal *j); int sd_journal_add_match(sd_journal *j, const void *data, size_t size); @@ -128,6 +129,7 @@ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes); int sd_journal_query_unique(sd_journal *j, const char *field); int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l); +int sd_journal_enumerate_available_unique(sd_journal *j, const void **data, size_t *l); void sd_journal_restart_unique(sd_journal *j); int sd_journal_enumerate_fields(sd_journal *j, const char **field); @@ -156,13 +158,13 @@ int sd_journal_has_persistent_files(sd_journal *j); if (sd_journal_seek_tail(j) < 0) { } \ else while (sd_journal_previous(j) > 0) -/* Iterate through the data fields of the current journal entry */ +/* Iterate through all available data fields of the current journal entry */ #define SD_JOURNAL_FOREACH_DATA(j, data, l) \ - for (sd_journal_restart_data(j); sd_journal_enumerate_data((j), &(data), &(l)) > 0; ) + for (sd_journal_restart_data(j); sd_journal_enumerate_available_data((j), &(data), &(l)) > 0; ) -/* Iterate through the all known values of a specific field */ +/* Iterate through all available values of a specific field */ #define SD_JOURNAL_FOREACH_UNIQUE(j, data, l) \ - for (sd_journal_restart_unique(j); sd_journal_enumerate_unique((j), &(data), &(l)) > 0; ) + for (sd_journal_restart_unique(j); sd_journal_enumerate_available_unique((j), &(data), &(l)) > 0; ) /* Iterate through all known field names */ #define SD_JOURNAL_FOREACH_FIELD(j, field) \