shaba/systemd-stable
1
1
Fork 0
mirror of https://github.com/systemd/systemd-stable.git synced 2025-02-08 05:57:26 +03:00
Code

basic: move compress.[ch] → src/basic/

Browse Source 
The compression helpers are used both in journal code and in coredump
code, and there's a good chance we'll use them later for other stuff.

Let's hence move them into src/basic/, to make them a proper internal
API we can use from everywhere where that's desirable. (pstore might be
a candidate, for example)

No real code changes, just some moving around, build system
rearrangements, and stripping of journal-def.h inclusion.
This commit is contained in:
Lennart Poettering 2022-04-20 15:35:28 +02:00
parent 6ed7b6977f
commit acc50c92eb
16 changed files with 166 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
meson.build
View File

@ -1472,7 +1472,7 @@ elif compression == 'xz' and not have_xz
endif endif
conf.set('DEFAULT_COMPRESSION', conf.set('DEFAULT_COMPRESSION',
compression == 'none' ? 0 : compression == 'none' ? 0 :
'OBJECT_COMPRESSED_@0@'.format(compression.to_upper())) 'COMPRESSION_@0@'.format(compression.to_upper()))
want_xkbcommon = get_option('xkbcommon') want_xkbcommon = get_option('xkbcommon')
if want_xkbcommon != 'false' and not skip_deps if want_xkbcommon != 'false' and not skip_deps
@ -1901,13 +1901,11 @@ libsystemd = shared_library(
link_args : ['-shared', link_args : ['-shared',
'-Wl,--version-script=' + libsystemd_sym_path], '-Wl,--version-script=' + libsystemd_sym_path],
link_with : [libbasic, link_with : [libbasic,
libbasic_gcrypt], libbasic_gcrypt,
libbasic_compress],
link_whole : [libsystemd_static], link_whole : [libsystemd_static],
dependencies : [threads, dependencies : [threads,
librt, librt],
libxz,
libzstd,
liblz4],
link_depends : libsystemd_sym, link_depends : libsystemd_sym,
install : true, install : true,
install_dir : rootlibdir) install_dir : rootlibdir)
@ -1917,6 +1915,7 @@ install_libsystemd_static = static_library(
libsystemd_sources, libsystemd_sources,
basic_sources, basic_sources,
basic_gcrypt_sources, basic_gcrypt_sources,
basic_compress_sources,
fundamental_sources, fundamental_sources,
disable_mempool_c, disable_mempool_c,
include_directories : libsystemd_includes, include_directories : libsystemd_includes,
@ -3031,7 +3030,8 @@ if conf.get('ENABLE_COREDUMP') == 1
'systemd-coredump', 'systemd-coredump',
systemd_coredump_sources, systemd_coredump_sources,
include_directories : includes, include_directories : includes,
link_with : [libshared], link_with : [libshared,
libbasic_compress],
dependencies : [threads, dependencies : [threads,
libacl, libacl,
libxz, libxz,
@ -3045,7 +3045,8 @@ if conf.get('ENABLE_COREDUMP') == 1
'coredumpctl', 'coredumpctl',
coredumpctl_sources, coredumpctl_sources,
include_directories : includes, include_directories : includes,
link_with : [libshared], link_with : [libshared,
libbasic_compress],
dependencies : [threads, dependencies : [threads,
libxz, libxz,
liblz4, liblz4,

39
src/libsystemd/sd-journal/compress.c → src/basic/compress.c
View File

@ -27,7 +27,6 @@
#include "fd-util.h" #include "fd-util.h"
#include "fileio.h" #include "fileio.h"
#include "io-util.h" #include "io-util.h"
#include "journal-def.h"
#include "macro.h" #include "macro.h"
#include "sparse-endian.h" #include "sparse-endian.h"
#include "string-table.h" #include "string-table.h"
@ -58,15 +57,15 @@ static int zstd_ret_to_errno(size_t ret) {
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t)) #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = { static const char* const compression_table[_COMPRESSION_MAX] = {
[OBJECT_COMPRESSED_XZ] = "XZ", [COMPRESSION_XZ] = "XZ",
[OBJECT_COMPRESSED_LZ4] = "LZ4", [COMPRESSION_LZ4] = "LZ4",
[OBJECT_COMPRESSED_ZSTD] = "ZSTD", [COMPRESSION_ZSTD] = "ZSTD",
/* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since /* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since
* the array key is actually a bitmask, not a plain enum */ * the array key is actually a bitmask, not a plain enum */
}; };
DEFINE_STRING_TABLE_LOOKUP(object_compressed, int); DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
int compress_blob_xz(const void *src, uint64_t src_size, int compress_blob_xz(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size) { void *dst, size_t dst_alloc_size, size_t *dst_size) {
@ -100,7 +99,7 @@ int compress_blob_xz(const void *src, uint64_t src_size,
return -ENOBUFS; return -ENOBUFS;
*dst_size = out_pos; *dst_size = out_pos;
return OBJECT_COMPRESSED_XZ; return COMPRESSION_XZ;
#else #else
return -EPROTONOSUPPORT; return -EPROTONOSUPPORT;
#endif #endif
@ -130,7 +129,7 @@ int compress_blob_lz4(const void *src, uint64_t src_size,
unaligned_write_le64(dst, src_size); unaligned_write_le64(dst, src_size);
*dst_size = r + 8; *dst_size = r + 8;
return OBJECT_COMPRESSED_LZ4; return COMPRESSION_LZ4;
#else #else
return -EPROTONOSUPPORT; return -EPROTONOSUPPORT;
#endif #endif
@ -153,7 +152,7 @@ int compress_blob_zstd(
return zstd_ret_to_errno(k); return zstd_ret_to_errno(k);
*dst_size = k; *dst_size = k;
return OBJECT_COMPRESSED_ZSTD; return COMPRESSION_ZSTD;
#else #else
return -EPROTONOSUPPORT; return -EPROTONOSUPPORT;
#endif #endif
@ -313,22 +312,22 @@ int decompress_blob_zstd(
} }
int decompress_blob( int decompress_blob(
int compression, Compression compression,
const void *src, const void *src,
uint64_t src_size, uint64_t src_size,
void **dst, void **dst,
size_t* dst_size, size_t* dst_size,
size_t dst_max) { size_t dst_max) {
if (compression == OBJECT_COMPRESSED_XZ) if (compression == COMPRESSION_XZ)
return decompress_blob_xz( return decompress_blob_xz(
src, src_size, src, src_size,
dst, dst_size, dst_max); dst, dst_size, dst_max);
else if (compression == OBJECT_COMPRESSED_LZ4) else if (compression == COMPRESSION_LZ4)
return decompress_blob_lz4( return decompress_blob_lz4(
src, src_size, src, src_size,
dst, dst_size, dst_max); dst, dst_size, dst_max);
else if (compression == OBJECT_COMPRESSED_ZSTD) else if (compression == COMPRESSION_ZSTD)
return decompress_blob_zstd( return decompress_blob_zstd(
src, src_size, src, src_size,
dst, dst_size, dst_max); dst, dst_size, dst_max);
@ -520,7 +519,7 @@ int decompress_startswith_zstd(
} }
int decompress_startswith( int decompress_startswith(
int compression, Compression compression,
const void *src, const void *src,
uint64_t src_size, uint64_t src_size,
void **buffer, void **buffer,
@ -528,20 +527,20 @@ int decompress_startswith(
size_t prefix_len, size_t prefix_len,
uint8_t extra) { uint8_t extra) {
if (compression == OBJECT_COMPRESSED_XZ) if (compression == COMPRESSION_XZ)
return decompress_startswith_xz( return decompress_startswith_xz(
src, src_size, src, src_size,
buffer, buffer,
prefix, prefix_len, prefix, prefix_len,
extra); extra);
else if (compression == OBJECT_COMPRESSED_LZ4) else if (compression == COMPRESSION_LZ4)
return decompress_startswith_lz4( return decompress_startswith_lz4(
src, src_size, src, src_size,
buffer, buffer,
prefix, prefix_len, prefix, prefix_len,
extra); extra);
else if (compression == OBJECT_COMPRESSED_ZSTD) else if (compression == COMPRESSION_ZSTD)
return decompress_startswith_zstd( return decompress_startswith_zstd(
src, src_size, src, src_size,
buffer, buffer,
@ -619,7 +618,7 @@ int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncom
s.total_in, s.total_out, s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100); (double) s.total_out / s.total_in * 100);
return OBJECT_COMPRESSED_XZ; return COMPRESSION_XZ;
} }
} }
} }
@ -710,7 +709,7 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
total_in, total_out, total_in, total_out,
(double) total_out / total_in * 100); (double) total_out / total_in * 100);
return OBJECT_COMPRESSED_LZ4; return COMPRESSION_LZ4;
#else #else
return -EPROTONOSUPPORT; return -EPROTONOSUPPORT;
#endif #endif
@ -954,7 +953,7 @@ int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unc
log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes)", log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes)",
in_bytes, max_bytes - left); in_bytes, max_bytes - left);
return OBJECT_COMPRESSED_ZSTD; return COMPRESSION_ZSTD;
#else #else
return -EPROTONOSUPPORT; return -EPROTONOSUPPORT;
#endif #endif

52
src/libsystemd/sd-journal/compress.h → src/basic/compress.h
View File

@ -3,10 +3,19 @@
#include <unistd.h> #include <unistd.h>
#include "journal-def.h" typedef enum Compression {
/* These are defined the same way as the relevant object types in journal-def.h,
* i.e. OBJECT_COMPRESSED_XZ, … */
COMPRESSION_NONE = 0,
COMPRESSION_XZ = 1,
COMPRESSION_LZ4 = 2,
COMPRESSION_ZSTD = 4,
_COMPRESSION_MAX,
_COMPRESSION_INVALID = -EINVAL,
} Compression;
const char* object_compressed_to_string(int compression); const char* compression_to_string(Compression compression);
int object_compressed_from_string(const char *compression); Compression compression_from_string(const char *compression);
int compress_blob_xz(const void *src, uint64_t src_size, int compress_blob_xz(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size); void *dst, size_t dst_alloc_size, size_t *dst_size);
@ -21,7 +30,7 @@ int decompress_blob_lz4(const void *src, uint64_t src_size,
void **dst, size_t* dst_size, size_t dst_max); void **dst, size_t* dst_size, size_t dst_max);
int decompress_blob_zstd(const void *src, uint64_t src_size, int decompress_blob_zstd(const void *src, uint64_t src_size,
void **dst, size_t* dst_size, size_t dst_max); void **dst, size_t* dst_size, size_t dst_max);
int decompress_blob(int compression, int decompress_blob(Compression compression,
const void *src, uint64_t src_size, const void *src, uint64_t src_size,
void **dst, size_t* dst_size, size_t dst_max); void **dst, size_t* dst_size, size_t dst_max);
@ -37,7 +46,7 @@ int decompress_startswith_zstd(const void *src, uint64_t src_size,
void **buffer, void **buffer,
const void *prefix, size_t prefix_len, const void *prefix, size_t prefix_len,
uint8_t extra); uint8_t extra);
int decompress_startswith(int compression, int decompress_startswith(Compression compression,
const void *src, uint64_t src_size, const void *src, uint64_t src_size,
void **buffer, void **buffer,
const void *prefix, size_t prefix_len, const void *prefix, size_t prefix_len,
@ -51,27 +60,36 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_size);
int decompress_stream_lz4(int fdf, int fdt, uint64_t max_size); int decompress_stream_lz4(int fdf, int fdt, uint64_t max_size);
int decompress_stream_zstd(int fdf, int fdt, uint64_t max_size); int decompress_stream_zstd(int fdf, int fdt, uint64_t max_size);
static inline int compress_blob(const void *src, uint64_t src_size, static inline int compress_blob_explicit(
void *dst, size_t dst_alloc_size, size_t *dst_size) { Compression compression,
switch (DEFAULT_COMPRESSION) { const void *src, uint64_t src_size,
case OBJECT_COMPRESSED_ZSTD: void *dst, size_t dst_alloc_size, size_t *dst_size) {
switch (compression) {
case COMPRESSION_ZSTD:
return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size); return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size);
case OBJECT_COMPRESSED_LZ4: case COMPRESSION_LZ4:
return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size); return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size);
case OBJECT_COMPRESSED_XZ: case COMPRESSION_XZ:
return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size); return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size);
default: default:
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
} }
#define compress_blob(src, src_size, dst, dst_alloc_size, dst_size) \
compress_blob_explicit( \
DEFAULT_COMPRESSION, \
src, src_size, \
dst, dst_alloc_size, dst_size)
static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
switch (DEFAULT_COMPRESSION) { switch (DEFAULT_COMPRESSION) {
case OBJECT_COMPRESSED_ZSTD: case COMPRESSION_ZSTD:
return compress_stream_zstd(fdf, fdt, max_bytes, ret_uncompressed_size); return compress_stream_zstd(fdf, fdt, max_bytes, ret_uncompressed_size);
case OBJECT_COMPRESSED_LZ4: case COMPRESSION_LZ4:
return compress_stream_lz4(fdf, fdt, max_bytes, ret_uncompressed_size); return compress_stream_lz4(fdf, fdt, max_bytes, ret_uncompressed_size);
case OBJECT_COMPRESSED_XZ: case COMPRESSION_XZ:
return compress_stream_xz(fdf, fdt, max_bytes, ret_uncompressed_size); return compress_stream_xz(fdf, fdt, max_bytes, ret_uncompressed_size);
default: default:
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -80,11 +98,11 @@ static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t
static inline const char* default_compression_extension(void) { static inline const char* default_compression_extension(void) {
switch (DEFAULT_COMPRESSION) { switch (DEFAULT_COMPRESSION) {
case OBJECT_COMPRESSED_ZSTD: case COMPRESSION_ZSTD:
return ".zst"; return ".zst";
case OBJECT_COMPRESSED_LZ4: case COMPRESSION_LZ4:
return ".lz4"; return ".lz4";
case OBJECT_COMPRESSED_XZ: case COMPRESSION_XZ:
return ".xz"; return ".xz";
default: default:
return ""; return "";

18
src/basic/meson.build
View File

@ -446,3 +446,21 @@ libbasic_gcrypt = static_library(
dependencies : [libgcrypt], dependencies : [libgcrypt],
c_args : ['-fvisibility=default'], c_args : ['-fvisibility=default'],
build_by_default : false) build_by_default : false)
############################################################
basic_compress_sources = files(
'compress.c',
'compress.h')
# A convenience library that is separate from libbasic to avoid unnecessary
# linking to the compression libraries.
libbasic_compress = static_library(
'basic-compress',
basic_compress_sources,
include_directories : basic_includes,
dependencies : [libxz,
libzstd,
liblz4],
c_args : ['-fvisibility=default'],
build_by_default : false)

21
src/fuzz/fuzz-compress.c
View File

@ -6,21 +6,10 @@
#include "compress.h" #include "compress.h"
#include "fuzz.h" #include "fuzz.h"
static int compress(int alg,
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size) {
if (alg == OBJECT_COMPRESSED_LZ4)
return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size);
if (alg == OBJECT_COMPRESSED_XZ)
return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size);
return -EOPNOTSUPP;
}
typedef struct header { typedef struct header {
uint32_t alg:2; /* We have only two compression algorithms so far, but we might add uint32_t alg:2; /* We have only three compression algorithms so far, but we might add more in the
* more in the future. Let's make this a bit wider so our fuzzer * future. Let's make this a bit wider so our fuzzer cases remain stable in the
* cases remain stable in the future. */ * future. */
uint32_t sw_len; uint32_t sw_len;
uint32_t sw_alloc; uint32_t sw_alloc;
uint32_t reserved[3]; /* Extra space to keep fuzz cases stable in case we need to uint32_t reserved[3]; /* Extra space to keep fuzz cases stable in case we need to
@ -46,7 +35,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
log_set_max_level(LOG_CRIT); log_set_max_level(LOG_CRIT);
log_info("Using compression %s, data size=%zu", log_info("Using compression %s, data size=%zu",
object_compressed_to_string(alg) ?: "(none)", compression_to_string(alg) ?: "(none)",
data_len); data_len);
buf = malloc(MAX(size, 128u)); /* Make the buffer a bit larger for very small data */ buf = malloc(MAX(size, 128u)); /* Make the buffer a bit larger for very small data */
@ -56,7 +45,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
} }
size_t csize; size_t csize;
r = compress(alg, h->data, data_len, buf, size, &csize); r = compress_blob_explicit(alg, h->data, data_len, buf, size, &csize);
if (r < 0) { if (r < 0) {
log_error_errno(r, "Compression failed: %m"); log_error_errno(r, "Compression failed: %m");
return 0; return 0;

2
src/journal/test-journal.c
View File

@ -203,7 +203,7 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
p = p + ALIGN64(le64toh(o->object.size)); p = p + ALIGN64(le64toh(o->object.size));
} }
is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0; is_compressed = COMPRESSION_FROM_OBJECT(o) != COMPRESSION_NONE;
(void) managed_journal_file_close(f); (void) managed_journal_file_close(f);

18
src/libsystemd/meson.build
View File

@ -5,8 +5,6 @@ sd_journal_sources = files(
'sd-journal/audit-type.h', 'sd-journal/audit-type.h',
'sd-journal/catalog.c', 'sd-journal/catalog.c',
'sd-journal/catalog.h', 'sd-journal/catalog.h',
'sd-journal/compress.c',
'sd-journal/compress.h',
'sd-journal/journal-def.h', 'sd-journal/journal-def.h',
'sd-journal/journal-file.c', 'sd-journal/journal-file.c',
'sd-journal/journal-file.h', 'sd-journal/journal-file.h',
@ -171,7 +169,8 @@ libsystemd_static = static_library(
libsystemd_sources, libsystemd_sources,
include_directories : libsystemd_includes, include_directories : libsystemd_includes,
c_args : libsystemd_c_args, c_args : libsystemd_c_args,
link_with : libbasic, link_with : [libbasic,
libbasic_compress],
dependencies : [threads, dependencies : [threads,
librt], librt],
build_by_default : false) build_by_default : false)
@ -206,19 +205,6 @@ tests += [
[files('sd-journal/test-catalog.c')], [files('sd-journal/test-catalog.c')],
[files('sd-journal/test-compress.c'),
[],
[liblz4,
libzstd,
libxz]],
[files('sd-journal/test-compress-benchmark.c'),
[],
[liblz4,
libzstd,
libxz],
[], '', 'timeout=90'],
[files('sd-journal/test-audit-type.c')], [files('sd-journal/test-audit-type.c')],
] ]

7
src/libsystemd/sd-journal/journal-def.h
View File

@ -42,14 +42,13 @@ typedef enum ObjectType {
_OBJECT_TYPE_MAX _OBJECT_TYPE_MAX
} ObjectType; } ObjectType;
/* Object flags /* Object flags (note that src/basic/compress.h uses the same values for the compression types) */
* The per-compression enums are also redefined in meson.build so that config.h is self-contained */
enum { enum {
OBJECT_COMPRESSED_XZ = 1 << 0, OBJECT_COMPRESSED_XZ = 1 << 0,
OBJECT_COMPRESSED_LZ4 = 1 << 1, OBJECT_COMPRESSED_LZ4 = 1 << 1,
OBJECT_COMPRESSED_ZSTD = 1 << 2, OBJECT_COMPRESSED_ZSTD = 1 << 2,
OBJECT_COMPRESSION_MASK = OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4 | OBJECT_COMPRESSED_ZSTD, _OBJECT_COMPRESSED_MASK = OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4 | OBJECT_COMPRESSED_ZSTD,
_OBJECT_COMPRESSED_MAX = OBJECT_COMPRESSION_MASK, _OBJECT_COMPRESSED_MAX = _OBJECT_COMPRESSED_MASK,
}; };
struct ObjectHeader { struct ObjectHeader {

27
src/libsystemd/sd-journal/journal-file.c
View File

@ -1372,6 +1372,7 @@ int journal_file_find_data_object_with_hash(
p = le64toh(f->data_hash_table[h].head_hash_offset); p = le64toh(f->data_hash_table[h].head_hash_offset);
while (p > 0) { while (p > 0) {
Compression c;
Object *o; Object *o;
r = journal_file_move_to_object(f, OBJECT_DATA, p, &o); r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
@ -1381,7 +1382,8 @@ int journal_file_find_data_object_with_hash(
if (le64toh(o->data.hash) != hash) if (le64toh(o->data.hash) != hash)
goto next; goto next;
if (o->object.flags & OBJECT_COMPRESSION_MASK) { c = COMPRESSION_FROM_OBJECT(o);
if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION #if HAVE_COMPRESSION
uint64_t l; uint64_t l;
size_t rsize = 0; size_t rsize = 0;
@ -1392,8 +1394,7 @@ int journal_file_find_data_object_with_hash(
l -= offsetof(Object, data.payload); l -= offsetof(Object, data.payload);
r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK, r = decompress_blob(c, o->data.payload, l, &f->compress_buffer, &rsize, 0);
o->data.payload, l, &f->compress_buffer, &rsize, 0);
if (r < 0) if (r < 0)
return r; return r;
@ -1590,7 +1591,7 @@ static int journal_file_append_data(
o->object.flags |= compression; o->object.flags |= compression;
log_debug("Compressed data object %"PRIu64" -> %zu using %s", log_debug("Compressed data object %"PRIu64" -> %zu using %s",
size, rsize, object_compressed_to_string(compression)); size, rsize, compression_to_string(compression));
} else } else
/* Compression didn't work, we don't really care why, let's continue without compression */ /* Compression didn't work, we don't really care why, let's continue without compression */
compression = 0; compression = 0;
@ -3143,6 +3144,7 @@ void journal_file_dump(JournalFile *f) {
p = le64toh(READ_NOW(f->header->header_size)); p = le64toh(READ_NOW(f->header->header_size));
while (p != 0) { while (p != 0) {
const char *s; const char *s;
Compression c;
r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o); r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
if (r < 0) if (r < 0)
@ -3180,9 +3182,10 @@ void journal_file_dump(JournalFile *f) {
break; break;
} }
if (o->object.flags & OBJECT_COMPRESSION_MASK) c = COMPRESSION_FROM_OBJECT(o);
if (c > COMPRESSION_NONE)
printf("Flags: %s\n", printf("Flags: %s\n",
object_compressed_to_string(o->object.flags & OBJECT_COMPRESSION_MASK)); compression_to_string(c));
if (p == le64toh(f->header->tail_object_offset)) if (p == le64toh(f->header->tail_object_offset))
p = 0; p = 0;
@ -3367,11 +3370,11 @@ int journal_file_open(
#endif #endif
}; };
if (DEFAULT_COMPRESSION == OBJECT_COMPRESSED_ZSTD) if (DEFAULT_COMPRESSION == COMPRESSION_ZSTD)
f->compress_zstd = FLAGS_SET(file_flags, JOURNAL_COMPRESS); f->compress_zstd = FLAGS_SET(file_flags, JOURNAL_COMPRESS);
else if (DEFAULT_COMPRESSION == OBJECT_COMPRESSED_LZ4) else if (DEFAULT_COMPRESSION == COMPRESSION_LZ4)
f->compress_lz4 = FLAGS_SET(file_flags, JOURNAL_COMPRESS); f->compress_lz4 = FLAGS_SET(file_flags, JOURNAL_COMPRESS);
else if (DEFAULT_COMPRESSION == OBJECT_COMPRESSED_XZ) else if (DEFAULT_COMPRESSION == COMPRESSION_XZ)
f->compress_xz = FLAGS_SET(file_flags, JOURNAL_COMPRESS); f->compress_xz = FLAGS_SET(file_flags, JOURNAL_COMPRESS);
/* We turn on keyed hashes by default, but provide an environment variable to turn them off, if /* We turn on keyed hashes by default, but provide an environment variable to turn them off, if
@ -3689,6 +3692,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
items = newa(EntryItem, n); items = newa(EntryItem, n);
for (uint64_t i = 0; i < n; i++) { for (uint64_t i = 0; i < n; i++) {
Compression c;
uint64_t l, h; uint64_t l, h;
size_t t; size_t t;
void *data; void *data;
@ -3711,12 +3715,13 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
if ((uint64_t) t != l) if ((uint64_t) t != l)
return -E2BIG; return -E2BIG;
if (o->object.flags & OBJECT_COMPRESSION_MASK) { c = COMPRESSION_FROM_OBJECT(o);
if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION #if HAVE_COMPRESSION
size_t rsize = 0; size_t rsize = 0;
r = decompress_blob( r = decompress_blob(
o->object.flags & OBJECT_COMPRESSION_MASK, c,
o->data.payload, l, o->data.payload, l,
&from->compress_buffer, &rsize, &from->compress_buffer, &rsize,
0); 0);

13
src/libsystemd/sd-journal/journal-file.h
View File

@ -11,6 +11,7 @@
#include "sd-event.h" #include "sd-event.h"
#include "sd-id128.h" #include "sd-id128.h"
#include "compress.h"
#include "hashmap.h" #include "hashmap.h"
#include "journal-def.h" #include "journal-def.h"
#include "mmap-cache.h" #include "mmap-cache.h"
@ -265,3 +266,15 @@ uint64_t journal_file_hash_data(JournalFile *f, const void *data, size_t sz);
bool journal_field_valid(const char *p, size_t l, bool allow_protected); bool journal_field_valid(const char *p, size_t l, bool allow_protected);
const char* journal_object_type_to_string(ObjectType type) _const_; const char* journal_object_type_to_string(ObjectType type) _const_;
static inline Compression COMPRESSION_FROM_OBJECT(const Object *o) {
assert(o);
/* These flags and enums are defined the same way. Make sure this is really the case. */
assert_cc((Compression) 0 == COMPRESSION_NONE);
assert_cc((Compression) OBJECT_COMPRESSED_XZ == COMPRESSION_XZ);
assert_cc((Compression) OBJECT_COMPRESSED_LZ4 == COMPRESSION_LZ4);
assert_cc((Compression) OBJECT_COMPRESSED_ZSTD == COMPRESSION_ZSTD);
return o->object.flags & _OBJECT_COMPRESSED_MASK;
}

11
src/libsystemd/sd-journal/journal-verify.c
View File

@ -111,21 +111,22 @@ static void flush_progress(void) {
} while (0) } while (0)
static int hash_payload(JournalFile *f, Object *o, uint64_t offset, const uint8_t *src, uint64_t size, uint64_t *res_hash) { static int hash_payload(JournalFile *f, Object *o, uint64_t offset, const uint8_t *src, uint64_t size, uint64_t *res_hash) {
int compression, r; Compression c;
int r;
assert(o); assert(o);
assert(src); assert(src);
assert(res_hash); assert(res_hash);
compression = o->object.flags & OBJECT_COMPRESSION_MASK; c = COMPRESSION_FROM_OBJECT(o);
if (compression) { if (c != COMPRESSION_NONE) {
_cleanup_free_ void *b = NULL; _cleanup_free_ void *b = NULL;
size_t b_size; size_t b_size;
r = decompress_blob(compression, src, size, &b, &b_size, 0); r = decompress_blob(c, src, size, &b, &b_size, 0);
if (r < 0) { if (r < 0) {
error_errno(offset, r, "%s decompression failed: %m", error_errno(offset, r, "%s decompression failed: %m",
object_compressed_to_string(compression)); compression_to_string(c));
return r; return r;
} }

36
src/libsystemd/sd-journal/sd-journal.c
View File

@ -2298,7 +2298,7 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
Object *d; Object *d;
uint64_t p, l; uint64_t p, l;
size_t t; size_t t;
int compression; Compression c;
p = le64toh(o->entry.items[i].object_offset); p = le64toh(o->entry.items[i].object_offset);
r = journal_file_move_to_object(f, OBJECT_DATA, p, &d); r = journal_file_move_to_object(f, OBJECT_DATA, p, &d);
@ -2311,24 +2311,26 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
l = le64toh(d->object.size) - offsetof(Object, data.payload); l = le64toh(d->object.size) - offsetof(Object, data.payload);
compression = d->object.flags & OBJECT_COMPRESSION_MASK; c = COMPRESSION_FROM_OBJECT(d);
if (compression) { if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION #if HAVE_COMPRESSION
r = decompress_startswith(compression, r = decompress_startswith(
d->data.payload, l, c,
&f->compress_buffer, d->data.payload, l,
field, field_length, '='); &f->compress_buffer,
field, field_length, '=');
if (r < 0) if (r < 0)
log_debug_errno(r, "Cannot decompress %s object of length %"PRIu64" at offset "OFSfmt": %m", log_debug_errno(r, "Cannot decompress %s object of length %"PRIu64" at offset "OFSfmt": %m",
object_compressed_to_string(compression), l, p); compression_to_string(c), l, p);
else if (r > 0) { else if (r > 0) {
size_t rsize; size_t rsize;
r = decompress_blob(compression, r = decompress_blob(
d->data.payload, l, c,
&f->compress_buffer, &rsize, d->data.payload, l,
j->data_threshold); &f->compress_buffer, &rsize,
j->data_threshold);
if (r < 0) if (r < 0)
return r; return r;
@ -2366,9 +2368,9 @@ static int return_data(
const void **ret_data, const void **ret_data,
size_t *ret_size) { size_t *ret_size) {
size_t t; Compression c;
uint64_t l; uint64_t l;
int compression; size_t t;
assert(j); assert(j);
assert(f); assert(f);
@ -2383,14 +2385,14 @@ static int return_data(
if ((uint64_t) t != l) if ((uint64_t) t != l)
return -E2BIG; return -E2BIG;
compression = o->object.flags & OBJECT_COMPRESSION_MASK; c = COMPRESSION_FROM_OBJECT(o);
if (compression) { if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION #if HAVE_COMPRESSION
size_t rsize; size_t rsize;
int r; int r;
r = decompress_blob( r = decompress_blob(
compression, c,
o->data.payload, l, o->data.payload, l,
&f->compress_buffer, &rsize, &f->compress_buffer, &rsize,
j->data_threshold); j->data_threshold);

10
src/test/meson.build
View File

@ -172,6 +172,16 @@ tests += [
[files('test-recurse-dir.c')], [files('test-recurse-dir.c')],
[files('test-compress.c'),
[libshared,
libbasic_compress]],
[files('test-compress-benchmark.c'),
[libshared,
libbasic_compress],
[],
[], '', 'timeout=90'],
[files('test-data-fd-util.c')], [files('test-data-fd-util.c')],
[files('test-static-destruct.c')], [files('test-static-destruct.c')],

0
src/libsystemd/sd-journal/test-compress-benchmark.c → src/test/test-compress-benchmark.c
View File

18
src/libsystemd/sd-journal/test-compress.c → src/test/test-compress.c
View File

@ -293,10 +293,10 @@ int main(int argc, char *argv[]) {
random_bytes(data + 7, sizeof(data) - 7); random_bytes(data + 7, sizeof(data) - 7);
#if HAVE_XZ #if HAVE_XZ
test_compress_decompress(OBJECT_COMPRESSED_XZ, "XZ", test_compress_decompress(COMPRESSION_XZ, "XZ",
compress_blob_xz, decompress_blob_xz, compress_blob_xz, decompress_blob_xz,
text, sizeof(text), false); text, sizeof(text), false);
test_compress_decompress(OBJECT_COMPRESSED_XZ, "XZ", test_compress_decompress(COMPRESSION_XZ, "XZ",
compress_blob_xz, decompress_blob_xz, compress_blob_xz, decompress_blob_xz,
data, sizeof(data), true); data, sizeof(data), true);
@ -310,7 +310,7 @@ int main(int argc, char *argv[]) {
compress_blob_xz, decompress_startswith_xz, compress_blob_xz, decompress_startswith_xz,
huge, HUGE_SIZE, true); huge, HUGE_SIZE, true);
test_compress_stream(OBJECT_COMPRESSED_XZ, "XZ", "xzcat", test_compress_stream(COMPRESSION_XZ, "XZ", "xzcat",
compress_stream_xz, decompress_stream_xz, srcfile); compress_stream_xz, decompress_stream_xz, srcfile);
test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz); test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
@ -320,10 +320,10 @@ int main(int argc, char *argv[]) {
#endif #endif
#if HAVE_LZ4 #if HAVE_LZ4
test_compress_decompress(OBJECT_COMPRESSED_LZ4, "LZ4", test_compress_decompress(COMPRESSION_LZ4, "LZ4",
compress_blob_lz4, decompress_blob_lz4, compress_blob_lz4, decompress_blob_lz4,
text, sizeof(text), false); text, sizeof(text), false);
test_compress_decompress(OBJECT_COMPRESSED_LZ4, "LZ4", test_compress_decompress(COMPRESSION_LZ4, "LZ4",
compress_blob_lz4, decompress_blob_lz4, compress_blob_lz4, decompress_blob_lz4,
data, sizeof(data), true); data, sizeof(data), true);
@ -337,7 +337,7 @@ int main(int argc, char *argv[]) {
compress_blob_lz4, decompress_startswith_lz4, compress_blob_lz4, decompress_startswith_lz4,
huge, HUGE_SIZE, true); huge, HUGE_SIZE, true);
test_compress_stream(OBJECT_COMPRESSED_LZ4, "LZ4", "lz4cat", test_compress_stream(COMPRESSION_LZ4, "LZ4", "lz4cat",
compress_stream_lz4, decompress_stream_lz4, srcfile); compress_stream_lz4, decompress_stream_lz4, srcfile);
test_lz4_decompress_partial(); test_lz4_decompress_partial();
@ -349,10 +349,10 @@ int main(int argc, char *argv[]) {
#endif #endif
#if HAVE_ZSTD #if HAVE_ZSTD
test_compress_decompress(OBJECT_COMPRESSED_ZSTD, "ZSTD", test_compress_decompress(COMPRESSION_ZSTD, "ZSTD",
compress_blob_zstd, decompress_blob_zstd, compress_blob_zstd, decompress_blob_zstd,
text, sizeof(text), false); text, sizeof(text), false);
test_compress_decompress(OBJECT_COMPRESSED_ZSTD, "ZSTD", test_compress_decompress(COMPRESSION_ZSTD, "ZSTD",
compress_blob_zstd, decompress_blob_zstd, compress_blob_zstd, decompress_blob_zstd,
data, sizeof(data), true); data, sizeof(data), true);
@ -366,7 +366,7 @@ int main(int argc, char *argv[]) {
compress_blob_zstd, decompress_startswith_zstd, compress_blob_zstd, decompress_startswith_zstd,
huge, HUGE_SIZE, true); huge, HUGE_SIZE, true);
test_compress_stream(OBJECT_COMPRESSED_ZSTD, "ZSTD", "zstdcat", test_compress_stream(COMPRESSION_ZSTD, "ZSTD", "zstdcat",
compress_stream_zstd, decompress_stream_zstd, srcfile); compress_stream_zstd, decompress_stream_zstd, srcfile);
test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd); test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);

2
src/test/test-tables.c
View File

@ -123,7 +123,7 @@ int main(int argc, char **argv) {
test_table(unit_type, UNIT_TYPE); test_table(unit_type, UNIT_TYPE);
test_table(virtualization, VIRTUALIZATION); test_table(virtualization, VIRTUALIZATION);
test_table_sparse(object_compressed, OBJECT_COMPRESSED); test_table_sparse(compression, COMPRESSION);
assert_cc(sizeof(sd_device_action_t) == sizeof(int64_t)); assert_cc(sizeof(sd_device_action_t) == sizeof(int64_t));