shaba/systemd
1
0
Fork 0
mirror of https://github.com/systemd/systemd.git synced 2024-12-23 21:35:11 +03:00
Code

compress: support streaming lz4 without full input mmap

Browse Source 
The advantage of stream compression is keeping a low memory profile,
but the lz4 stream compressor usage mmaps the whole file in memory.

Change it to read bits by bits, like the other stream compression
helpers.
This commit is contained in:
Luca Boccassi 2021-03-09 14:57:44 +00:00 committed by Zbigniew Jędrzejewski-Szmek
parent bcceabcfc2
commit 834bab01f9
1 changed files with 36 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
src/libsystemd/sd-journal/compress.c
View File

@ -614,14 +614,10 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
#if HAVE_LZ4 #if HAVE_LZ4
LZ4F_errorCode_t c; LZ4F_errorCode_t c;
_cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL; _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
_cleanup_free_ char *buf = NULL; _cleanup_free_ void *in_buff = NULL;
char *src = NULL; _cleanup_free_ char *out_buff = NULL;
size_t size, n, total_in = 0, total_out, offset = 0, frame_size; size_t out_allocsize, n, total_in = 0, total_out, offset = 0, frame_size;
struct stat st;
int r; int r;
static const LZ4F_compressOptions_t options = {
.stableSrc = 1,
};
static const LZ4F_preferences_t preferences = { static const LZ4F_preferences_t preferences = {
.frameInfo.blockSizeID = 5, .frameInfo.blockSizeID = 5,
}; };
@ -630,74 +626,66 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
if (LZ4F_isError(c)) if (LZ4F_isError(c))
return -ENOMEM; return -ENOMEM;
if (fstat(fdf, &st) < 0)
return log_debug_errno(errno, "fstat() failed: %m");
frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences); frame_size = LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
size = frame_size + 64*1024; /* add some space for header and trailer */ out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */
buf = malloc(size); out_buff = malloc(out_allocsize);
if (!buf) if (!out_buff)
return -ENOMEM; return -ENOMEM;
n = offset = total_out = LZ4F_compressBegin(ctx, buf, size, &preferences); in_buff = malloc(LZ4_BUFSIZE);
if (!in_buff)
return -ENOMEM;
n = offset = total_out = LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences);
if (LZ4F_isError(n)) if (LZ4F_isError(n))
return -EINVAL; return -EINVAL;
src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0); log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n);
if (src == MAP_FAILED)
return -errno;
log_debug("Buffer size is %zu bytes, header size %zu bytes.", size, n); for (;;) {
while (total_in < (size_t) st.st_size) {
ssize_t k; ssize_t k;
k = MIN(LZ4_BUFSIZE, st.st_size - total_in); k = loop_read(fdf, in_buff, LZ4_BUFSIZE, true);
n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, if (k < 0)
src + total_in, k, &options); return k;
if (LZ4F_isError(n)) { if (k == 0)
r = -ENOTRECOVERABLE; break;
goto cleanup; n = LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset,
} in_buff, k, NULL);
if (LZ4F_isError(n))
return -ENOTRECOVERABLE;
total_in += k; total_in += k;
offset += n; offset += n;
total_out += n; total_out += n;
if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) { if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes)
r = log_debug_errno(SYNTHETIC_ERRNO(EFBIG), return log_debug_errno(SYNTHETIC_ERRNO(EFBIG),
"Compressed stream longer than %" PRIu64 " bytes", max_bytes); "Compressed stream longer than %" PRIu64 " bytes", max_bytes);
goto cleanup;
}
if (size - offset < frame_size + 4) { if (out_allocsize - offset < frame_size + 4) {
k = loop_write(fdt, buf, offset, false); k = loop_write(fdt, out_buff, offset, false);
if (k < 0) { if (k < 0)
r = k; return k;
goto cleanup;
}
offset = 0; offset = 0;
} }
} }
n = LZ4F_compressEnd(ctx, buf + offset, size - offset, &options); n = LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL);
if (LZ4F_isError(n)) { if (LZ4F_isError(n))
r = -ENOTRECOVERABLE; return -ENOTRECOVERABLE;
goto cleanup;
}
offset += n; offset += n;
total_out += n; total_out += n;
r = loop_write(fdt, buf, offset, false); r = loop_write(fdt, out_buff, offset, false);
if (r < 0) if (r < 0)
goto cleanup; return r;
log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)", log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out, total_in, total_out,
(double) total_out / total_in * 100); (double) total_out / total_in * 100);
cleanup:
munmap(src, st.st_size); return 0;
return r;
#else #else
return -EPROTONOSUPPORT; return -EPROTONOSUPPORT;
#endif #endif