mirror of
https://github.com/systemd/systemd.git
synced 2024-12-23 21:35:11 +03:00
journal: fix space reservation limit enforcement
This commit is contained in:
parent
466ccd92e2
commit
bc85bfee87
@ -135,9 +135,6 @@ _packed_ struct Header {
|
||||
sd_id128_t seqnum_id;
|
||||
uint64_t arena_offset;
|
||||
uint64_t arena_size;
|
||||
uint64_t arena_max_size; /* obsolete */
|
||||
uint64_t arena_min_size; /* obsolete */
|
||||
uint64_t arena_keep_free; /* obsolete */
|
||||
uint64_t data_hash_table_offset; /* for looking up data objects */
|
||||
uint64_t data_hash_table_size;
|
||||
uint64_t field_hash_table_offset; /* for looking up field objects */
|
||||
|
@ -31,12 +31,6 @@
|
||||
#include "journal-file.h"
|
||||
#include "lookup3.h"
|
||||
|
||||
#define DEFAULT_ARENA_MAX_SIZE (16ULL*1024ULL*1024ULL*1024ULL)
|
||||
#define DEFAULT_ARENA_MIN_SIZE (256ULL*1024ULL)
|
||||
#define DEFAULT_ARENA_KEEP_FREE (1ULL*1024ULL*1024ULL)
|
||||
|
||||
#define DEFAULT_MAX_USE (16ULL*1024ULL*1024ULL*16ULL)
|
||||
|
||||
#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*16ULL)
|
||||
#define DEFAULT_FIELD_HASH_TABLE_SIZE (2047ULL*16ULL)
|
||||
|
||||
@ -76,9 +70,6 @@ static int journal_file_init_header(JournalFile *f, JournalFile *template) {
|
||||
zero(h);
|
||||
memcpy(h.signature, signature, 8);
|
||||
h.arena_offset = htole64(ALIGN64(sizeof(h)));
|
||||
h.arena_max_size = htole64(DEFAULT_ARENA_MAX_SIZE);
|
||||
h.arena_min_size = htole64(DEFAULT_ARENA_MIN_SIZE);
|
||||
h.arena_keep_free = htole64(DEFAULT_ARENA_KEEP_FREE);
|
||||
|
||||
r = sd_id128_randomize(&h.file_id);
|
||||
if (r < 0)
|
||||
@ -161,16 +152,10 @@ static int journal_file_verify_header(JournalFile *f) {
|
||||
}
|
||||
|
||||
static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) {
|
||||
uint64_t asize;
|
||||
uint64_t old_size, new_size;
|
||||
|
||||
assert(f);
|
||||
|
||||
if (offset < le64toh(f->header->arena_offset))
|
||||
return -EINVAL;
|
||||
|
||||
new_size = PAGE_ALIGN(offset + size);
|
||||
|
||||
/* We assume that this file is not sparse, and we know that
|
||||
* for sure, since we always call posix_fallocate()
|
||||
* ourselves */
|
||||
@ -179,12 +164,19 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size)
|
||||
le64toh(f->header->arena_offset) +
|
||||
le64toh(f->header->arena_size);
|
||||
|
||||
if (old_size >= new_size)
|
||||
new_size = PAGE_ALIGN(offset + size);
|
||||
if (new_size < le64toh(f->header->arena_offset))
|
||||
new_size = le64toh(f->header->arena_offset);
|
||||
|
||||
if (new_size <= old_size)
|
||||
return 0;
|
||||
|
||||
asize = new_size - le64toh(f->header->arena_offset);
|
||||
if (f->metrics.max_size > 0 &&
|
||||
new_size > f->metrics.max_size)
|
||||
return -E2BIG;
|
||||
|
||||
if (asize > le64toh(f->header->arena_min_size)) {
|
||||
if (new_size > f->metrics.min_size &&
|
||||
f->metrics.keep_free > 0) {
|
||||
struct statvfs svfs;
|
||||
|
||||
if (fstatvfs(f->fd, &svfs) >= 0) {
|
||||
@ -192,8 +184,8 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size)
|
||||
|
||||
available = svfs.f_bfree * svfs.f_bsize;
|
||||
|
||||
if (available >= f->header->arena_keep_free)
|
||||
available -= f->header->arena_keep_free;
|
||||
if (available >= f->metrics.keep_free)
|
||||
available -= f->metrics.keep_free;
|
||||
else
|
||||
available = 0;
|
||||
|
||||
@ -202,16 +194,16 @@ static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size)
|
||||
}
|
||||
}
|
||||
|
||||
if (asize > le64toh(f->header->arena_max_size))
|
||||
return -E2BIG;
|
||||
|
||||
/* Note that the glibc fallocate() fallback is very
|
||||
inefficient, hence we try to minimize the allocation area
|
||||
as we can. */
|
||||
if (posix_fallocate(f->fd, old_size, new_size - old_size) < 0)
|
||||
return -errno;
|
||||
|
||||
if (fstat(f->fd, &f->last_stat) < 0)
|
||||
return -errno;
|
||||
|
||||
f->header->arena_size = htole64(asize);
|
||||
f->header->arena_size = new_size - htole64(f->header->arena_offset);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -576,6 +568,9 @@ int journal_file_find_data_object_with_hash(
|
||||
|
||||
osize = offsetof(Object, data.payload) + size;
|
||||
|
||||
if (f->header->data_hash_table_size == 0)
|
||||
return -EBADMSG;
|
||||
|
||||
h = hash % (le64toh(f->header->data_hash_table_size) / sizeof(HashItem));
|
||||
p = le64toh(f->data_hash_table[h].head_hash_offset);
|
||||
|
||||
@ -816,7 +811,7 @@ static int journal_file_link_entry(JournalFile *f, Object *o, uint64_t offset) {
|
||||
if (r < 0)
|
||||
return r;
|
||||
|
||||
log_error("%s %lu", f->path, (unsigned long) f->header->n_entries);
|
||||
log_error("=> %s seqnr=%lu n_entries=%lu", f->path, (unsigned long) o->entry.seqnum, (unsigned long) f->header->n_entries);
|
||||
|
||||
if (f->header->head_entry_realtime == 0)
|
||||
f->header->head_entry_realtime = o->entry.realtime;
|
||||
@ -887,6 +882,8 @@ static void journal_file_post_change(JournalFile *f) {
|
||||
* trigger IN_MODIFY by truncating the journal file to its
|
||||
* current size which triggers IN_MODIFY. */
|
||||
|
||||
__sync_synchronize();
|
||||
|
||||
if (ftruncate(f->fd, f->last_stat.st_size) < 0)
|
||||
log_error("Failed to to truncate file to its own size: %m");
|
||||
}
|
||||
@ -1626,6 +1623,10 @@ int journal_file_open(
|
||||
f->writable = (flags & O_ACCMODE) != O_RDONLY;
|
||||
f->prot = prot_from_flags(flags);
|
||||
|
||||
f->metrics.max_size = DEFAULT_MAX_SIZE;
|
||||
f->metrics.min_size = DEFAULT_MIN_SIZE;
|
||||
f->metrics.keep_free = DEFAULT_KEEP_FREE;
|
||||
|
||||
f->path = strdup(fname);
|
||||
if (!f->path) {
|
||||
r = -ENOMEM;
|
||||
|
@ -28,6 +28,11 @@
|
||||
#include "util.h"
|
||||
#include "sd-id128.h"
|
||||
|
||||
#define DEFAULT_MAX_SIZE (1024ULL*128ULL)
|
||||
#define DEFAULT_MIN_SIZE (256ULL*1024ULL)
|
||||
#define DEFAULT_KEEP_FREE (1ULL*1024ULL*1024ULL)
|
||||
#define DEFAULT_MAX_USE (16ULL*1024ULL*1024ULL*16ULL)
|
||||
|
||||
typedef struct Window {
|
||||
void *ptr;
|
||||
uint64_t offset;
|
||||
@ -45,6 +50,12 @@ enum {
|
||||
_WINDOW_MAX
|
||||
};
|
||||
|
||||
typedef struct JournalMetrics {
|
||||
uint64_t max_size;
|
||||
uint64_t min_size;
|
||||
uint64_t keep_free;
|
||||
} JournalMetrics;
|
||||
|
||||
typedef struct JournalFile {
|
||||
int fd;
|
||||
char *path;
|
||||
@ -62,6 +73,8 @@ typedef struct JournalFile {
|
||||
Window windows[_WINDOW_MAX];
|
||||
|
||||
uint64_t current_offset;
|
||||
|
||||
JournalMetrics metrics;
|
||||
} JournalFile;
|
||||
|
||||
typedef enum direction {
|
||||
|
@ -54,6 +54,9 @@ typedef struct Server {
|
||||
|
||||
char *buffer;
|
||||
size_t buffer_size;
|
||||
|
||||
JournalMetrics metrics;
|
||||
uint64_t max_use;
|
||||
} Server;
|
||||
|
||||
static void fix_perms(JournalFile *f, uid_t uid) {
|
||||
@ -153,6 +156,66 @@ static JournalFile* find_journal(Server *s, uid_t uid) {
|
||||
return f;
|
||||
}
|
||||
|
||||
static void server_vacuum(Server *s) {
|
||||
Iterator i;
|
||||
void *k;
|
||||
char *p;
|
||||
char ids[33];
|
||||
sd_id128_t machine;
|
||||
int r;
|
||||
JournalFile *f;
|
||||
|
||||
log_info("Rotating...");
|
||||
|
||||
if (s->runtime_journal) {
|
||||
r = journal_file_rotate(&s->runtime_journal);
|
||||
if (r < 0)
|
||||
log_error("Failed to rotate %s: %s", s->runtime_journal->path, strerror(-r));
|
||||
}
|
||||
|
||||
if (s->system_journal) {
|
||||
r = journal_file_rotate(&s->system_journal);
|
||||
if (r < 0)
|
||||
log_error("Failed to rotate %s: %s", s->system_journal->path, strerror(-r));
|
||||
}
|
||||
|
||||
HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) {
|
||||
r = journal_file_rotate(&f);
|
||||
if (r < 0)
|
||||
log_error("Failed to rotate %s: %s", f->path, strerror(-r));
|
||||
else
|
||||
hashmap_replace(s->user_journals, k, f);
|
||||
}
|
||||
|
||||
log_info("Vacuuming...");
|
||||
|
||||
r = sd_id128_get_machine(&machine);
|
||||
if (r < 0) {
|
||||
log_error("Failed to get machine ID: %s", strerror(-r));
|
||||
return;
|
||||
}
|
||||
|
||||
if (asprintf(&p, "/var/log/journal/%s", sd_id128_to_string(machine, ids)) < 0) {
|
||||
log_error("Out of memory.");
|
||||
return;
|
||||
}
|
||||
|
||||
r = journal_directory_vacuum(p, s->max_use, s->metrics.keep_free);
|
||||
if (r < 0 && r != -ENOENT)
|
||||
log_error("Failed to vacuum %s: %s", p, strerror(-r));
|
||||
free(p);
|
||||
|
||||
if (asprintf(&p, "/run/log/journal/%s", ids) < 0) {
|
||||
log_error("Out of memory.");
|
||||
return;
|
||||
}
|
||||
|
||||
r = journal_directory_vacuum(p, s->max_use, s->metrics.keep_free);
|
||||
if (r < 0 && r != -ENOENT)
|
||||
log_error("Failed to vacuum %s: %s", p, strerror(-r));
|
||||
free(p);
|
||||
}
|
||||
|
||||
static void dispatch_message(Server *s, struct iovec *iovec, unsigned n, unsigned m, struct ucred *ucred, struct timeval *tv) {
|
||||
char *pid = NULL, *uid = NULL, *gid = NULL,
|
||||
*source_time = NULL, *boot_id = NULL, *machine_id = NULL,
|
||||
@ -166,6 +229,7 @@ static void dispatch_message(Server *s, struct iovec *iovec, unsigned n, unsigne
|
||||
char *t;
|
||||
uid_t loginuid = 0, realuid = 0;
|
||||
JournalFile *f;
|
||||
bool vacuumed = false;
|
||||
|
||||
assert(s);
|
||||
assert(iovec || n == 0);
|
||||
@ -262,12 +326,23 @@ static void dispatch_message(Server *s, struct iovec *iovec, unsigned n, unsigne
|
||||
|
||||
assert(n <= m);
|
||||
|
||||
retry:
|
||||
f = find_journal(s, realuid == 0 ? 0 : loginuid);
|
||||
if (!f)
|
||||
log_warning("Dropping message, as we can't find a place to store the data.");
|
||||
else {
|
||||
r = journal_file_append_entry(f, NULL, iovec, n, &s->seqnum, NULL, NULL);
|
||||
|
||||
if (r == -E2BIG && !vacuumed) {
|
||||
log_info("Allocation limit reached.");
|
||||
|
||||
server_vacuum(s);
|
||||
vacuumed = true;
|
||||
|
||||
log_info("Retrying write.");
|
||||
goto retry;
|
||||
}
|
||||
|
||||
if (r < 0)
|
||||
log_error("Failed to write entry, ignoring: %s", strerror(-r));
|
||||
}
|
||||
@ -715,6 +790,10 @@ static int server_init(Server *s) {
|
||||
|
||||
zero(*s);
|
||||
s->syslog_fd = s->native_fd = s->signal_fd = -1;
|
||||
s->metrics.max_size = DEFAULT_MAX_SIZE;
|
||||
s->metrics.min_size = DEFAULT_MIN_SIZE;
|
||||
s->metrics.keep_free = DEFAULT_KEEP_FREE;
|
||||
s->max_use = DEFAULT_MAX_USE;
|
||||
|
||||
s->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
|
||||
if (s->epoll_fd < 0) {
|
||||
|
@ -38,7 +38,6 @@
|
||||
* - accelerate looking for "all hostnames" and suchlike.
|
||||
* - throttling
|
||||
* - cryptographic hash
|
||||
* - fix space reservation logic
|
||||
* - compression
|
||||
*/
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user