shaba/lvm2
1
0
Fork 0
mirror of git://sourceware.org/git/lvm2.git synced 2024-12-21 13:34:40 +03:00
Code Releases Activity

archiving: refactor code to allocate less memory

Browse Source 
Do not store full path with each archived name reduces memory usage if
the directory has thousands of entries and just add 'dir' path when
needed.

Also emit info print message to a user if the total size of archived
files for a VG is more then 128MiB or 8192 files.

TODO: Consider wheather adding a new 'lvm.conf  archive{option}' to support
trimming these wild archive sizes can make situation better.
We already support retain_min && retain_days  - but if user is
generating too many and too large archives with minutes - maybe archiving
should be disabled by a user - as it's not producing anything largely usable
and just slows-down command ??
If we add 'retain_max & retain_max_size' the condition will go against
each other and we need to chose priorities.

mm
This commit is contained in:
Zdenek Kabelac 2021-10-01 16:19:53 +02:00
parent 98c57f7ce4
commit 5ea426e656
2 changed files with 65 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
WHATS_NEW
View File

@ -1,5 +1,6 @@
Version 2.03.14 - Version 2.03.14 -
================================== ==================================
Print info message with too many or too large archived files.
Reduce metadata readings during scanning phase. Reduce metadata readings during scanning phase.
Optimize computation of crc32 check sum with multiple PVs. Optimize computation of crc32 check sum with multiple PVs.
Enhance recover path on cache creation failure. Enhance recover path on cache creation failure.

106
lib/format_text/archive.c
View File

@ -49,7 +49,7 @@
struct archive_file { struct archive_file {
struct dm_list list; struct dm_list list;
const char *path; const char *name;
uint32_t index; uint32_t index;
}; };
@ -105,18 +105,6 @@ static void _insert_archive_file(struct dm_list *head, struct archive_file *b)
dm_list_add_h(&bf->list, &b->list); dm_list_add_h(&bf->list, &b->list);
} }
static char *_join_file_to_dir(struct dm_pool *mem, const char *dir, const char *name)
{
if (!dm_pool_begin_object(mem, 32) ||
!dm_pool_grow_object(mem, dir, strlen(dir)) ||
!dm_pool_grow_object(mem, "/", 1) ||
!dm_pool_grow_object(mem, name, strlen(name)) ||
!dm_pool_grow_object(mem, "\0", 1))
return_NULL;
return dm_pool_end_object(mem);
}
/* /*
* Returns a list of archive_files. * Returns a list of archive_files.
*/ */
@ -125,7 +113,7 @@ static struct dm_list *_scan_archive(struct dm_pool *mem,
{ {
int i, count; int i, count;
uint32_t ix; uint32_t ix;
char vgname_found[64], *path; char vgname_found[64], *name;
struct dirent **dirent = NULL; struct dirent **dirent = NULL;
struct archive_file *af; struct archive_file *af;
struct dm_list *results; struct dm_list *results;
@ -159,7 +147,7 @@ static struct dm_list *_scan_archive(struct dm_pool *mem,
if (strcmp(vgname, vgname_found)) if (strcmp(vgname, vgname_found))
continue; continue;
if (!(path = _join_file_to_dir(mem, dir, dirent[i]->d_name))) if (!(name = dm_pool_strdup(mem, dirent[i]->d_name)))
goto_out; goto_out;
/* /*
@ -172,7 +160,7 @@ static struct dm_list *_scan_archive(struct dm_pool *mem,
} }
af->index = ix; af->index = ix;
af->path = path; af->name = name;
/* /*
* Insert it to the correct part of the list. * Insert it to the correct part of the list.
@ -188,12 +176,15 @@ static struct dm_list *_scan_archive(struct dm_pool *mem,
return results; return results;
} }
static void _remove_expired(struct dm_list *archives, uint32_t archives_size, static void _remove_expired(const char *dir, const char *vgname,
struct dm_list *archives, uint32_t archives_size,
uint32_t retain_days, uint32_t min_archive) uint32_t retain_days, uint32_t min_archive)
{ {
struct archive_file *bf; struct archive_file *bf;
struct stat sb; struct stat sb;
time_t retain_time; time_t retain_time;
uint64_t sum = 0;
char path[PATH_MAX];
/* Make sure there are enough archives to even bother looking for /* Make sure there are enough archives to even bother looking for
* expired ones... */ * expired ones... */
@ -205,23 +196,32 @@ static void _remove_expired(struct dm_list *archives, uint32_t archives_size,
/* Assume list is ordered newest first (by index) */ /* Assume list is ordered newest first (by index) */
dm_list_iterate_back_items(bf, archives) { dm_list_iterate_back_items(bf, archives) {
if (dm_snprintf(path, sizeof(path), "%s/%s", dir, bf->name) < 0)
continue;
/* Get the mtime of the file and unlink if too old */ /* Get the mtime of the file and unlink if too old */
if (stat(bf->path, &sb)) { if (stat(path, &sb)) {
log_sys_error("stat", bf->path); log_sys_debug("stat", path);
continue; continue;
} }
sum += sb.st_size;
if (sb.st_mtime > retain_time) if (sb.st_mtime > retain_time)
return; continue;
log_very_verbose("Expiring archive %s", bf->path); log_very_verbose("Expiring archive %s", path);
if (unlink(bf->path)) if (unlink(path))
log_sys_error("unlink", bf->path); log_sys_debug("unlink", path);
/* Don't delete any more if we've reached the minimum */ /* Don't delete any more if we've reached the minimum */
if (--archives_size <= min_archive) if (--archives_size <= min_archive)
return; break;
} }
sum /= 1024 * 1024;
if (sum > 128 || archives_size > 8192)
log_print_unless_silent("Consider prunning %s VG archive with more then %u MiB in %u files (check archiving is needed in lvm.conf).",
vgname, (unsigned)sum, archives_size);
} }
int archive_vg(struct volume_group *vg, int archive_vg(struct volume_group *vg,
@ -292,25 +292,30 @@ int archive_vg(struct volume_group *vg,
if (!renamed) if (!renamed)
log_error("Archive rename failed for %s", temp_file); log_error("Archive rename failed for %s", temp_file);
_remove_expired(archives, dm_list_size(archives) + renamed, retain_days, _remove_expired(dir, vg->name, archives, dm_list_size(archives) + renamed, retain_days,
min_archive); min_archive);
return 1; return 1;
} }
static void _display_archive(struct cmd_context *cmd, struct archive_file *af) static void _display_archive(struct cmd_context *cmd, const char *dir, struct archive_file *af)
{ {
struct volume_group *vg = NULL; struct volume_group *vg = NULL;
struct format_instance *tf; struct format_instance *tf;
struct format_instance_ctx fic; struct format_instance_ctx fic;
struct text_context tc = {.path_live = af->path, struct text_context tc = { NULL };
.path_edit = NULL, char path[PATH_MAX];
.desc = NULL};
time_t when; time_t when;
char *desc; char *desc;
if (dm_snprintf(path, sizeof(path), "%s/%s", dir, af->name) < 0) {
log_debug("Created path %s/%s is too long.", dir, af->name);
return;
}
log_print(" "); log_print(" ");
log_print("File:\t\t%s", af->path); log_print("File:\t\t%s/%s", path, af->name);
tc.path_live = path;
fic.type = FMT_INSTANCE_PRIVATE_MDAS; fic.type = FMT_INSTANCE_PRIVATE_MDAS;
fic.context.private = &tc; fic.context.private = &tc;
@ -324,7 +329,7 @@ static void _display_archive(struct cmd_context *cmd, struct archive_file *af)
* retrieve the archive time and description. * retrieve the archive time and description.
*/ */
/* FIXME Use variation on _vg_read */ /* FIXME Use variation on _vg_read */
if (!(vg = text_read_metadata_file(tf, af->path, &when, &desc))) { if (!(vg = text_read_metadata_file(tf, path, &when, &desc))) {
log_error("Unable to read archive file."); log_error("Unable to read archive file.");
tf->fmt->ops->destroy_instance(tf); tf->fmt->ops->destroy_instance(tf);
return; return;
@ -349,7 +354,7 @@ int archive_list(struct cmd_context *cmd, const char *dir, const char *vgname)
log_print("No archives found in %s.", dir); log_print("No archives found in %s.", dir);
dm_list_iterate_back_items(af, archives) dm_list_iterate_back_items(af, archives)
_display_archive(cmd, af); _display_archive(cmd, dir, af);
dm_pool_free(cmd->mem, archives); dm_pool_free(cmd->mem, archives);
@ -358,29 +363,46 @@ int archive_list(struct cmd_context *cmd, const char *dir, const char *vgname)
int archive_list_file(struct cmd_context *cmd, const char *file) int archive_list_file(struct cmd_context *cmd, const char *file)
{ {
struct archive_file af; struct archive_file af = { 0 };
char path[PATH_MAX];
size_t len;
af.path = file; if (!path_exists(file)) {
log_error("Archive file %s not found.", file);
if (!path_exists(af.path)) {
log_error("Archive file %s not found.", af.path);
return 0; return 0;
} }
_display_archive(cmd, &af); if (!(af.name = strrchr(file, '/'))) {
log_error("No '/' in file path %s found.", file);
return 0;
}
len = (size_t)(af.name - file);
if (len >= sizeof(path)) {
log_error(INTERNAL_ERROR "Passed file path name %s is too long.", file);
return 0;
}
memcpy(path, file, len);
path[len] = 0;
af.name++; /* jump over '/' */
_display_archive(cmd, path, &af);
return 1; return 1;
} }
int backup_list(struct cmd_context *cmd, const char *dir, const char *vgname) int backup_list(struct cmd_context *cmd, const char *dir, const char *vgname)
{ {
struct archive_file af; struct archive_file af = { .name = vgname };
char path[PATH_MAX];
if (!(af.path = _join_file_to_dir(cmd->mem, dir, vgname))) if (dm_snprintf(path, sizeof(path), "%s/%s", dir, vgname) < 0)
return_0; return_0;
if (path_exists(af.path)) if (path_exists(path))
_display_archive(cmd, &af); _display_archive(cmd, dir, &af);
return 1; return 1;
} }