5
0
mirror of git://git.proxmox.com/git/pve-storage.git synced 2025-02-26 21:57:31 +03:00

Add prune_backups to storage API

Implement it for generic storages supporting backups
(i.e. directory-based storages) and add a wrapper for PBS.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
This commit is contained in:
Fabian Ebner 2020-07-09 14:45:42 +02:00 committed by Thomas Lamprecht
parent 3353698f45
commit 8f26b3910d
5 changed files with 565 additions and 2 deletions

View File

@ -40,11 +40,11 @@ use PVE::Storage::DRBDPlugin;
use PVE::Storage::PBSPlugin;
# Storage API version. Icrement it on changes in storage API interface.
use constant APIVER => 5;
use constant APIVER => 6;
# Age is the number of versions we're backward compatible with.
# This is like having 'current=APIVER' and age='APIAGE' in libtool,
# see https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
use constant APIAGE => 4;
use constant APIAGE => 5;
# load standard plugins
PVE::Storage::DirPlugin->register();
@ -1542,6 +1542,93 @@ sub extract_vzdump_config {
}
}
sub prune_backups {
my ($cfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
my $scfg = storage_config($cfg, $storeid);
die "storage '$storeid' does not support backups\n" if !$scfg->{content}->{backup};
if (!defined($keep)) {
die "no prune-backups options configured for storage '$storeid'\n"
if !defined($scfg->{'prune-backups'});
$keep = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'});
}
my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
return $plugin->prune_backups($scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc);
}
my $prune_mark = sub {
my ($prune_entries, $keep_count, $id_func) = @_;
return if !$keep_count;
my $already_included = {};
my $newly_included = {};
foreach my $prune_entry (@{$prune_entries}) {
my $mark = $prune_entry->{mark};
my $id = $id_func->($prune_entry->{ctime});
next if $already_included->{$id};
if (defined($mark)) {
$already_included->{$id} = 1 if $mark eq 'keep';
next;
}
if (!$newly_included->{$id}) {
last if scalar(keys %{$newly_included}) >= $keep_count;
$newly_included->{$id} = 1;
$prune_entry->{mark} = 'keep';
} else {
$prune_entry->{mark} = 'remove';
}
}
};
sub prune_mark_backup_group {
my ($backup_group, $keep) = @_;
my $prune_list = [ sort { $b->{ctime} <=> $a->{ctime} } @{$backup_group} ];
$prune_mark->($prune_list, $keep->{'keep-last'}, sub {
my ($ctime) = @_;
return $ctime;
});
$prune_mark->($prune_list, $keep->{'keep-hourly'}, sub {
my ($ctime) = @_;
my (undef, undef, $hour, $day, $month, $year) = localtime($ctime);
return "$hour/$day/$month/$year";
});
$prune_mark->($prune_list, $keep->{'keep-daily'}, sub {
my ($ctime) = @_;
my (undef, undef, undef, $day, $month, $year) = localtime($ctime);
return "$day/$month/$year";
});
$prune_mark->($prune_list, $keep->{'keep-weekly'}, sub {
my ($ctime) = @_;
my ($sec, $min, $hour, $day, $month, $year) = localtime($ctime);
my $iso_week = int(strftime("%V", $sec, $min, $hour, $day, $month - 1, $year - 1900));
my $iso_week_year = int(strftime("%G", $sec, $min, $hour, $day, $month - 1, $year - 1900));
return "$iso_week/$iso_week_year";
});
$prune_mark->($prune_list, $keep->{'keep-monthly'}, sub {
my ($ctime) = @_;
my (undef, undef, undef, undef, $month, $year) = localtime($ctime);
return "$month/$year";
});
$prune_mark->($prune_list, $keep->{'keep-yearly'}, sub {
my ($ctime) = @_;
my $year = (localtime($ctime))[5];
return "$year";
});
foreach my $prune_entry (@{$prune_list}) {
$prune_entry->{mark} //= 'remove';
}
}
sub volume_export {
my ($cfg, $fh, $volid, $format, $snapshot, $base_snapshot, $with_snapshots) = @_;

View File

@ -266,6 +266,74 @@ sub extract_vzdump_config {
return $config;
}
sub prune_backups {
my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
$logfunc //= sub { print "$_[1]\n" };
my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']);
$type = 'vm' if defined($type) && $type eq 'qemu';
$type = 'ct' if defined($type) && $type eq 'lxc';
my $backup_groups = {};
foreach my $backup (@{$backups}) {
(my $backup_type = $backup->{format}) =~ s/^pbs-//;
next if defined($type) && $backup_type ne $type;
my $backup_group = "$backup_type/$backup->{vmid}";
$backup_groups->{$backup_group} = 1;
}
my @param;
foreach my $opt (keys %{$keep}) {
push @param, "--$opt";
push @param, "$keep->{$opt}";
}
push @param, '--dry-run' if $dryrun;
my $prune_list = [];
my $failed;
foreach my $backup_group (keys %{$backup_groups}) {
$logfunc->('info', "running 'proxmox-backup-client prune' for '$backup_group'")
if !$dryrun;
eval {
my $res = run_client_cmd($scfg, $storeid, 'prune', [ $backup_group, @param ]);
foreach my $backup (@{$res}) {
die "result from proxmox-backup-client is not as expected\n"
if !defined($backup->{'backup-time'})
|| !defined($backup->{'backup-type'})
|| !defined($backup->{'backup-id'})
|| !defined($backup->{'keep'});
my $ctime = $backup->{'backup-time'};
my $type = $backup->{'backup-type'};
my $vmid = $backup->{'backup-id'};
my $volid = print_volid($storeid, $type, $vmid, $ctime);
push @{$prune_list}, {
ctime => $ctime,
mark => $backup->{keep} ? 'keep' : 'remove',
type => $type eq 'vm' ? 'qemu' : 'lxc',
vmid => $vmid,
volid => $volid,
};
}
};
if (my $err = $@) {
$logfunc->('err', "prune '$backup_group': $err\n");
$failed = 1;
}
}
die "error pruning backups - check log\n" if $failed;
return $prune_list;
}
my $autogen_encryption_key = sub {
my ($scfg, $storeid) = @_;
my $encfile = pbs_encryption_key_file_name($scfg, $storeid);

View File

@ -1174,6 +1174,71 @@ sub check_connection {
return 1;
}
sub prune_backups {
my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
$logfunc //= sub { print "$_[1]\n" };
my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']);
my $backup_groups = {};
my $prune_list = [];
foreach my $backup (@{$backups}) {
my $volid = $backup->{volid};
my $backup_vmid = $backup->{vmid};
my $archive_info = eval { PVE::Storage::archive_info($volid) } // {};
my $backup_type = $archive_info->{type} // 'unknown';
next if defined($type) && $type ne $backup_type;
my $prune_entry = {
ctime => $backup->{ctime},
type => $backup_type,
volid => $volid,
};
$prune_entry->{vmid} = $backup_vmid if defined($backup_vmid);
if ($archive_info->{is_std_name}) {
$prune_entry->{ctime} = $archive_info->{ctime};
my $group = "$backup_type/$backup_vmid";
push @{$backup_groups->{$group}}, $prune_entry;
} else {
# ignore backups that don't use the standard naming scheme
$prune_entry->{mark} = 'protected';
}
push @{$prune_list}, $prune_entry;
}
foreach my $backup_group (values %{$backup_groups}) {
PVE::Storage::prune_mark_backup_group($backup_group, $keep);
}
my $failed;
if (!$dryrun) {
foreach my $prune_entry (@{$prune_list}) {
next if $prune_entry->{mark} ne 'remove';
my $volid = $prune_entry->{volid};
$logfunc->('info', "removing backup '$volid'");
eval {
my (undef, $volname) = parse_volume_id($volid);
my $archive_path = $class->filesystem_path($scfg, $volname);
PVE::Storage::archive_remove($archive_path);
};
if (my $err = $@) {
$logfunc->('err', "error when removing backup '$volid' - $err\n");
$failed = 1;
}
}
}
die "error pruning backups - check log\n" if $failed;
return $prune_list;
}
# Import/Export interface:
# Any path based storage is assumed to support 'raw' and 'tar' streams, so
# the default implementations will return this if $scfg->{path} is set,

342
test/prune_backups_test.pm Normal file
View File

@ -0,0 +1,342 @@
package PVE::Storage::TestPruneBackups;
use strict;
use warnings;
use lib qw(..);
use PVE::Storage;
use Test::More;
use Test::MockModule;
my $storeid = 'BackTest123';
my @vmids = (1234, 9001);
# only includes the information needed for prune_backups
my $mocked_backups_lists = {};
my $basetime = 1577881101; # 2020_01_01-12_18_21 UTC
foreach my $vmid (@vmids) {
push @{$mocked_backups_lists->{default}}, (
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2018_05_26-11_18_21.tar.zst",
'ctime' => $basetime - 585*24*60*60 - 60*60,
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_18_21.tar.zst",
'ctime' => $basetime - 24*60*60 - 60*60,
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_19_21.tar.zst",
'ctime' => $basetime - 24*60*60 - 60*60 + 60,
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-11_18_21.tar.zst",
'ctime' => $basetime - 60*60,
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-12_18_21.tar.zst",
'ctime' => $basetime,
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-lxc-$vmid-2020_01_01-12_18_21.tar.zst",
'ctime' => $basetime,
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-$vmid-renamed.tar.zst",
'ctime' => 1234,
'vmid' => $vmid,
},
);
}
push @{$mocked_backups_lists->{year1970}}, (
{
'volid' => "$storeid:backup/vzdump-lxc-321-1970_01_01-00_01_23.tar.zst",
'ctime' => 83,
'vmid' => 321,
},
{
'volid' => "$storeid:backup/vzdump-lxc-321-2070_01_01-00_01_00.tar.zst",
'ctime' => 60*60*24 * (365*100 + 25) + 60,
'vmid' => 321,
},
);
push @{$mocked_backups_lists->{novmid}}, (
{
'volid' => "$storeid:backup/vzdump-lxc-novmid.tar.gz",
'ctime' => 1234,
},
);
my $current_list;
my $mock_plugin = Test::MockModule->new('PVE::Storage::Plugin');
$mock_plugin->redefine(list_volumes => sub {
my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
my $list = $mocked_backups_lists->{$current_list};
return $list if !defined($vmid);
return [ grep { $_->{vmid} eq $vmid } @{$list} ];
});
sub generate_expected {
my ($vmids, $type, $marks) = @_;
my @expected;
foreach my $vmid (@{$vmids}) {
push @expected, (
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2018_05_26-11_18_21.tar.zst",
'type' => 'qemu',
'ctime' => $basetime - 585*24*60*60 - 60*60,
'mark' => $marks->[0],
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_18_21.tar.zst",
'type' => 'qemu',
'ctime' => $basetime - 24*60*60 - 60*60,
'mark' => $marks->[1],
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_19_21.tar.zst",
'type' => 'qemu',
'ctime' => $basetime - 24*60*60 - 60*60 + 60,
'mark' => $marks->[2],
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-11_18_21.tar.zst",
'type' => 'qemu',
'ctime' => $basetime - 60*60,
'mark' => $marks->[3],
'vmid' => $vmid,
},
{
'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-12_18_21.tar.zst",
'type' => 'qemu',
'ctime' => $basetime,
'mark' => $marks->[4],
'vmid' => $vmid,
},
) if !defined($type) || $type eq 'qemu';
push @expected, (
{
'volid' => "$storeid:backup/vzdump-lxc-$vmid-2020_01_01-12_18_21.tar.zst",
'type' => 'lxc',
'ctime' => $basetime,
'mark' => $marks->[5],
'vmid' => $vmid,
},
) if !defined($type) || $type eq 'lxc';
push @expected, (
{
'volid' => "$storeid:backup/vzdump-$vmid-renamed.tar.zst",
'type' => 'unknown',
'ctime' => 1234,
'mark' => 'protected',
'vmid' => $vmid,
},
) if !defined($type);
}
return [ sort { $a->{volid} cmp $b->{volid} } @expected ];
}
# an array of test cases, each test is comprised of the following keys:
# description => to identify a single test
# vmid => VMID or undef for all
# type => 'qemu' or 'lxc' or undef for all
# keep => options describing what to keep
# list => backups list to use. defaults to 'default'
# expected => what prune_backups should return
#
# most of them are created further below
my $tests = [
{
description => 'last=3, multiple IDs',
keep => {
'keep-last' => 3,
},
expected => generate_expected(\@vmids, undef, ['remove', 'remove', 'keep', 'keep', 'keep', 'keep']),
},
{
description => 'weekly=2, one ID',
vmid => $vmids[0],
keep => {
'keep-weekly' => 2,
},
expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'remove', 'remove', 'keep', 'keep']),
},
{
description => 'daily=weekly=monthly=1, multiple IDs',
keep => {
'keep-hourly' => 0,
'keep-daily' => 1,
'keep-weekly' => 1,
'keep-monthly' => 1,
},
expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']),
},
{
description => 'hourly=4, one ID',
vmid => $vmids[0],
keep => {
'keep-hourly' => 4,
'keep-daily' => 0,
},
expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'keep', 'keep', 'keep', 'keep']),
},
{
description => 'yearly=2, multiple IDs',
keep => {
'keep-hourly' => 0,
'keep-daily' => 0,
'keep-weekly' => 0,
'keep-monthly' => 0,
'keep-yearly' => 2,
},
expected => generate_expected(\@vmids, undef, ['remove', 'remove', 'keep', 'remove', 'keep', 'keep']),
},
{
description => 'last=2,hourly=2 one ID',
vmid => $vmids[0],
keep => {
'keep-last' => 2,
'keep-hourly' => 2,
},
expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'keep', 'keep', 'keep', 'keep']),
},
{
description => 'last=1,monthly=2, multiple IDs',
keep => {
'keep-last' => 1,
'keep-monthly' => 2,
},
expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']),
},
{
description => 'monthly=3, one ID',
vmid => $vmids[0],
keep => {
'keep-monthly' => 3,
},
expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']),
},
{
description => 'last=daily=weekly=1, multiple IDs',
keep => {
'keep-last' => 1,
'keep-daily' => 1,
'keep-weekly' => 1,
},
expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']),
},
{
description => 'daily=2, one ID',
vmid => $vmids[0],
keep => {
'keep-daily' => 2,
},
expected => generate_expected([$vmids[0]], undef, ['remove', 'remove', 'keep', 'remove', 'keep', 'keep']),
},
{
description => 'weekly=monthly=1, multiple IDs',
keep => {
'keep-weekly' => 1,
'keep-monthly' => 1,
},
expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'remove', 'remove', 'keep', 'keep']),
},
{
description => 'weekly=yearly=1, one ID',
vmid => $vmids[0],
keep => {
'keep-weekly' => 1,
'keep-yearly' => 1,
},
expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'remove', 'remove', 'keep', 'keep']),
},
{
description => 'weekly=yearly=1, one ID, type qemu',
vmid => $vmids[0],
type => 'qemu',
keep => {
'keep-weekly' => 1,
'keep-yearly' => 1,
},
expected => generate_expected([$vmids[0]], 'qemu', ['keep', 'remove', 'remove', 'remove', 'keep', '']),
},
{
description => 'week=yearly=1, one ID, type lxc',
vmid => $vmids[0],
type => 'lxc',
keep => {
'keep-last' => 1,
},
expected => generate_expected([$vmids[0]], 'lxc', ['', '', '', '', '', 'keep']),
},
{
description => 'yearly=1, year before 2000',
keep => {
'keep-yearly' => 1,
},
list => 'year1970',
expected => [
{
'volid' => "$storeid:backup/vzdump-lxc-321-1970_01_01-00_01_23.tar.zst",
'ctime' => 83,
'mark' => 'remove',
'type' => 'lxc',
'vmid' => 321,
},
{
'volid' => "$storeid:backup/vzdump-lxc-321-2070_01_01-00_01_00.tar.zst",
'ctime' => 60*60*24 * (365*100 + 25) + 60,
'mark' => 'keep',
'type' => 'lxc',
'vmid' => 321,
},
],
},
{
description => 'last=1, ne ID, year before 2000',
keep => {
'keep-last' => 1,
},
list => 'novmid',
expected => [
{
'volid' => "$storeid:backup/vzdump-lxc-novmid.tar.gz",
'ctime' => 1234,
'mark' => 'protected',
'type' => 'lxc',
},
],
},
];
plan tests => scalar @$tests;
for my $tt (@$tests) {
my $got = eval {
$current_list = $tt->{list} // 'default';
my $res = PVE::Storage::Plugin->prune_backups($tt->{scfg}, $storeid, $tt->{keep}, $tt->{vmid}, $tt->{type}, 1);
return [ sort { $a->{volid} cmp $b->{volid} } @{$res} ];
};
$got = $@ if $@;
is_deeply($got, $tt->{expected}, $tt->{description}) || diag(explain($got));
}
done_testing();
1;

View File

@ -16,6 +16,7 @@ my $res = $harness->runtests(
"path_to_volume_id_test.pm",
"get_subdir_test.pm",
"filesystem_path_test.pm",
"prune_backups_test.pm",
);
exit -1 if !$res || $res->{failed} || $res->{parse_errors};