From 014d36dbbbbe99d31714e21e6af3d0bce8b4cf6b Mon Sep 17 00:00:00 2001 From: Alwin Antreich Date: Tue, 28 Apr 2020 15:58:23 +0200 Subject: [PATCH] Fix: #2124 storage: add zstd support Signed-off-by: Alwin Antreich --- PVE/Storage.pm | 4 +++- PVE/Storage/Plugin.pm | 2 +- test/archive_info_test.pm | 4 +++- test/list_volumes_test.pm | 18 ++++++++++++++++++ test/parse_volname_test.pm | 6 +++--- test/path_to_volume_id_test.pm | 16 ++++++++++++++++ 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/PVE/Storage.pm b/PVE/Storage.pm index 0b2745e..87550b1 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -1366,10 +1366,12 @@ sub decompressor_info { tar => { gz => ['tar', '-z'], lzo => ['tar', '--lzop'], + zst => ['tar', '--zstd'], }, vma => { gz => ['zcat'], lzo => ['lzop', '-d', '-c'], + zst => ['zstd', '-q', '-d', '-c'], }, }; @@ -1460,7 +1462,7 @@ sub extract_vzdump_config_vma { my $errstring; my $err = sub { my $output = shift; - if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: stdout: Broken pipe/) { + if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error : Broken pipe/) { $broken_pipe = 1; } elsif (!defined ($errstring) && $output !~ m/^\s*$/) { $errstring = "Failed to extract config from VMA archive: $output\n"; diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 5f3e4c1..e9da403 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -18,7 +18,7 @@ use JSON; use base qw(PVE::SectionConfig); -use constant COMPRESSOR_RE => 'gz|lzo'; +use constant COMPRESSOR_RE => 'gz|lzo|zst'; our @COMMON_TAR_FLAGS = qw( --one-file-system diff --git a/test/archive_info_test.pm b/test/archive_info_test.pm index c9bb1b7..283fe47 100644 --- a/test/archive_info_test.pm +++ b/test/archive_info_test.pm @@ -45,10 +45,12 @@ my $decompressor = { tar => { gz => ['tar', '-z'], lzo => ['tar', '--lzop'], + zst => ['tar', '--zstd'], }, vma => { gz => ['zcat'], lzo => ['lzop', '-d', '-c'], + zst => ['zstd', '-q', '-d', '-c'], }, }; @@ -85,7 +87,7 @@ foreach my $virt (keys %$bkp_suffix) { my $non_bkp_suffix = { 'openvz' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ], 'lxc' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ], - 'qemu' => [ 'vma.xz', 'vms.gz', '', ], + 'qemu' => [ 'vma.xz', 'vms.gz', 'vmx.zst', '', ], 'none' => [ 'tar.gz', ], }; diff --git a/test/list_volumes_test.pm b/test/list_volumes_test.pm index 941b903..efcb547 100644 --- a/test/list_volumes_test.pm +++ b/test/list_volumes_test.pm @@ -93,6 +93,7 @@ my @tests = ( "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma", + "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst", "$storage_dir/snippets/userconfig.yaml", "$storage_dir/snippets/hookscript.pl", ], @@ -151,6 +152,14 @@ my @tests = ( 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma', }, + { + 'content' => 'backup', + 'ctime' => 1585595635, + 'format' => 'vma.zst', + 'size' => DEFAULT_SIZE, + 'vmid' => '16110', + 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst', + }, { 'content' => 'snippets', 'ctime' => DEFAULT_CTIME, @@ -174,6 +183,7 @@ my @tests = ( "$storage_dir/images/16112/vm-16112-disk-0.raw", "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo", "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz", + "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst", "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz", ], expected => [ @@ -203,6 +213,14 @@ my @tests = ( 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz', }, + { + 'content' => 'backup', + 'ctime' => 1585597770, + 'format' => 'tar.zst', + 'size' => DEFAULT_SIZE, + 'vmid' => '16112', + 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst', + }, { 'content' => 'backup', 'ctime' => 1585598370, diff --git a/test/parse_volname_test.pm b/test/parse_volname_test.pm index 87c758c..d6ac885 100644 --- a/test/parse_volname_test.pm +++ b/test/parse_volname_test.pm @@ -176,9 +176,9 @@ foreach my $s (@$disk_suffix) { # create more test cases for backup files matches my $bkp_suffix = { - qemu => [ 'vma', 'vma.gz', 'vma.lzo' ], - lxc => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo' ], - openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo' ], + qemu => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst' ], + lxc => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ], + openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ], }; foreach my $virt (keys %$bkp_suffix) { diff --git a/test/path_to_volume_id_test.pm b/test/path_to_volume_id_test.pm index e5e24c1..5eee2f6 100644 --- a/test/path_to_volume_id_test.pm +++ b/test/path_to_volume_id_test.pm @@ -100,6 +100,22 @@ my @tests = ( 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', ], }, + { + description => 'Backup, vma.zst', + volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst", + expected => [ + 'backup', + 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst' + ], + }, + { + description => 'Backup, tar.zst', + volname => "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst", + expected => [ + 'backup', + 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst' + ], + }, { description => 'ISO file',