tsarevvv/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

datastore: remove unused data blob writer

Browse Source 
This is leftover code that is not currently used outside of its own
tests.

Should we need it again, we can just revert this commit.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
This commit is contained in:
Dominik Csapak 2024-08-05 11:24:10 +02:00 committed by Thomas Lamprecht
parent 42e5be0f87
commit b43845aa07
3 changed files with 0 additions and 319 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
pbs-datastore/src/data_blob_writer.rs
View File

@ -1,212 +0,0 @@
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
use anyhow::Error;
use proxmox_io::WriteExt;
use pbs_tools::crypt_config::CryptConfig;
use crate::checksum_writer::ChecksumWriter;
use crate::crypt_writer::CryptWriter;
use crate::file_formats::{self, DataBlobHeader, EncryptedDataBlobHeader};
enum BlobWriterState<'writer, W: Write> {
Uncompressed {
csum_writer: ChecksumWriter<W>,
},
Compressed {
compr: zstd::stream::write::Encoder<'writer, ChecksumWriter<W>>,
},
Encrypted {
crypt_writer: CryptWriter<ChecksumWriter<W>>,
},
EncryptedCompressed {
compr: zstd::stream::write::Encoder<'writer, CryptWriter<ChecksumWriter<W>>>,
},
}
/// Data blob writer
pub struct DataBlobWriter<'writer, W: Write> {
state: BlobWriterState<'writer, W>,
}
impl<W: Write + Seek> DataBlobWriter<'_, W> {
pub fn new_uncompressed(mut writer: W) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = DataBlobHeader {
magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0,
crc: [0; 4],
};
unsafe {
writer.write_le_value(head)?;
}
let csum_writer = ChecksumWriter::new(writer, None);
Ok(Self {
state: BlobWriterState::Uncompressed { csum_writer },
})
}
pub fn new_compressed(mut writer: W) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = DataBlobHeader {
magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0,
crc: [0; 4],
};
unsafe {
writer.write_le_value(head)?;
}
let csum_writer = ChecksumWriter::new(writer, None);
let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?;
Ok(Self {
state: BlobWriterState::Compressed { compr },
})
}
pub fn new_encrypted(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = EncryptedDataBlobHeader {
head: DataBlobHeader {
magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0,
crc: [0; 4],
},
iv: [0u8; 16],
tag: [0u8; 16],
};
unsafe {
writer.write_le_value(head)?;
}
let csum_writer = ChecksumWriter::new(writer, None);
let crypt_writer = CryptWriter::new(csum_writer, config)?;
Ok(Self {
state: BlobWriterState::Encrypted { crypt_writer },
})
}
pub fn new_encrypted_compressed(
mut writer: W,
config: Arc<CryptConfig>,
) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = EncryptedDataBlobHeader {
head: DataBlobHeader {
magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0,
crc: [0; 4],
},
iv: [0u8; 16],
tag: [0u8; 16],
};
unsafe {
writer.write_le_value(head)?;
}
let csum_writer = ChecksumWriter::new(writer, None);
let crypt_writer = CryptWriter::new(csum_writer, config)?;
let compr = zstd::stream::write::Encoder::new(crypt_writer, 1)?;
Ok(Self {
state: BlobWriterState::EncryptedCompressed { compr },
})
}
pub fn finish(self) -> Result<W, Error> {
match self.state {
BlobWriterState::Uncompressed { csum_writer } => {
// write CRC
let (mut writer, crc, _) = csum_writer.finish()?;
let head = DataBlobHeader {
magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0,
crc: crc.to_le_bytes(),
};
writer.seek(SeekFrom::Start(0))?;
unsafe {
writer.write_le_value(head)?;
}
Ok(writer)
}
BlobWriterState::Compressed { compr } => {
let csum_writer = compr.finish()?;
let (mut writer, crc, _) = csum_writer.finish()?;
let head = DataBlobHeader {
magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0,
crc: crc.to_le_bytes(),
};
writer.seek(SeekFrom::Start(0))?;
unsafe {
writer.write_le_value(head)?;
}
Ok(writer)
}
BlobWriterState::Encrypted { crypt_writer } => {
let (csum_writer, iv, tag) = crypt_writer.finish()?;
let (mut writer, crc, _) = csum_writer.finish()?;
let head = EncryptedDataBlobHeader {
head: DataBlobHeader {
magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0,
crc: crc.to_le_bytes(),
},
iv,
tag,
};
writer.seek(SeekFrom::Start(0))?;
unsafe {
writer.write_le_value(head)?;
}
Ok(writer)
}
BlobWriterState::EncryptedCompressed { compr } => {
let crypt_writer = compr.finish()?;
let (csum_writer, iv, tag) = crypt_writer.finish()?;
let (mut writer, crc, _) = csum_writer.finish()?;
let head = EncryptedDataBlobHeader {
head: DataBlobHeader {
magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0,
crc: crc.to_le_bytes(),
},
iv,
tag,
};
writer.seek(SeekFrom::Start(0))?;
unsafe {
writer.write_le_value(head)?;
}
Ok(writer)
}
}
}
}
impl<W: Write + Seek> Write for DataBlobWriter<'_, W> {
fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
match self.state {
BlobWriterState::Uncompressed {
ref mut csum_writer,
} => csum_writer.write(buf),
BlobWriterState::Compressed { ref mut compr } => compr.write(buf),
BlobWriterState::Encrypted {
ref mut crypt_writer,
} => crypt_writer.write(buf),
BlobWriterState::EncryptedCompressed { ref mut compr } => compr.write(buf),
}
}
fn flush(&mut self) -> Result<(), std::io::Error> {
match self.state {
BlobWriterState::Uncompressed {
ref mut csum_writer,
} => csum_writer.flush(),
BlobWriterState::Compressed { ref mut compr } => compr.flush(),
BlobWriterState::Encrypted {
ref mut crypt_writer,
} => crypt_writer.flush(),
BlobWriterState::EncryptedCompressed { ref mut compr } => compr.flush(),
}
}
}

2
pbs-datastore/src/lib.rs
View File

@ -179,7 +179,6 @@ pub mod crypt_reader;
pub mod crypt_writer; pub mod crypt_writer;
pub mod data_blob; pub mod data_blob;
pub mod data_blob_reader; pub mod data_blob_reader;
pub mod data_blob_writer;
pub mod file_formats; pub mod file_formats;
pub mod index; pub mod index;
pub mod manifest; pub mod manifest;
@ -201,7 +200,6 @@ pub use crypt_reader::CryptReader;
pub use crypt_writer::CryptWriter; pub use crypt_writer::CryptWriter;
pub use data_blob::DataBlob; pub use data_blob::DataBlob;
pub use data_blob_reader::DataBlobReader; pub use data_blob_reader::DataBlobReader;
pub use data_blob_writer::DataBlobWriter;
pub use manifest::BackupManifest; pub use manifest::BackupManifest;
pub use store_progress::StoreProgress; pub use store_progress::StoreProgress;

105
tests/blob_writer.rs
View File

@ -1,105 +0,0 @@
use std::io::Cursor;
use std::io::{Read, Seek, SeekFrom, Write};
use std::sync::Arc;
use anyhow::{bail, Error};
use lazy_static::lazy_static;
use pbs_datastore::{DataBlob, DataBlobReader, DataBlobWriter};
use pbs_tools::crypt_config::CryptConfig;
lazy_static! {
static ref TEST_DATA: Vec<u8> = {
let mut data = Vec::new();
for i in 0..100_000 {
data.push((i % 255) as u8);
}
data
};
static ref CRYPT_CONFIG: Arc<CryptConfig> = {
let key = [1u8; 32];
Arc::new(CryptConfig::new(key).unwrap())
};
static ref TEST_DIGEST_PLAIN: [u8; 32] = [
83, 154, 96, 195, 167, 204, 38, 142, 204, 224, 130, 201, 24, 71, 2, 188, 130, 155, 177, 6,
162, 100, 61, 238, 38, 219, 63, 240, 191, 132, 87, 238
];
static ref TEST_DIGEST_ENC: [u8; 32] = [
50, 162, 191, 93, 255, 132, 9, 14, 127, 23, 92, 39, 246, 102, 245, 204, 130, 104, 4, 106,
182, 239, 218, 14, 80, 17, 150, 188, 239, 253, 198, 117
];
}
fn verify_test_blob(mut cursor: Cursor<Vec<u8>>, digest: &[u8; 32]) -> Result<(), Error> {
// run read tests with different buffer sizes
for size in [1, 3, 64 * 1024].iter() {
println!("Starting DataBlobReader test (size = {})", size);
cursor.seek(SeekFrom::Start(0))?;
let mut reader = DataBlobReader::new(&mut cursor, Some(CRYPT_CONFIG.clone()))?;
let mut buffer = Vec::<u8>::new();
// read the whole file
//reader.read_to_end(&mut buffer)?;
let mut buf = vec![0u8; *size];
loop {
let count = reader.read(&mut buf)?;
if count == 0 {
break;
}
buffer.extend(&buf[..count]);
}
reader.finish()?;
if buffer != *TEST_DATA {
bail!("blob data is wrong (read buffer size {})", size);
}
}
let raw_data = cursor.into_inner();
let blob = DataBlob::load_from_reader(&mut &raw_data[..])?;
let data = blob.decode(Some(&CRYPT_CONFIG), Some(digest))?;
if data != *TEST_DATA {
bail!("blob data is wrong (decode)");
}
Ok(())
}
#[test]
fn test_uncompressed_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_uncompressed(tmp)?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN)
}
#[test]
fn test_compressed_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_compressed(tmp)?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN)
}
#[test]
fn test_encrypted_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_encrypted(tmp, CRYPT_CONFIG.clone())?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC)
}
#[test]
fn test_encrypted_compressed_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_encrypted_compressed(tmp, CRYPT_CONFIG.clone())?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC)
}