diff --git a/pbs-datastore/src/data_blob_writer.rs b/pbs-datastore/src/data_blob_writer.rs deleted file mode 100644 index 30d9645fc..000000000 --- a/pbs-datastore/src/data_blob_writer.rs +++ /dev/null @@ -1,212 +0,0 @@ -use std::io::{Seek, SeekFrom, Write}; -use std::sync::Arc; - -use anyhow::Error; - -use proxmox_io::WriteExt; - -use pbs_tools::crypt_config::CryptConfig; - -use crate::checksum_writer::ChecksumWriter; -use crate::crypt_writer::CryptWriter; -use crate::file_formats::{self, DataBlobHeader, EncryptedDataBlobHeader}; - -enum BlobWriterState<'writer, W: Write> { - Uncompressed { - csum_writer: ChecksumWriter, - }, - Compressed { - compr: zstd::stream::write::Encoder<'writer, ChecksumWriter>, - }, - Encrypted { - crypt_writer: CryptWriter>, - }, - EncryptedCompressed { - compr: zstd::stream::write::Encoder<'writer, CryptWriter>>, - }, -} - -/// Data blob writer -pub struct DataBlobWriter<'writer, W: Write> { - state: BlobWriterState<'writer, W>, -} - -impl DataBlobWriter<'_, W> { - pub fn new_uncompressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { - magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0, - crc: [0; 4], - }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - Ok(Self { - state: BlobWriterState::Uncompressed { csum_writer }, - }) - } - - pub fn new_compressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { - magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0, - crc: [0; 4], - }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; - Ok(Self { - state: BlobWriterState::Compressed { compr }, - }) - } - - pub fn new_encrypted(mut writer: W, config: Arc) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0, - crc: [0; 4], - }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - Ok(Self { - state: BlobWriterState::Encrypted { crypt_writer }, - }) - } - - pub fn new_encrypted_compressed( - mut writer: W, - config: Arc, - ) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0, - crc: [0; 4], - }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - let compr = zstd::stream::write::Encoder::new(crypt_writer, 1)?; - Ok(Self { - state: BlobWriterState::EncryptedCompressed { compr }, - }) - } - - pub fn finish(self) -> Result { - match self.state { - BlobWriterState::Uncompressed { csum_writer } => { - // write CRC - let (mut writer, crc, _) = csum_writer.finish()?; - let head = DataBlobHeader { - magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - Ok(writer) - } - BlobWriterState::Compressed { compr } => { - let csum_writer = compr.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = DataBlobHeader { - magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - Ok(writer) - } - BlobWriterState::Encrypted { crypt_writer } => { - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }, - iv, - tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - Ok(writer) - } - BlobWriterState::EncryptedCompressed { compr } => { - let crypt_writer = compr.finish()?; - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }, - iv, - tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - Ok(writer) - } - } - } -} - -impl Write for DataBlobWriter<'_, W> { - fn write(&mut self, buf: &[u8]) -> Result { - match self.state { - BlobWriterState::Uncompressed { - ref mut csum_writer, - } => csum_writer.write(buf), - BlobWriterState::Compressed { ref mut compr } => compr.write(buf), - BlobWriterState::Encrypted { - ref mut crypt_writer, - } => crypt_writer.write(buf), - BlobWriterState::EncryptedCompressed { ref mut compr } => compr.write(buf), - } - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - match self.state { - BlobWriterState::Uncompressed { - ref mut csum_writer, - } => csum_writer.flush(), - BlobWriterState::Compressed { ref mut compr } => compr.flush(), - BlobWriterState::Encrypted { - ref mut crypt_writer, - } => crypt_writer.flush(), - BlobWriterState::EncryptedCompressed { ref mut compr } => compr.flush(), - } - } -} diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 3e4aa34c2..202b09558 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -179,7 +179,6 @@ pub mod crypt_reader; pub mod crypt_writer; pub mod data_blob; pub mod data_blob_reader; -pub mod data_blob_writer; pub mod file_formats; pub mod index; pub mod manifest; @@ -201,7 +200,6 @@ pub use crypt_reader::CryptReader; pub use crypt_writer::CryptWriter; pub use data_blob::DataBlob; pub use data_blob_reader::DataBlobReader; -pub use data_blob_writer::DataBlobWriter; pub use manifest::BackupManifest; pub use store_progress::StoreProgress; diff --git a/tests/blob_writer.rs b/tests/blob_writer.rs deleted file mode 100644 index 23a3283d4..000000000 --- a/tests/blob_writer.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::io::Cursor; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::sync::Arc; - -use anyhow::{bail, Error}; -use lazy_static::lazy_static; - -use pbs_datastore::{DataBlob, DataBlobReader, DataBlobWriter}; -use pbs_tools::crypt_config::CryptConfig; - -lazy_static! { - static ref TEST_DATA: Vec = { - let mut data = Vec::new(); - - for i in 0..100_000 { - data.push((i % 255) as u8); - } - - data - }; - static ref CRYPT_CONFIG: Arc = { - let key = [1u8; 32]; - Arc::new(CryptConfig::new(key).unwrap()) - }; - static ref TEST_DIGEST_PLAIN: [u8; 32] = [ - 83, 154, 96, 195, 167, 204, 38, 142, 204, 224, 130, 201, 24, 71, 2, 188, 130, 155, 177, 6, - 162, 100, 61, 238, 38, 219, 63, 240, 191, 132, 87, 238 - ]; - static ref TEST_DIGEST_ENC: [u8; 32] = [ - 50, 162, 191, 93, 255, 132, 9, 14, 127, 23, 92, 39, 246, 102, 245, 204, 130, 104, 4, 106, - 182, 239, 218, 14, 80, 17, 150, 188, 239, 253, 198, 117 - ]; -} - -fn verify_test_blob(mut cursor: Cursor>, digest: &[u8; 32]) -> Result<(), Error> { - // run read tests with different buffer sizes - for size in [1, 3, 64 * 1024].iter() { - println!("Starting DataBlobReader test (size = {})", size); - - cursor.seek(SeekFrom::Start(0))?; - let mut reader = DataBlobReader::new(&mut cursor, Some(CRYPT_CONFIG.clone()))?; - let mut buffer = Vec::::new(); - // read the whole file - //reader.read_to_end(&mut buffer)?; - let mut buf = vec![0u8; *size]; - loop { - let count = reader.read(&mut buf)?; - if count == 0 { - break; - } - buffer.extend(&buf[..count]); - } - - reader.finish()?; - if buffer != *TEST_DATA { - bail!("blob data is wrong (read buffer size {})", size); - } - } - - let raw_data = cursor.into_inner(); - - let blob = DataBlob::load_from_reader(&mut &raw_data[..])?; - - let data = blob.decode(Some(&CRYPT_CONFIG), Some(digest))?; - if data != *TEST_DATA { - bail!("blob data is wrong (decode)"); - } - Ok(()) -} - -#[test] -fn test_uncompressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_uncompressed(tmp)?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN) -} - -#[test] -fn test_compressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_compressed(tmp)?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN) -} - -#[test] -fn test_encrypted_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_encrypted(tmp, CRYPT_CONFIG.clone())?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC) -} - -#[test] -fn test_encrypted_compressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_encrypted_compressed(tmp, CRYPT_CONFIG.clone())?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC) -}