From b43845aa0740646c33715030203f509bb99f1ab7 Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Mon, 5 Aug 2024 11:24:10 +0200 Subject: [PATCH] datastore: remove unused data blob writer This is leftover code that is not currently used outside of its own tests. Should we need it again, we can just revert this commit. Signed-off-by: Dominik Csapak --- pbs-datastore/src/data_blob_writer.rs | 212 -------------------------- pbs-datastore/src/lib.rs | 2 - tests/blob_writer.rs | 105 ------------- 3 files changed, 319 deletions(-) delete mode 100644 pbs-datastore/src/data_blob_writer.rs delete mode 100644 tests/blob_writer.rs diff --git a/pbs-datastore/src/data_blob_writer.rs b/pbs-datastore/src/data_blob_writer.rs deleted file mode 100644 index 30d9645fc..000000000 --- a/pbs-datastore/src/data_blob_writer.rs +++ /dev/null @@ -1,212 +0,0 @@ -use std::io::{Seek, SeekFrom, Write}; -use std::sync::Arc; - -use anyhow::Error; - -use proxmox_io::WriteExt; - -use pbs_tools::crypt_config::CryptConfig; - -use crate::checksum_writer::ChecksumWriter; -use crate::crypt_writer::CryptWriter; -use crate::file_formats::{self, DataBlobHeader, EncryptedDataBlobHeader}; - -enum BlobWriterState<'writer, W: Write> { - Uncompressed { - csum_writer: ChecksumWriter, - }, - Compressed { - compr: zstd::stream::write::Encoder<'writer, ChecksumWriter>, - }, - Encrypted { - crypt_writer: CryptWriter>, - }, - EncryptedCompressed { - compr: zstd::stream::write::Encoder<'writer, CryptWriter>>, - }, -} - -/// Data blob writer -pub struct DataBlobWriter<'writer, W: Write> { - state: BlobWriterState<'writer, W>, -} - -impl DataBlobWriter<'_, W> { - pub fn new_uncompressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { - magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0, - crc: [0; 4], - }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - Ok(Self { - state: BlobWriterState::Uncompressed { csum_writer }, - }) - } - - pub fn new_compressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { - magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0, - crc: [0; 4], - }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; - Ok(Self { - state: BlobWriterState::Compressed { compr }, - }) - } - - pub fn new_encrypted(mut writer: W, config: Arc) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0, - crc: [0; 4], - }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - Ok(Self { - state: BlobWriterState::Encrypted { crypt_writer }, - }) - } - - pub fn new_encrypted_compressed( - mut writer: W, - config: Arc, - ) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0, - crc: [0; 4], - }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - let compr = zstd::stream::write::Encoder::new(crypt_writer, 1)?; - Ok(Self { - state: BlobWriterState::EncryptedCompressed { compr }, - }) - } - - pub fn finish(self) -> Result { - match self.state { - BlobWriterState::Uncompressed { csum_writer } => { - // write CRC - let (mut writer, crc, _) = csum_writer.finish()?; - let head = DataBlobHeader { - magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - Ok(writer) - } - BlobWriterState::Compressed { compr } => { - let csum_writer = compr.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = DataBlobHeader { - magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - Ok(writer) - } - BlobWriterState::Encrypted { crypt_writer } => { - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }, - iv, - tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - Ok(writer) - } - BlobWriterState::EncryptedCompressed { compr } => { - let crypt_writer = compr.finish()?; - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }, - iv, - tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - Ok(writer) - } - } - } -} - -impl Write for DataBlobWriter<'_, W> { - fn write(&mut self, buf: &[u8]) -> Result { - match self.state { - BlobWriterState::Uncompressed { - ref mut csum_writer, - } => csum_writer.write(buf), - BlobWriterState::Compressed { ref mut compr } => compr.write(buf), - BlobWriterState::Encrypted { - ref mut crypt_writer, - } => crypt_writer.write(buf), - BlobWriterState::EncryptedCompressed { ref mut compr } => compr.write(buf), - } - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - match self.state { - BlobWriterState::Uncompressed { - ref mut csum_writer, - } => csum_writer.flush(), - BlobWriterState::Compressed { ref mut compr } => compr.flush(), - BlobWriterState::Encrypted { - ref mut crypt_writer, - } => crypt_writer.flush(), - BlobWriterState::EncryptedCompressed { ref mut compr } => compr.flush(), - } - } -} diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 3e4aa34c2..202b09558 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -179,7 +179,6 @@ pub mod crypt_reader; pub mod crypt_writer; pub mod data_blob; pub mod data_blob_reader; -pub mod data_blob_writer; pub mod file_formats; pub mod index; pub mod manifest; @@ -201,7 +200,6 @@ pub use crypt_reader::CryptReader; pub use crypt_writer::CryptWriter; pub use data_blob::DataBlob; pub use data_blob_reader::DataBlobReader; -pub use data_blob_writer::DataBlobWriter; pub use manifest::BackupManifest; pub use store_progress::StoreProgress; diff --git a/tests/blob_writer.rs b/tests/blob_writer.rs deleted file mode 100644 index 23a3283d4..000000000 --- a/tests/blob_writer.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::io::Cursor; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::sync::Arc; - -use anyhow::{bail, Error}; -use lazy_static::lazy_static; - -use pbs_datastore::{DataBlob, DataBlobReader, DataBlobWriter}; -use pbs_tools::crypt_config::CryptConfig; - -lazy_static! { - static ref TEST_DATA: Vec = { - let mut data = Vec::new(); - - for i in 0..100_000 { - data.push((i % 255) as u8); - } - - data - }; - static ref CRYPT_CONFIG: Arc = { - let key = [1u8; 32]; - Arc::new(CryptConfig::new(key).unwrap()) - }; - static ref TEST_DIGEST_PLAIN: [u8; 32] = [ - 83, 154, 96, 195, 167, 204, 38, 142, 204, 224, 130, 201, 24, 71, 2, 188, 130, 155, 177, 6, - 162, 100, 61, 238, 38, 219, 63, 240, 191, 132, 87, 238 - ]; - static ref TEST_DIGEST_ENC: [u8; 32] = [ - 50, 162, 191, 93, 255, 132, 9, 14, 127, 23, 92, 39, 246, 102, 245, 204, 130, 104, 4, 106, - 182, 239, 218, 14, 80, 17, 150, 188, 239, 253, 198, 117 - ]; -} - -fn verify_test_blob(mut cursor: Cursor>, digest: &[u8; 32]) -> Result<(), Error> { - // run read tests with different buffer sizes - for size in [1, 3, 64 * 1024].iter() { - println!("Starting DataBlobReader test (size = {})", size); - - cursor.seek(SeekFrom::Start(0))?; - let mut reader = DataBlobReader::new(&mut cursor, Some(CRYPT_CONFIG.clone()))?; - let mut buffer = Vec::::new(); - // read the whole file - //reader.read_to_end(&mut buffer)?; - let mut buf = vec![0u8; *size]; - loop { - let count = reader.read(&mut buf)?; - if count == 0 { - break; - } - buffer.extend(&buf[..count]); - } - - reader.finish()?; - if buffer != *TEST_DATA { - bail!("blob data is wrong (read buffer size {})", size); - } - } - - let raw_data = cursor.into_inner(); - - let blob = DataBlob::load_from_reader(&mut &raw_data[..])?; - - let data = blob.decode(Some(&CRYPT_CONFIG), Some(digest))?; - if data != *TEST_DATA { - bail!("blob data is wrong (decode)"); - } - Ok(()) -} - -#[test] -fn test_uncompressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_uncompressed(tmp)?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN) -} - -#[test] -fn test_compressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_compressed(tmp)?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN) -} - -#[test] -fn test_encrypted_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_encrypted(tmp, CRYPT_CONFIG.clone())?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC) -} - -#[test] -fn test_encrypted_compressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_encrypted_compressed(tmp, CRYPT_CONFIG.clone())?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC) -}