Proxmox/proxmox
5
0
Fork 1
Code Issues Pull Requests Releases Activity

add proxmox-io crate

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2021-10-06 14:52:59 +02:00
parent 91f59f9f59
commit 01a8b6f1bf
15 changed files with 196 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@ -4,6 +4,7 @@ members = [
"proxmox-api-macro",
"proxmox-borrow",
"proxmox-http",
"proxmox-io",
"proxmox-lang",
"proxmox-sortable-macro",
"proxmox-tfa",

1
Makefile
View File

@ -5,6 +5,7 @@ CRATES = \
proxmox-api-macro \
proxmox-borrow \
proxmox-http \
proxmox-io \
proxmox-lang \
proxmox-sortable-macro \
proxmox-tfa \

14
proxmox-io/Cargo.toml Normal file
View File

@ -0,0 +1,14 @@
[package]
name = "proxmox-io"
version = "1.0.0"
authors = ["Proxmox Support Team <support@proxmox.com>"]
edition = "2018"
license = "AGPL-3"
description = "extension traits for Read and Write"
exclude = [ "debian" ]
[dependencies]
endian_trait = { version = "0.6", features = ["arrays"] }
# this is also the feature name:
tokio = { version = "1.0", features = [ "io-util" ], optional = true }

5
proxmox-io/debian/changelog Normal file
View File

@ -0,0 +1,5 @@
rust-proxmox-io (1.0.0-1) stable; urgency=medium
* initial split out of `librust-proxmox-dev`
-- Proxmox Support Team <support@proxmox.com> Wed, 06 Oct 2021 11:04:36 +0200

52
proxmox-io/debian/control Normal file
View File

@ -0,0 +1,52 @@
Source: rust-proxmox-io
Section: rust
Priority: optional
Build-Depends: debhelper (>= 12),
dh-cargo (>= 24),
cargo:native <!nocheck>,
rustc:native <!nocheck>,
libstd-rust-dev <!nocheck>,
librust-endian-trait-0.6+arrays-dev <!nocheck>,
librust-endian-trait-0.6+default-dev <!nocheck>
Maintainer: Proxmox Support Team <support@proxmox.com>
Standards-Version: 4.5.1
Vcs-Git: git://git.proxmox.com/git/proxmox.git
Vcs-Browser: https://git.proxmox.com/?p=proxmox.git
Rules-Requires-Root: no
Package: librust-proxmox-io-dev
Architecture: any
Multi-Arch: same
Depends:
${misc:Depends},
librust-endian-trait-0.6+arrays-dev,
librust-endian-trait-0.6+default-dev
Suggests:
librust-proxmox-io+tokio-dev (= ${binary:Version})
Provides:
librust-proxmox-io+default-dev (= ${binary:Version}),
librust-proxmox-io-1-dev (= ${binary:Version}),
librust-proxmox-io-1+default-dev (= ${binary:Version}),
librust-proxmox-io-1.0-dev (= ${binary:Version}),
librust-proxmox-io-1.0+default-dev (= ${binary:Version}),
librust-proxmox-io-1.0.0-dev (= ${binary:Version}),
librust-proxmox-io-1.0.0+default-dev (= ${binary:Version})
Description: Extension traits for Read and Write - Rust source code
This package contains the source for the Rust proxmox-io crate, packaged by
debcargo for use with cargo and dh-cargo.
Package: librust-proxmox-io+tokio-dev
Architecture: any
Multi-Arch: same
Depends:
${misc:Depends},
librust-proxmox-io-dev (= ${binary:Version}),
librust-tokio-1+default-dev,
librust-tokio-1+io-util-dev
Provides:
librust-proxmox-io-1+tokio-dev (= ${binary:Version}),
librust-proxmox-io-1.0+tokio-dev (= ${binary:Version}),
librust-proxmox-io-1.0.0+tokio-dev (= ${binary:Version})
Description: Extension traits for Read and Write - feature "tokio"
This metapackage enables feature "tokio" for the Rust proxmox-io crate, by
pulling in any additional dependencies needed by that feature.

16
proxmox-io/debian/copyright Normal file
View File

@ -0,0 +1,16 @@
Copyright (C) 2021 Proxmox Server Solutions GmbH
This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

7
proxmox-io/debian/debcargo.toml Normal file
View File

@ -0,0 +1,7 @@
overlay = "."
crate_src_path = ".."
maintainer = "Proxmox Support Team <support@proxmox.com>"
[source]
vcs_git = "git://git.proxmox.com/git/proxmox.git"
vcs_browser = "https://git.proxmox.com/?p=proxmox.git"

16
proxmox/src/tools/byte_buffer.rs → proxmox-io/src/byte_buffer.rs
View File

@ -5,7 +5,7 @@
//! Example:
//! ```
//! # use std::io::Read;
//! # use proxmox::tools::byte_buffer::ByteBuffer;
//! # use proxmox_io::ByteBuffer;
//! fn code<T: Read + ?Sized>(input: &mut T) -> std::io::Result<Box<[u8]>> {
//! let mut buffer = ByteBuffer::new();
//! let amount = buffer.read_from(input)?;
@ -18,7 +18,7 @@
use std::io::{Read, Result};
use crate::tools::vec;
use crate::vec;
#[cfg(feature = "tokio")]
use tokio::io::{AsyncRead, AsyncReadExt};
@ -66,7 +66,7 @@ impl ByteBuffer {
///
/// Example:
/// ```
/// # use proxmox::tools::byte_buffer::ByteBuffer;
/// # use proxmox_io::ByteBuffer;
/// let mut buf = ByteBuffer::new();
/// buf.get_free_mut_slice()[..1].copy_from_slice(&[1u8]);
/// buf.add_size(1);
@ -74,7 +74,7 @@ impl ByteBuffer {
///
/// This code will panic:
/// ```should_panic
/// # use proxmox::tools::byte_buffer::ByteBuffer;
/// # use proxmox_io::ByteBuffer;
/// let mut buf = ByteBuffer::with_capacity(128);
/// buf.add_size(256);
/// ```
@ -101,7 +101,7 @@ impl ByteBuffer {
///
/// Example:
/// ```
/// # use proxmox::tools::byte_buffer::ByteBuffer;
/// # use proxmox_io::ByteBuffer;
/// let mut buf = ByteBuffer::new();
/// buf.get_free_mut_slice()[..2].copy_from_slice(&[1u8, 2u8]);
/// buf.add_size(2);
@ -126,7 +126,7 @@ impl ByteBuffer {
///
/// Example:
/// ```
/// # use proxmox::tools::byte_buffer::ByteBuffer;
/// # use proxmox_io::ByteBuffer;
/// let mut buf = ByteBuffer::new();
/// buf.get_free_mut_slice()[..2].copy_from_slice(&[1u8, 2u8]);
/// buf.add_size(2);
@ -155,7 +155,7 @@ impl ByteBuffer {
/// Example:
/// ```
/// # use std::io;
/// # use proxmox::tools::byte_buffer::ByteBuffer;
/// # use proxmox_io::ByteBuffer;
/// fn code<R: io::Read>(mut reader: R) -> io::Result<()> {
/// let mut buf = ByteBuffer::new();
/// let res = buf.read_from(&mut reader)?;
@ -201,7 +201,7 @@ impl std::ops::DerefMut for ByteBuffer {
#[cfg(test)]
mod test {
use crate::tools::byte_buffer::ByteBuffer;
use super::ByteBuffer;
#[test]
fn test1() {

21
proxmox-io/src/lib.rs Normal file
View File

@ -0,0 +1,21 @@
//! Module providing I/O helpers (sync and async).
//!
//! The [`ReadExt`] trait provides additional operations for handling byte buffers for types
//! implementing [`Read`](std::io::Read).
mod read;
pub use read::ReadExt;
mod write;
pub use write::WriteExt;
mod sparse_copy;
pub use sparse_copy::{buffer_is_zero, sparse_copy, SparseCopyResult};
#[cfg(feature = "tokio")]
pub use sparse_copy::sparse_copy_async;
mod byte_buffer;
pub use byte_buffer::ByteBuffer;
pub mod vec;

20
proxmox/src/tools/io/read.rs → proxmox-io/src/read.rs
View File

@ -5,7 +5,7 @@ use std::mem;
use endian_trait::Endian;
use crate::tools::vec::{self, ByteVecExt};
use crate::vec::{self, ByteVecExt};
/// Adds some additional related functionality for types implementing [`Read`](std::io::Read).
///
@ -14,7 +14,7 @@ use crate::tools::vec::{self, ByteVecExt};
///
/// Examples:
/// ```no_run
/// use proxmox::tools::io::ReadExt;
/// use proxmox_io::ReadExt;
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("some.data")?;
@ -33,7 +33,7 @@ use crate::tools::vec::{self, ByteVecExt};
///
/// ```no_run
/// # use endian_trait::Endian;
/// # use proxmox::tools::io::ReadExt;
/// # use proxmox_io::ReadExt;
///
/// #[derive(Endian)]
/// #[repr(C)]
@ -63,7 +63,7 @@ pub trait ReadExt {
///
/// With this trait, we just use:
/// ```no_run
/// use proxmox::tools::io::ReadExt;
/// use proxmox_io::ReadExt;
/// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> {
/// let data = reader.read_exact_allocated(len)?;
/// # Ok(())
@ -87,7 +87,7 @@ pub trait ReadExt {
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::ReadExt;
/// use proxmox_io::ReadExt;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
@ -124,7 +124,7 @@ pub trait ReadExt {
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::ReadExt;
/// use proxmox_io::ReadExt;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
@ -161,7 +161,7 @@ pub trait ReadExt {
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::ReadExt;
/// use proxmox_io::ReadExt;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
@ -194,8 +194,8 @@ pub trait ReadExt {
/// immediately on all values.
///
/// ```
/// # use proxmox::tools::vec;
/// use proxmox::tools::io::ReadExt;
/// # use proxmox_io::vec;
/// use proxmox_io::ReadExt;
///
/// #[repr(C)]
/// struct Data {
@ -204,7 +204,7 @@ pub trait ReadExt {
/// }
///
/// # let mut input = [0u8; 4096];
/// # use proxmox::tools::io::WriteExt;
/// # use proxmox_io::WriteExt;
/// # let mut writer = &mut input[..];
/// # unsafe { writer.write_host_value(32u64).unwrap() };
/// # let mut file = &input[..];

84
proxmox/src/tools/io/mod.rs → proxmox-io/src/sparse_copy.rs
View File

@ -1,37 +1,33 @@
//! Module providing I/O helpers (sync and async).
//!
//! The [`ReadExt`] trait provides additional operations for handling byte buffers for types
//! implementing [`Read`](std::io::Read).
use std::io::{self, ErrorKind, Read, Seek, SeekFrom, Write};
mod read;
pub use read::*;
#[cfg(feature = "tokio")]
use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
mod write;
pub use write::*;
fn buffer_is_zero(buf: &[u8]) -> bool {
/// Efficiently check whether a byte slice contains only zeroes.
///
/// This is implemented in a way which allows the compiler to utilize SIMD instructions.
pub fn buffer_is_zero(buf: &[u8]) -> bool {
!buf.chunks(128)
.map(|aa| aa.iter().fold(0, |a, b| a | b) != 0)
.any(|a| a)
}
/// Result of a sparse copy call
/// contains the amount of written/seeked bytes
/// and if the last operation was a seek
/// Result of a sparse copy call.
///
/// Contains the amount of written/seeked bytes and if the last operation was a seek.
#[must_use = "if sparse_copy ended with a seek(), the output file's size may need to be adjusted"]
pub struct SparseCopyResult {
pub written: u64,
pub seeked_last: bool,
}
/// copy similar to io::copy, but seeks the target when encountering
/// zero bytes instead of writing them
/// Copy similar to [`io::copy`](std::io::copy), but seeks the target when encountering zero bytes
/// instead of writing them.
///
/// Example use:
/// ```
/// # use std::io;
/// # use proxmox::tools::io::sparse_copy;
/// # use proxmox_io::sparse_copy;
/// fn code<R, W>(mut reader: R, mut writer: W) -> io::Result<()>
/// where
/// R: io::Read,
@ -49,7 +45,7 @@ pub fn sparse_copy<R: Read + ?Sized, W: Write + Seek + ?Sized>(
reader: &mut R,
writer: &mut W,
) -> Result<SparseCopyResult, io::Error> {
let mut buf = crate::tools::byte_buffer::ByteBuffer::with_capacity(4096);
let mut buf = crate::byte_buffer::ByteBuffer::with_capacity(4096);
let mut written = 0;
let mut seek_amount: i64 = 0;
let mut seeked_last = false;
@ -86,10 +82,6 @@ pub fn sparse_copy<R: Read + ?Sized, W: Write + Seek + ?Sized>(
}
}
#[cfg(feature = "tokio")]
use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
#[cfg(feature = "tokio")]
/// copy similar to tokio::io::copy, but seeks the target when encountering
/// zero bytes instead of writing them
///
@ -97,7 +89,7 @@ use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
/// ```no_run
/// # use std::io;
/// # use tokio::io::{AsyncRead, AsyncWrite, AsyncSeek};
/// # use proxmox::tools::io::sparse_copy_async;
/// # use proxmox_io::sparse_copy_async;
/// async fn code<R, W>(mut reader: R, mut writer: W) -> io::Result<()>
/// where
/// R: AsyncRead + Unpin,
@ -111,6 +103,7 @@ use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
/// Ok(())
/// }
/// ```
#[cfg(feature = "tokio")]
pub async fn sparse_copy_async<R, W>(
reader: &mut R,
writer: &mut W,
@ -119,7 +112,7 @@ where
R: AsyncRead + Unpin,
W: AsyncWrite + AsyncSeek + Unpin,
{
let mut buf = crate::tools::byte_buffer::ByteBuffer::with_capacity(4096);
let mut buf = crate::byte_buffer::ByteBuffer::with_capacity(4096);
let mut written = 0;
let mut seek_amount: i64 = 0;
let mut seeked_last = false;
@ -160,8 +153,7 @@ where
mod test {
use std::io::Cursor;
use crate::test::io::{AsyncBlockingReader, AsyncBlockingWriter};
use crate::tools::io::{sparse_copy, sparse_copy_async};
use super::sparse_copy;
const LEN: usize = 10000;
@ -203,9 +195,23 @@ mod test {
}
}
#[cfg(feature = "tokio")]
#[test]
fn test_sparse_copy_async() {
let fut = async {
use std::future::Future;
use std::sync::Arc;
use std::task::{Context, Poll};
use super::sparse_copy_async;
struct PanicWaker;
impl std::task::Wake for PanicWaker {
fn wake(self: Arc<Self>) {
panic!("this test should not block");
}
}
let mut fut = async {
// test sparse
let mut test_data = Vec::new();
for _ in 0..LEN / 2 {
@ -214,8 +220,8 @@ mod test {
for _ in 0..LEN / 2 {
test_data.push(0u8);
}
let mut test_data = AsyncBlockingReader::new(Cursor::new(test_data));
let mut result_data = AsyncBlockingWriter::new(Cursor::new(vec![0; LEN]));
let mut test_data = Cursor::new(test_data);
let mut result_data = Cursor::new(vec![0; LEN]);
let result = sparse_copy_async(&mut test_data, &mut result_data)
.await
@ -225,15 +231,15 @@ mod test {
assert_eq!(result.seeked_last, true);
for i in 0..LEN {
if i < LEN / 2 {
assert_eq!(result_data.inner().get_ref()[i], 1);
assert_eq!(result_data.get_ref()[i], 1);
} else {
assert_eq!(result_data.inner().get_ref()[i], 0);
assert_eq!(result_data.get_ref()[i], 0);
}
}
// test non sparse
let mut test_data = AsyncBlockingReader::new(Cursor::new(vec![1; LEN]));
let mut result_data = AsyncBlockingWriter::new(Cursor::new(vec![0; LEN]));
let mut test_data = Cursor::new(vec![1; LEN]);
let mut result_data = Cursor::new(vec![0; LEN]);
let result = sparse_copy_async(&mut test_data, &mut result_data)
.await
@ -242,11 +248,17 @@ mod test {
assert_eq!(result.written, LEN as u64);
assert_eq!(result.seeked_last, false);
for i in 0..LEN {
assert_eq!(result_data.inner().get_ref()[i], 1);
assert_eq!(result_data.get_ref()[i], 1);
}
Ok(())
Ok::<(), std::io::Error>(())
};
crate::test::task::poll_result_once(fut).expect("ok")
let fut = unsafe { std::pin::Pin::new_unchecked(&mut fut) };
let waker = std::task::Waker::from(Arc::new(PanicWaker));
let mut context = Context::from_waker(&waker);
match fut.poll(&mut context) {
Poll::Ready(res) => res.expect("expected ok"),
Poll::Pending => panic!("yielded on blocking implementation"),
}
}
}

12
proxmox/src/tools/vec/byte_vec.rs → proxmox-io/src/vec/byte_vec.rs
View File

@ -3,7 +3,7 @@
//! Example:
//! ```
//! # use std::io::Read;
//! use proxmox::tools::vec::{self, ByteVecExt};
//! use proxmox_io::vec::{self, ByteVecExt};
//!
//! fn append_1024_to_vec<T: Read>(mut input: T, buffer: &mut Vec<u8>) -> std::io::Result<()> {
//! input.read_exact(unsafe { buffer.grow_uninitialized(1024) })
@ -14,8 +14,8 @@
/// Example:
/// ```
/// # use std::io::Read;
/// # use proxmox::tools::io::{self, ReadExt};
/// use proxmox::tools::vec::{self, ByteVecExt};
/// # use proxmox_io::ReadExt;
/// use proxmox_io::vec::{self, ByteVecExt};
///
/// # fn code(mut file: std::fs::File, mut data: Vec<u8>) -> std::io::Result<()> {
/// file.read_exact(unsafe {
@ -32,7 +32,7 @@
/// file.append_to_vec(&mut data, 1024)?;
/// ```
///
/// [`ReadExt`]: crate::tools::io::ReadExt
/// [`ReadExt`]: crate::ReadExt
pub trait ByteVecExt {
/// Grow a vector without initializing its elements. The difference to simply using `reserve`
/// is that it also updates the actual length, making the newly allocated data part of the
@ -50,7 +50,7 @@ pub trait ByteVecExt {
/// This returns a mutable slice to the newly allocated space, so it can be used inline:
/// ```
/// # use std::io::Read;
/// # use proxmox::tools::vec::ByteVecExt;
/// # use proxmox_io::vec::ByteVecExt;
/// # fn test(mut file: std::fs::File, buffer: &mut Vec<u8>) -> std::io::Result<()> {
/// file.read_exact(unsafe { buffer.grow_uninitialized(1024) })?;
/// # Ok(())
@ -70,7 +70,7 @@ pub trait ByteVecExt {
/// the previously contained content. Since we cannot track this state through the type system,
/// this method is marked as an unsafe API for good measure.
///
/// [`ReadExt`]: crate::tools::io::ReadExt
/// [`ReadExt`]: crate::ReadExt
unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8];
/// Resize a vector to a specific size without initializing its data. This is a shortcut for:

6
proxmox/src/tools/vec/mod.rs → proxmox-io/src/vec/mod.rs
View File

@ -13,11 +13,11 @@
//!
//! This module provides some helpers for this kind of code. Many of these are supposed to stay on
//! a lower level, with I/O helpers for types implementing [`Read`](std::io::Read) being available
//! in the [`tools::io`](crate::tools::io) module.
//! in this module.
//!
//! Examples:
//! ```no_run
//! use proxmox::tools::vec::{self, ByteVecExt};
//! use proxmox_io::vec::{self, ByteVecExt};
//!
//! # let size = 64usize;
//! # let more = 64usize;
@ -32,7 +32,7 @@
//! ```
mod byte_vec;
pub use byte_vec::*;
pub use byte_vec::ByteVecExt;
/// Create an uninitialized byte vector of a specific size.
///

8
proxmox/src/tools/io/write.rs → proxmox-io/src/write.rs
View File

@ -11,7 +11,7 @@ use endian_trait::Endian;
/// Examples:
/// ```no_run
/// # use endian_trait::Endian;
/// # use proxmox::tools::io::WriteExt;
/// # use proxmox_io::WriteExt;
///
/// #[derive(Endian)]
/// #[repr(C)]
@ -47,7 +47,7 @@ pub trait WriteExt {
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::WriteExt;
/// use proxmox_io::WriteExt;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
@ -90,7 +90,7 @@ pub trait WriteExt {
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::WriteExt;
/// use proxmox_io::WriteExt;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
@ -133,7 +133,7 @@ pub trait WriteExt {
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::WriteExt;
/// use proxmox_io::WriteExt;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]

3
proxmox/src/tools/mod.rs
View File

@ -6,16 +6,13 @@ use anyhow::*;
use lazy_static::lazy_static;
pub mod as_any;
pub mod byte_buffer;
pub mod common_regex;
pub mod email;
pub mod fd;
pub mod fs;
pub mod io;
pub mod mmap;
pub mod parse;
pub mod serde;
pub mod vec;
pub mod systemd;
#[doc(inline)]