Proxmox/proxmox
5
0
Fork 1
Code Issues Pull Requests Releases Activity

initial import, starting with vec & io helpers

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2019-06-06 13:47:38 +02:00
commit 2e6520a987
12 changed files with 503 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
Cargo.lock
/target
**/*.rs.bk

5
Cargo.toml Normal file
View File

@ -0,0 +1,5 @@
[workspace]
members = [
"proxmox-tools",
"proxmox",
]

21
proxmox-tools/Cargo.toml Normal file
View File

@ -0,0 +1,21 @@
[package]
name = "proxmox-tools"
edition = "2018"
version = "0.1.0"
authors = [
"Dietmar Maurer <dietmar@proxmox.com>",
"Wolfgang Bumiller <w.bumiller@proxmox.com>",
]
[dependencies]
endian_trait = { version = "0.6", features = ["arrays"] }
libc = "0.2"
valgrind_request = { version = "1.1.0", optional = true }
[features]
default = [ "valgrind" ]
valgrind = [ "valgrind_request" ]
# Docs should be able to reference the proxmox crate.
[dev-dependencies]
proxmox = { path = "../proxmox" }

10
proxmox-tools/src/io.rs Normal file
View File

@ -0,0 +1,10 @@
//! Module providing I/O helpers (sync and async).
//!
//! The [`ops`](io::ops) module provides helper traits for types implementing [`Read`](std::io::Read).
//!
//! The top level functions in of this module here are used for standalone implementations of
//! various functionality which is actually intended to be available as methods to types
//! implementing `AsyncRead`, which, however, without async/await cannot be methods due to them
//! having non-static lifetimes in that case.
pub mod ops;

226
proxmox-tools/src/io/ops.rs Normal file
View File

@ -0,0 +1,226 @@
//! This module provides additional operations for handling byte buffers for types implementing
//! [`Read`](std::io::Read).
//!
//! See the [`ReadExtOps`](ops::ReadExtOps) trait for examples.
use std::io;
use endian_trait::Endian;
use crate::vec::{self, ops::*};
/// Adds some additional related functionality for types implementing [`Read`](std::io::Read).
///
/// Particularly for reading into a newly allocated buffer, appending to a `Vec<u8>` or reading
/// values of a specific endianess (types implementing [`Endian`]).
///
/// Examples:
/// ```no_run
/// use proxmox::tools::io::ops::*;
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("some.data")?;
///
/// // read some bytes into a newly allocated Vec<u8>:
/// let mut data = file.read_exact_allocated(64)?;
///
/// // appending data to a vector:
/// let actually_appended = file.append_to_vec(&mut data, 64)?; // .read() version
/// file.append_exact_to_vec(&mut data, 64)?; // .read_exact() version
/// # Ok(())
/// # }
/// ```
///
/// Or for reading values of a defined representation and endianess:
///
/// ```no_run
/// # use endian_trait::Endian;
/// # use proxmox::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C)]
/// struct Header {
/// version: u16,
/// data_size: u16,
/// }
///
/// # fn code(mut file: std::fs::File) -> std::io::Result<()> {
/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe:
/// let header: Header = unsafe { file.read_le_value()? };
/// let mut blob = file.read_exact_allocated(header.data_size as usize)?;
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
pub trait ReadExtOps {
/// Read data into a newly allocated vector. This is a shortcut for:
/// ```ignore
/// let mut data = Vec::with_capacity(len);
/// unsafe {
/// data.set_len(len);
/// }
/// reader.read_exact(&mut data)?;
/// ```
///
/// With this trait, we just use:
/// ```no_run
/// use proxmox::tools::io::ops::*;
/// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> {
/// let data = reader.read_exact_allocated(len)?;
/// # Ok(())
/// # }
/// ```
fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>>;
/// Append data to a vector, growing it as necessary. Returns the amount of data appended.
fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize>;
/// Append an exact amount of data to a vector, growing it as necessary.
fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()>;
/// Read a value with host endianess.
///
/// This is limited to types implementing the [`Endian`] trait under the assumption that
/// this is only done for types which are supposed to be read/writable directly.
///
/// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
/// this is considered unsafe.
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
/// struct Data {
/// value: u16,
/// count: u32,
/// }
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("my-raw.dat")?;
/// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
/// // safely use our helper:
/// let data: Data = unsafe { file.read_host_value()? };
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T>;
/// Read a little endian value.
///
/// The return type is required to implement the [`Endian`] trait, and we make the
/// assumption that this is only done for types which are supposed to be read/writable
/// directly.
///
/// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
/// this is considered unsafe.
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
/// struct Data {
/// value: u16,
/// count: u32,
/// }
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("my-little-endian.dat")?;
/// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
/// // safely use our helper:
/// let data: Data = unsafe { file.read_le_value()? };
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T>;
/// Read a big endian value.
///
/// The return type is required to implement the [`Endian`] trait, and we make the
/// assumption that this is only done for types which are supposed to be read/writable
/// directly.
///
/// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
/// this is considered unsafe.
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
/// struct Data {
/// value: u16,
/// count: u32,
/// }
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("my-big-endian.dat")?;
/// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
/// // safely use our helper:
/// let data: Data = unsafe { file.read_be_value()? };
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T>;
}
impl<R: io::Read> ReadExtOps for R {
fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>> {
let mut out = unsafe { vec::uninitialized(size) };
self.read_exact(&mut out)?;
Ok(out)
}
fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize> {
let pos = out.len();
unsafe {
out.grow_uninitialized(size);
}
let got = self.read(&mut out[pos..])?;
unsafe {
out.set_len(pos + got);
}
Ok(got)
}
fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()> {
let pos = out.len();
unsafe {
out.grow_uninitialized(size);
}
self.read_exact(&mut out[pos..])?;
Ok(())
}
unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T> {
let mut value: T = std::mem::uninitialized();
self.read_exact(std::slice::from_raw_parts_mut(
&mut value as *mut T as *mut u8,
std::mem::size_of::<T>(),
))?;
Ok(value)
}
unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T> {
Ok(self.read_host_value::<T>()?.
from_le()
)
}
unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T> {
Ok(self.read_host_value::<T>()?
.from_be()
)
}
}

4
proxmox-tools/src/lib.rs Normal file
View File

@ -0,0 +1,4 @@
//! This is a general utility crate used by all our rust projects.
pub mod vec;
pub mod io;

121
proxmox-tools/src/vec.rs Normal file
View File

@ -0,0 +1,121 @@
//! Byte vector helpers.
//!
//! We have a lot of I/O code such as:
//! ```ignore
//! let mut buffer = vec![0u8; header_size];
//! file.read_exact(&mut buffer)?;
//! ```
//! (We even have this case with a 4M buffer!)
//!
//! This needlessly initializes the buffer to zero (which not only wastes time (an insane amount of
//! time on debug builds, actually) but also prevents tools such as valgrind from pointing out
//! access to actually uninitialized data, which may hide bugs...)
//!
//! This module provides some helpers for this kind of code. Many of these are supposed to stay on
//! a lower level, with I/O helpers for types implementing [`Read`](std::io::Read) being available
//! in the [`tools::io`](crate::io) module.
//!
//! Examples:
//! ```no_run
//! use proxmox::tools::vec::{self, ops::*};
//!
//! # let size = 64usize;
//! # let more = 64usize;
//! let mut buffer = vec::undefined(size); // A zero-initialized buffer with valgrind support
//!
//! let mut buffer = unsafe { vec::uninitialized(size) }; // an actually uninitialized buffer
//! vec::clear(&mut buffer); // zero out an &mut [u8]
//!
//! vec::clear(unsafe {
//! buffer.grow_uninitialized(more) // grow the buffer with uninitialized bytes
//! });
//! ```
pub mod ops;
/// Create an uninitialized byte vector of a specific size.
///
/// This is just a shortcut for:
/// ```no_run
/// # let len = 64usize;
/// let mut v = Vec::<u8>::with_capacity(len);
/// unsafe {
/// v.set_len(len);
/// }
/// ```
#[inline]
pub unsafe fn uninitialized(len: usize) -> Vec<u8> {
let mut out = Vec::with_capacity(len);
out.set_len(len);
out
}
/// Shortcut to zero out a slice of bytes.
#[inline]
pub fn clear(data: &mut [u8]) {
unsafe {
std::ptr::write_bytes(data.as_mut_ptr(), 0, data.len());
}
}
/// Create a newly allocated, zero initialized byte vector.
#[inline]
pub fn zeroed(len: usize) -> Vec<u8> {
unsafe {
let mut out = uninitialized(len);
clear(&mut out);
out
}
}
/// Create a newly allocated byte vector of a specific size with "undefined" content.
///
/// The data will be zero initialized, but, if the `valgrind` feature is activated, it will be
/// marked as uninitialized for debugging.
#[inline]
pub fn undefined(len: usize) -> Vec<u8> {
undefined_impl(len)
}
#[cfg(not(feature = "valgrind"))]
fn undefined_impl(len: usize) -> Vec<u8> {
zeroed(len)
}
#[cfg(feature = "valgrind")]
fn undefined_impl(len: usize) -> Vec<u8> {
let out = zeroed(len);
vg::make_slice_undefined(&out[..]);
out
}
#[cfg(feature = "valgrind")]
mod vg {
type ValgrindValue = valgrind_request::Value;
/// Mark a memory region as undefined when using valgrind, causing it to treat read access to
/// it as error.
#[inline]
pub(crate) fn make_mem_undefined(addr: *const u8, len: usize) -> ValgrindValue {
const MAKE_MEM_UNDEFINED: ValgrindValue =
(((b'M' as ValgrindValue) << 24) | ((b'C' as ValgrindValue) << 16)) + 1;
unsafe {
valgrind_request::do_client_request(
0,
&[
MAKE_MEM_UNDEFINED,
addr as usize as ValgrindValue,
len as ValgrindValue,
0, 0, 0,
],
)
}
}
/// Mark a slice of bytes as undefined when using valgrind, causing it to treat read access to
/// it as error.
#[inline]
pub(crate) fn make_slice_undefined(data: &[u8]) -> ValgrindValue {
make_mem_undefined(data.as_ptr(), data.len())
}
}

99
proxmox-tools/src/vec/ops.rs Normal file
View File

@ -0,0 +1,99 @@
//! This module provides additional operations for `Vec<u8>`.
//!
//! Example:
//! ```
//! # use std::io::Read;
//! use proxmox::tools::vec::{self, ops::*};
//!
//! fn append_1024_to_vec<T: Read>(mut input: T, buffer: &mut Vec<u8>) -> std::io::Result<()> {
//! input.read_exact(unsafe { buffer.grow_uninitialized(1024) })
//! }
//! ```
/// Some additional byte vector operations useful for I/O code.
/// Example:
/// ```
/// # use std::io::Read;
/// # use proxmox::tools::io::{self, ops::*};
/// use proxmox::tools::vec::{self, ops::*};
///
/// # fn code(mut file: std::fs::File, mut data: Vec<u8>) -> std::io::Result<()> {
/// file.read_exact(unsafe {
/// data.grow_uninitialized(1024)
/// })?;
/// # Ok(())
/// # }
/// ```
///
/// Note that this module also provides a safe alternative for the case where
/// `grow_uninitialized()` is directly followed by a `read_exact()` call via the [`ReadExtOps`]
/// trait:
/// ```ignore
/// file.append_to_vec(&mut data, 1024)?;
/// ```
///
/// [`ReadExtOps`]: crate::io::ops::ReadExtOps
pub trait VecU8ExtOps {
/// Grow a vector without initializing its elements. The difference to simply using `reserve`
/// is that it also updates the actual length, making the newly allocated data part of the
/// slice.
///
/// This is a shortcut for:
/// ```ignore
/// vec.reserve(more);
/// let total = vec.len() + more;
/// unsafe {
/// vec.set_len(total);
/// }
/// ```
///
/// This returns a mutable slice to the newly allocated space, so it can be used inline:
/// ```
/// # use std::io::Read;
/// # use proxmox::tools::vec::ops::*;
/// # fn test(mut file: std::fs::File, buffer: &mut Vec<u8>) -> std::io::Result<()> {
/// file.read_exact(unsafe { buffer.grow_uninitialized(1024) })?;
/// # Ok(())
/// # }
/// ```
///
/// Although for the above case it is recommended to use the even shorter version from the
/// [`ReadExtOps`] trait:
/// ```ignore
/// // use crate::tools::vec::ops::ReadExtOps;
/// file.append_to_vec(&mut buffer, 1024)?;
/// ```
///
/// [`ReadExtOps`]: crate::io::ops::ReadExtOps
unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8];
/// Resize a vector to a specific size without initializing its data. This is a shortcut for:
/// ```ignore
/// if new_size <= vec.len() {
/// vec.truncate(new_size);
/// } else {
/// unsafe {
/// vec.grow_uninitialized(new_size - vec.len());
/// }
/// }
/// ```
unsafe fn resize_uninitialized(&mut self, total: usize);
}
impl VecU8ExtOps for Vec<u8> {
unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8] {
let old_len = self.len();
self.reserve(more);
let total = old_len + more;
self.set_len(total);
&mut self[old_len..]
}
unsafe fn resize_uninitialized(&mut self, new_size: usize) {
if new_size <= self.len() {
self.truncate(new_size);
} else {
self.grow_uninitialized(new_size - self.len());
}
}
}

11
proxmox/Cargo.toml Normal file
View File

@ -0,0 +1,11 @@
[package]
name = "proxmox"
edition = "2018"
version = "0.1.0"
authors = [
"Dietmar Maurer <dietmar@proxmox.com>",
"Wolfgang Bumiller <w.bumiller@proxmox.com>",
]
[dependencies]
proxmox-tools = { path = "../proxmox-tools" }

1
proxmox/src/lib.rs Normal file
View File

@ -0,0 +1 @@
pub use proxmox_tools as tools;

1
rust-toolchain Normal file
View File

@ -0,0 +1 @@
nightly

1
rustfmt.toml Normal file
View File

@ -0,0 +1 @@
edition = "2018"