tsarevvv/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move client to pbs-client subcrate

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2021-07-19 10:50:18 +02:00
parent 72fbe9ffa5
commit 2b7f8dd5ea
74 changed files with 802 additions and 753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -22,6 +22,7 @@ exclude = [ "build", "debian", "tests/catar_data/test_symlink/symlink1"]
[workspace]
members = [
"pbs-buildcfg",
"pbs-client",
"pbs-datastore",
"pbs-runtime",
"pbs-systemd",
@ -95,6 +96,7 @@ proxmox-openid = "0.6.0"
pbs-api-types = { path = "pbs-api-types" }
pbs-buildcfg = { path = "pbs-buildcfg" }
pbs-client = { path = "pbs-client" }
pbs-datastore = { path = "pbs-datastore" }
pbs-runtime = { path = "pbs-runtime" }
pbs-systemd = { path = "pbs-systemd" }

1
Makefile
View File

@ -33,6 +33,7 @@ RESTORE_BIN := \
SUBCRATES := \
pbs-api-types \
pbs-buildcfg \
pbs-client \
pbs-datastore \
pbs-runtime \
pbs-systemd \

4
examples/download-speed.rs
View File

@ -2,8 +2,8 @@ use std::io::Write;
use anyhow::{Error};
use proxmox_backup::api2::types::Authid;
use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupReader};
use pbs_api_types::Authid;
use pbs_client::{HttpClient, HttpClientOptions, BackupReader};
pub struct DummyWriter {
bytes: usize,

4
examples/upload-speed.rs
View File

@ -1,7 +1,7 @@
use anyhow::{Error};
use proxmox_backup::api2::types::Authid;
use proxmox_backup::client::*;
use pbs_client::{HttpClient, HttpClientOptions, BackupWriter};
use pbs_api_types::Authid;
async fn upload_speed() -> Result<f64, Error> {

37
pbs-api-types/src/lib.rs
View File

@ -40,6 +40,13 @@ pub use userid::{Tokenname, TokennameRef};
pub use userid::{Username, UsernameRef};
pub use userid::{PROXMOX_GROUP_ID_SCHEMA, PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN_NAME_SCHEMA};
#[macro_use]
mod user;
pub use user::{ApiToken, User, UserWithTokens};
pub use user::{
EMAIL_SCHEMA, ENABLE_USER_SCHEMA, EXPIRE_USER_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA,
};
pub mod upid;
pub use upid::UPID;
@ -146,33 +153,31 @@ pub const CERT_FINGERPRINT_SHA256_SCHEMA: Schema =
.format(&FINGERPRINT_SHA256_FORMAT)
.schema();
pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new(
"Number of daily backups to keep.")
pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new("Number of daily backups to keep.")
.minimum(1)
.schema();
pub const PRUNE_SCHEMA_KEEP_HOURLY: Schema = IntegerSchema::new(
"Number of hourly backups to keep.")
pub const PRUNE_SCHEMA_KEEP_HOURLY: Schema =
IntegerSchema::new("Number of hourly backups to keep.")
.minimum(1)
.schema();
pub const PRUNE_SCHEMA_KEEP_LAST: Schema = IntegerSchema::new(
"Number of backups to keep.")
pub const PRUNE_SCHEMA_KEEP_LAST: Schema = IntegerSchema::new("Number of backups to keep.")
.minimum(1)
.schema();
pub const PRUNE_SCHEMA_KEEP_MONTHLY: Schema = IntegerSchema::new(
"Number of monthly backups to keep.")
pub const PRUNE_SCHEMA_KEEP_MONTHLY: Schema =
IntegerSchema::new("Number of monthly backups to keep.")
.minimum(1)
.schema();
pub const PRUNE_SCHEMA_KEEP_WEEKLY: Schema = IntegerSchema::new(
"Number of weekly backups to keep.")
pub const PRUNE_SCHEMA_KEEP_WEEKLY: Schema =
IntegerSchema::new("Number of weekly backups to keep.")
.minimum(1)
.schema();
pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = IntegerSchema::new(
"Number of yearly backups to keep.")
pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema =
IntegerSchema::new("Number of yearly backups to keep.")
.minimum(1)
.schema();
@ -186,6 +191,14 @@ pub const SINGLE_LINE_COMMENT_SCHEMA: Schema = StringSchema::new("Comment (singl
.format(&SINGLE_LINE_COMMENT_FORMAT)
.schema();
pub const PROXMOX_CONFIG_DIGEST_SCHEMA: Schema = StringSchema::new(
"Prevent changes if current configuration file has different \
SHA256 digest. This can be used to prevent concurrent \
modifications.",
)
.format(&PVE_CONFIG_DIGEST_FORMAT)
.schema();
pub const BACKUP_ID_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&BACKUP_ID_REGEX);
/// API schema format definition for repository URLs

205
pbs-api-types/src/user.rs Normal file
View File

@ -0,0 +1,205 @@
use serde::{Deserialize, Serialize};
use proxmox::api::api;
use proxmox::api::schema::{BooleanSchema, IntegerSchema, Schema, StringSchema};
use super::{SINGLE_LINE_COMMENT_FORMAT, SINGLE_LINE_COMMENT_SCHEMA};
use super::userid::{Authid, Userid, PROXMOX_TOKEN_ID_SCHEMA};
pub const ENABLE_USER_SCHEMA: Schema = BooleanSchema::new(
"Enable the account (default). You can set this to '0' to disable the account.")
.default(true)
.schema();
pub const EXPIRE_USER_SCHEMA: Schema = IntegerSchema::new(
"Account expiration date (seconds since epoch). '0' means no expiration date.")
.default(0)
.minimum(0)
.schema();
pub const FIRST_NAME_SCHEMA: Schema = StringSchema::new("First name.")
.format(&SINGLE_LINE_COMMENT_FORMAT)
.min_length(2)
.max_length(64)
.schema();
pub const LAST_NAME_SCHEMA: Schema = StringSchema::new("Last name.")
.format(&SINGLE_LINE_COMMENT_FORMAT)
.min_length(2)
.max_length(64)
.schema();
pub const EMAIL_SCHEMA: Schema = StringSchema::new("E-Mail Address.")
.format(&SINGLE_LINE_COMMENT_FORMAT)
.min_length(2)
.max_length(64)
.schema();
#[api(
properties: {
userid: {
type: Userid,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
enable: {
optional: true,
schema: ENABLE_USER_SCHEMA,
},
expire: {
optional: true,
schema: EXPIRE_USER_SCHEMA,
},
firstname: {
optional: true,
schema: FIRST_NAME_SCHEMA,
},
lastname: {
schema: LAST_NAME_SCHEMA,
optional: true,
},
email: {
schema: EMAIL_SCHEMA,
optional: true,
},
tokens: {
type: Array,
optional: true,
description: "List of user's API tokens.",
items: {
type: ApiToken
},
},
}
)]
#[derive(Serialize,Deserialize)]
/// User properties with added list of ApiTokens
pub struct UserWithTokens {
pub userid: Userid,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub enable: Option<bool>,
#[serde(skip_serializing_if="Option::is_none")]
pub expire: Option<i64>,
#[serde(skip_serializing_if="Option::is_none")]
pub firstname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub lastname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub email: Option<String>,
#[serde(skip_serializing_if="Vec::is_empty", default)]
pub tokens: Vec<ApiToken>,
}
#[api(
properties: {
tokenid: {
schema: PROXMOX_TOKEN_ID_SCHEMA,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
enable: {
optional: true,
schema: ENABLE_USER_SCHEMA,
},
expire: {
optional: true,
schema: EXPIRE_USER_SCHEMA,
},
}
)]
#[derive(Serialize,Deserialize)]
/// ApiToken properties.
pub struct ApiToken {
pub tokenid: Authid,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub enable: Option<bool>,
#[serde(skip_serializing_if="Option::is_none")]
pub expire: Option<i64>,
}
impl ApiToken {
pub fn is_active(&self) -> bool {
if !self.enable.unwrap_or(true) {
return false;
}
if let Some(expire) = self.expire {
let now = proxmox::tools::time::epoch_i64();
if expire > 0 && expire <= now {
return false;
}
}
true
}
}
#[api(
properties: {
userid: {
type: Userid,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
enable: {
optional: true,
schema: ENABLE_USER_SCHEMA,
},
expire: {
optional: true,
schema: EXPIRE_USER_SCHEMA,
},
firstname: {
optional: true,
schema: FIRST_NAME_SCHEMA,
},
lastname: {
schema: LAST_NAME_SCHEMA,
optional: true,
},
email: {
schema: EMAIL_SCHEMA,
optional: true,
},
}
)]
#[derive(Serialize,Deserialize)]
/// User properties.
pub struct User {
pub userid: Userid,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub enable: Option<bool>,
#[serde(skip_serializing_if="Option::is_none")]
pub expire: Option<i64>,
#[serde(skip_serializing_if="Option::is_none")]
pub firstname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub lastname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub email: Option<String>,
}
impl User {
pub fn is_active(&self) -> bool {
if !self.enable.unwrap_or(true) {
return false;
}
if let Some(expire) = self.expire {
let now = proxmox::tools::time::epoch_i64();
if expire > 0 && expire <= now {
return false;
}
}
true
}
}

1
pbs-api-types/src/userid.rs
View File

@ -98,7 +98,6 @@ pub const PROXMOX_AUTH_REALM_STRING_SCHEMA: StringSchema =
.max_length(32);
pub const PROXMOX_AUTH_REALM_SCHEMA: Schema = PROXMOX_AUTH_REALM_STRING_SCHEMA.schema();
#[api(
type: String,
format: &PROXMOX_USER_NAME_FORMAT,

40
pbs-client/Cargo.toml Normal file
View File

@ -0,0 +1,40 @@
[package]
name = "pbs-client"
version = "0.1.0"
authors = ["Wolfgang Bumiller <w.bumiller@proxmox.com>"]
edition = "2018"
description = "The main proxmox backup client crate"
[dependencies]
anyhow = "1.0"
bitflags = "1.2.1"
bytes = "1.0"
futures = "0.3"
h2 = { version = "0.3", features = [ "stream" ] }
http = "0.2"
hyper = { version = "0.14", features = [ "full" ] }
lazy_static = "1.4"
libc = "0.2"
nix = "0.19.1"
openssl = "0.10"
percent-encoding = "2.1"
pin-project = "1.0"
regex = "1.2"
rustyline = "7"
serde_json = "1.0"
tokio = { version = "1.6", features = [ "fs", "signal" ] }
tokio-stream = "0.1.0"
tower-service = "0.3.0"
xdg = "2.2"
pathpatterns = "0.1.2"
proxmox = { version = "0.11.5", default-features = false, features = [ "cli" ] }
proxmox-fuse = "0.1.1"
proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "websocket" ] }
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-buildcfg = { path = "../pbs-buildcfg" }
pbs-datastore = { path = "../pbs-datastore" }
pbs-runtime = { path = "../pbs-runtime" }
pbs-tools = { path = "../pbs-tools" }

2
src/client/backup_reader.rs → pbs-client/src/backup_reader.rs
View File

@ -9,7 +9,7 @@ use serde_json::{json, Value};
use proxmox::tools::digest_to_hex;
use pbs_datastore::{CryptConfig, BackupManifest};
use pbs_datastore::{PROXMOX_BACKUP_READER_PROTOCOL_ID_V1, CryptConfig, BackupManifest};
use pbs_datastore::data_blob::DataBlob;
use pbs_datastore::data_blob_reader::DataBlobReader;
use pbs_datastore::dynamic_index::DynamicIndexReader;

0
src/client/backup_repo.rs → pbs-client/src/backup_repo.rs
View File

0
src/client/backup_specification.rs → pbs-client/src/backup_specification.rs
View File

2
src/client/backup_writer.rs → pbs-client/src/backup_writer.rs
View File

@ -14,7 +14,7 @@ use tokio_stream::wrappers::ReceiverStream;
use proxmox::tools::digest_to_hex;
use pbs_datastore::{CATALOG_NAME, CryptConfig};
use pbs_datastore::{CATALOG_NAME, PROXMOX_BACKUP_PROTOCOL_ID_V1, CryptConfig};
use pbs_datastore::data_blob::{ChunkInfo, DataBlob, DataChunkBuilder};
use pbs_datastore::dynamic_index::DynamicIndexReader;
use pbs_datastore::fixed_index::FixedIndexReader;

18
src/backup/catalog_shell.rs → pbs-client/src/catalog_shell.rs
View File

@ -18,13 +18,14 @@ use proxmox::api::cli::{self, CliCommand, CliCommandMap, CliHelper, CommandLineI
use proxmox::tools::fs::{create_path, CreateOptions};
use pxar::{EntryKind, Metadata};
use crate::backup::catalog::{self, DirEntryAttribute};
use crate::pxar::fuse::{Accessor, FileEntry};
use crate::pxar::Flags;
use pbs_runtime::block_in_place;
use crate::tools::ControlFlow;
use pbs_datastore::catalog::{self, DirEntryAttribute};
use pbs_tools::ops::ControlFlow;
type CatalogReader = crate::backup::CatalogReader<std::fs::File>;
use crate::pxar::Flags;
use crate::pxar::fuse::{Accessor, FileEntry};
type CatalogReader = pbs_datastore::catalog::CatalogReader<std::fs::File>;
const MAX_SYMLINK_COUNT: usize = 40;
@ -78,13 +79,13 @@ pub fn catalog_shell_cli() -> CommandLineInterface {
"restore-selected",
CliCommand::new(&API_METHOD_RESTORE_SELECTED_COMMAND)
.arg_param(&["target"])
.completion_cb("target", crate::tools::complete_file_name),
.completion_cb("target", pbs_tools::fs::complete_file_name),
)
.insert(
"restore",
CliCommand::new(&API_METHOD_RESTORE_COMMAND)
.arg_param(&["target"])
.completion_cb("target", crate::tools::complete_file_name),
.completion_cb("target", pbs_tools::fs::complete_file_name),
)
.insert(
"find",
@ -985,7 +986,8 @@ impl Shell {
.metadata()
.clone();
let extractor = crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT);
let extractor =
crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT);
let mut extractor = ExtractorState::new(
&mut self.catalog,

0
src/client/http_client.rs → pbs-client/src/http_client.rs
View File

4
src/client/mod.rs → pbs-client/src/lib.rs
View File

@ -10,6 +10,10 @@ use pbs_tools::ticket::Ticket;
use pbs_tools::cert::CertInfo;
use pbs_tools::auth::private_auth_key;
pub mod catalog_shell;
pub mod pxar;
pub mod tools;
mod merge_known_chunks;
pub mod pipe_to_stream;

0
src/client/merge_known_chunks.rs → pbs-client/src/merge_known_chunks.rs
View File

0
src/client/pipe_to_stream.rs → pbs-client/src/pipe_to_stream.rs
View File

7
src/pxar/create.rs → pbs-client/src/pxar/create.rs
View File

@ -23,14 +23,15 @@ use proxmox::c_str;
use proxmox::sys::error::SysError;
use proxmox::tools::fd::RawFdNum;
use proxmox::tools::vec;
use proxmox::tools::fd::Fd;
use pbs_datastore::catalog::BackupCatalogWriter;
use pbs_tools::fs;
use pbs_tools::{acl, fs, xattr};
use pbs_tools::str::strip_ascii_whitespace;
use crate::pxar::metadata::errno_is_unsupported;
use crate::pxar::Flags;
use crate::pxar::tools::assert_single_path_component;
use crate::tools::{acl, xattr, Fd};
/// Pxar options for creating a pxar archive/stream
#[derive(Default, Clone)]
@ -362,7 +363,7 @@ impl Archiver {
}
};
let line = crate::tools::strip_ascii_whitespace(&line);
let line = strip_ascii_whitespace(&line);
if line.is_empty() || line[0] == b'#' {
continue;

0
src/pxar/dir_stack.rs → pbs-client/src/pxar/dir_stack.rs
View File

6
src/pxar/extract.rs → pbs-client/src/pxar/extract.rs
View File

@ -27,12 +27,12 @@ use proxmox::tools::{
io::{sparse_copy, sparse_copy_async},
};
use pbs_tools::zip::{ZipEncoder, ZipEntry};
use crate::pxar::dir_stack::PxarDirStack;
use crate::pxar::metadata;
use crate::pxar::Flags;
use crate::tools::zip::{ZipEncoder, ZipEntry};
pub struct PxarExtractOptions<'a> {
pub match_list: &'a[MatchEntry],
pub extract_match_default: bool,
@ -215,7 +215,7 @@ where
}
/// Common state for file extraction.
pub(crate) struct Extractor {
pub struct Extractor {
feature_flags: Flags,
allow_existing_dirs: bool,
dir_stack: PxarDirStack,

0
src/pxar/flags.rs → pbs-client/src/pxar/flags.rs
View File

2
src/pxar/fuse.rs → pbs-client/src/pxar/fuse.rs
View File

@ -26,7 +26,7 @@ use pxar::accessor::{self, EntryRangeInfo, ReadAt};
use proxmox_fuse::requests::{self, FuseRequest};
use proxmox_fuse::{EntryParam, Fuse, ReplyBufState, Request, ROOT_ID};
use crate::tools::xattr;
use pbs_tools::xattr;
/// We mark inodes for regular files this way so we know how to access them.
const NON_DIRECTORY_INODE: u64 = 1u64 << 63;

3
src/pxar/metadata.rs → pbs-client/src/pxar/metadata.rs
View File

@ -13,11 +13,10 @@ use proxmox::c_result;
use proxmox::sys::error::SysError;
use proxmox::tools::fd::RawFdNum;
use pbs_tools::fs;
use pbs_tools::{acl, fs, xattr};
use crate::pxar::tools::perms_from_metadata;
use crate::pxar::Flags;
use crate::tools::{acl, xattr};
//
// utility functions

2
src/pxar/mod.rs → pbs-client/src/pxar/mod.rs
View File

@ -50,8 +50,8 @@
pub(crate) mod create;
pub(crate) mod dir_stack;
pub(crate) mod extract;
pub(crate) mod metadata;
pub mod fuse;
pub(crate) mod metadata;
pub(crate) mod tools;
mod flags;

0
src/pxar/tools.rs → pbs-client/src/pxar/tools.rs
View File

0
src/client/pxar_backup_stream.rs → pbs-client/src/pxar_backup_stream.rs
View File

0
src/client/remote_chunk_reader.rs → pbs-client/src/remote_chunk_reader.rs
View File

0
src/client/task_log.rs → pbs-client/src/task_log.rs
View File

2
src/bin/proxmox_client_tools/key_source.rs → pbs-client/src/tools/key_source.rs
View File

@ -10,7 +10,7 @@ use proxmox::api::schema::*;
use proxmox::sys::linux::tty;
use proxmox::tools::fs::file_get_contents;
use proxmox_backup::backup::CryptMode;
use pbs_api_types::CryptMode;
pub const DEFAULT_ENCRYPTION_KEY_FILE_NAME: &str = "encryption-key.json";
pub const DEFAULT_MASTER_PUBKEY_FILE_NAME: &str = "master-public.pem";

21
src/bin/proxmox_client_tools/mod.rs → pbs-client/src/tools/mod.rs
View File

@ -10,14 +10,11 @@ use proxmox::{
tools::fs::file_get_json,
};
use pbs_api_types::{BACKUP_REPO_URL, Authid};
use pbs_buildcfg;
use pbs_api_types::{BACKUP_REPO_URL, Authid, UserWithTokens};
use pbs_datastore::BackupDir;
use pbs_tools::json::json_object_to_query;
use proxmox_backup::api2::access::user::UserWithTokens;
use proxmox_backup::client::{BackupRepository, HttpClient, HttpClientOptions};
use proxmox_backup::tools;
use crate::{BackupRepository, HttpClient, HttpClientOptions};
pub mod key_source;
@ -342,7 +339,7 @@ pub fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec
return result;
}
let files = tools::complete_file_name(data[1], param);
let files = pbs_tools::fs::complete_file_name(data[1], param);
for file in files {
result.push(format!("{}:{}", data[0], file));
@ -375,15 +372,3 @@ pub fn place_xdg_file(
.and_then(|base| base.place_config_file(file_name).map_err(Error::from))
.with_context(|| format!("failed to place {} in xdg home", description))
}
/// Returns a runtime dir owned by the current user.
/// Note that XDG_RUNTIME_DIR is not always available, especially for non-login users like
/// "www-data", so we use a custom one in /run/proxmox-backup/<uid> instead.
pub fn get_user_run_dir() -> Result<std::path::PathBuf, Error> {
let uid = nix::unistd::Uid::current();
let mut path: std::path::PathBuf = pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR.into();
path.push(uid.to_string());
tools::create_run_dir()?;
std::fs::create_dir_all(&path)?;
Ok(path)
}

0
src/client/vsock_client.rs → pbs-client/src/vsock_client.rs
View File

9
pbs-tools/Cargo.toml
View File

@ -9,6 +9,10 @@ description = "common tools used throughout pbs"
[dependencies]
anyhow = "1.0"
base64 = "0.12"
bytes = "1.0"
crc32fast = "1"
endian_trait = { version = "0.6", features = ["arrays"] }
flate2 = "1.0"
foreign-types = "0.3"
futures = "0.3"
lazy_static = "1.4"
@ -21,9 +25,10 @@ regex = "1.2"
serde = "1.0"
serde_json = "1.0"
# rt-multi-thread is required for block_in_place
tokio = { version = "1.6", features = [ "rt", "rt-multi-thread", "sync" ] }
tokio = { version = "1.6", features = [ "fs", "io-util", "rt", "rt-multi-thread", "sync" ] }
url = "2.1"
walkdir = "2"
proxmox = { version = "0.11.5", default-features = false, features = [] }
proxmox = { version = "0.11.5", default-features = false, features = [ "tokio" ] }
pbs-buildcfg = { path = "../pbs-buildcfg" }

0
src/tools/acl.rs → pbs-tools/src/acl.rs
View File

194
pbs-tools/src/compression.rs Normal file
View File

@ -0,0 +1,194 @@
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use anyhow::Error;
use bytes::Bytes;
use flate2::{Compress, Compression, FlushCompress};
use futures::ready;
use futures::stream::Stream;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use proxmox::io_format_err;
use proxmox::tools::byte_buffer::ByteBuffer;
const BUFFER_SIZE: usize = 8192;
pub enum Level {
Fastest,
Best,
Default,
Precise(u32),
}
#[derive(Eq, PartialEq)]
enum EncoderState {
Reading,
Writing,
Flushing,
Finished,
}
pub struct DeflateEncoder<T> {
inner: T,
compressor: Compress,
buffer: ByteBuffer,
input_buffer: Bytes,
state: EncoderState,
}
impl<T> DeflateEncoder<T> {
pub fn new(inner: T) -> Self {
Self::with_quality(inner, Level::Default)
}
pub fn with_quality(inner: T, level: Level) -> Self {
let level = match level {
Level::Fastest => Compression::fast(),
Level::Best => Compression::best(),
Level::Default => Compression::new(3),
Level::Precise(val) => Compression::new(val),
};
Self {
inner,
compressor: Compress::new(level, false),
buffer: ByteBuffer::with_capacity(BUFFER_SIZE),
input_buffer: Bytes::new(),
state: EncoderState::Reading,
}
}
pub fn total_in(&self) -> u64 {
self.compressor.total_in()
}
pub fn total_out(&self) -> u64 {
self.compressor.total_out()
}
pub fn into_inner(self) -> T {
self.inner
}
fn encode(
&mut self,
inbuf: &[u8],
flush: FlushCompress,
) -> Result<(usize, flate2::Status), io::Error> {
let old_in = self.compressor.total_in();
let old_out = self.compressor.total_out();
let res = self
.compressor
.compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?;
let new_in = (self.compressor.total_in() - old_in) as usize;
let new_out = (self.compressor.total_out() - old_out) as usize;
self.buffer.add_size(new_out);
Ok((new_in, res))
}
}
impl DeflateEncoder<Vec<u8>> {
// assume small files
pub async fn compress_vec<R>(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error>
where
R: AsyncRead + Unpin,
{
let mut buffer = Vec::with_capacity(size_hint);
reader.read_to_end(&mut buffer).await?;
self.inner.reserve(size_hint); // should be enough since we want smalller files
self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?;
Ok(())
}
}
impl<T: AsyncWrite + Unpin> DeflateEncoder<T> {
pub async fn compress<R>(&mut self, reader: &mut R) -> Result<(), Error>
where
R: AsyncRead + Unpin,
{
let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE);
let mut eof = false;
loop {
if !eof && !buffer.is_full() {
let read = buffer.read_from_async(reader).await?;
if read == 0 {
eof = true;
}
}
let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?;
buffer.consume(read);
self.inner.write_all(&self.buffer[..]).await?;
self.buffer.clear();
if buffer.is_empty() && eof {
break;
}
}
loop {
let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?;
self.inner.write_all(&self.buffer[..]).await?;
self.buffer.clear();
if res == flate2::Status::StreamEnd {
break;
}
}
Ok(())
}
}
impl<T, O> Stream for DeflateEncoder<T>
where
T: Stream<Item = Result<O, io::Error>> + Unpin,
O: Into<Bytes>
{
type Item = Result<Bytes, io::Error>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
loop {
match this.state {
EncoderState::Reading => {
if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) {
let buf = res?;
this.input_buffer = buf.into();
this.state = EncoderState::Writing;
} else {
this.state = EncoderState::Flushing;
}
}
EncoderState::Writing => {
if this.input_buffer.is_empty() {
return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
}
let mut buf = this.input_buffer.split_off(0);
let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
this.input_buffer = buf.split_off(read);
if this.input_buffer.is_empty() {
this.state = EncoderState::Reading;
}
if this.buffer.is_full() || res == flate2::Status::BufError {
let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
return Poll::Ready(Some(Ok(bytes.into())));
}
}
EncoderState::Flushing => {
let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?;
if !this.buffer.is_empty() {
let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
return Poll::Ready(Some(Ok(bytes.into())));
}
if res == flate2::Status::StreamEnd {
this.state = EncoderState::Finished;
}
}
EncoderState::Finished => return Poll::Ready(None),
}
}
}
}

59
pbs-tools/src/fs.rs
View File

@ -1,13 +1,15 @@
//! File system helper utilities.
use std::borrow::{Borrow, BorrowMut};
use std::collections::HashMap;
use std::hash::BuildHasher;
use std::ops::{Deref, DerefMut};
use std::os::unix::io::{AsRawFd, RawFd};
use anyhow::{bail, format_err, Error};
use nix::dir;
use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::fcntl::{AtFlags, OFlag};
use nix::sys::stat::Mode;
use regex::Regex;
@ -344,3 +346,58 @@ fn do_lock_dir_noblock(
Ok(handle)
}
pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
where
S: BuildHasher,
{
let mut result = vec![];
let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
Err(_) => false,
};
if !is_dir {
if let Some(parent) = dirname.parent() {
dirname = parent.to_owned();
}
}
let mut dir =
match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
Ok(d) => d,
Err(_) => return result,
};
for item in dir.iter() {
if let Ok(entry) = item {
if let Ok(name) = entry.file_name().to_str() {
if name == "." || name == ".." {
continue;
}
let mut newpath = dirname.clone();
newpath.push(name);
if let Ok(stat) =
nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
{
if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
newpath.push("");
if let Some(newpath) = newpath.to_str() {
result.push(newpath.to_owned());
}
continue;
}
}
if let Some(newpath) = newpath.to_str() {
result.push(newpath.to_owned());
}
}
}
}
result
}

5
pbs-tools/src/lib.rs
View File

@ -1,11 +1,14 @@
pub mod acl;
pub mod auth;
pub mod borrow;
pub mod broadcast_future;
pub mod cert;
pub mod compression;
pub mod format;
pub mod fs;
pub mod json;
pub mod nom;
pub mod ops;
pub mod percent_encoding;
pub mod process_locker;
pub mod sha;
@ -13,6 +16,8 @@ pub mod str;
pub mod sync;
pub mod ticket;
pub mod tokio;
pub mod xattr;
pub mod zip;
mod command;
pub use command::{command_output, command_output_as_string, run_command};

12
pbs-tools/src/ops.rs Normal file
View File

@ -0,0 +1,12 @@
//! std::ops extensions
/// Modeled after the nightly `std::ops::ControlFlow`.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ControlFlow<B, C = ()> {
Continue(C),
Break(B),
}
impl<B> ControlFlow<B> {
pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
}

10
pbs-tools/src/str.rs
View File

@ -15,3 +15,13 @@ pub fn join<S: Borrow<str>>(data: &[S], sep: char) -> String {
list
}
pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
Some(n) => &line[n..],
None => return &[],
};
match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
Some(n) => &line[..(line.len() - n)],
None => &[],
}
}

0
src/tools/xattr.rs → pbs-tools/src/xattr.rs
View File

2
src/tools/zip.rs → pbs-tools/src/zip.rs
View File

@ -22,7 +22,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
use crc32fast::Hasher;
use proxmox::tools::time::gmtime;
use crate::tools::compression::{DeflateEncoder, Level};
use crate::compression::{DeflateEncoder, Level};
const LOCAL_FH_SIG: u32 = 0x04034B50;
const LOCAL_FF_SIG: u32 = 0x08074B50;

75
src/api2/access/user.rs
View File

@ -10,7 +10,11 @@ use proxmox::api::router::SubdirMap;
use proxmox::api::schema::{Schema, StringSchema};
use proxmox::tools::fs::open_file_locked;
use crate::api2::types::*;
use pbs_api_types::{
PASSWORD_FORMAT, PROXMOX_CONFIG_DIGEST_SCHEMA, SINGLE_LINE_COMMENT_SCHEMA, Authid,
Tokenname, UserWithTokens, Userid,
};
use crate::config::user;
use crate::config::token_shadow;
use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
@ -22,68 +26,8 @@ pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.")
.max_length(64)
.schema();
#[api(
properties: {
userid: {
type: Userid,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
enable: {
optional: true,
schema: user::ENABLE_USER_SCHEMA,
},
expire: {
optional: true,
schema: user::EXPIRE_USER_SCHEMA,
},
firstname: {
optional: true,
schema: user::FIRST_NAME_SCHEMA,
},
lastname: {
schema: user::LAST_NAME_SCHEMA,
optional: true,
},
email: {
schema: user::EMAIL_SCHEMA,
optional: true,
},
tokens: {
type: Array,
optional: true,
description: "List of user's API tokens.",
items: {
type: user::ApiToken
},
},
}
)]
#[derive(Serialize,Deserialize)]
/// User properties with added list of ApiTokens
pub struct UserWithTokens {
pub userid: Userid,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub enable: Option<bool>,
#[serde(skip_serializing_if="Option::is_none")]
pub expire: Option<i64>,
#[serde(skip_serializing_if="Option::is_none")]
pub firstname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub lastname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub email: Option<String>,
#[serde(skip_serializing_if="Vec::is_empty", default)]
pub tokens: Vec<user::ApiToken>,
}
impl UserWithTokens {
fn new(user: user::User) -> Self {
Self {
fn new_user_with_tokens(user: user::User) -> UserWithTokens {
UserWithTokens {
userid: user.userid,
comment: user.comment,
enable: user.enable,
@ -94,7 +38,6 @@ impl UserWithTokens {
tokens: Vec::new(),
}
}
}
#[api(
input: {
@ -165,13 +108,13 @@ pub fn list_users(
});
iter
.map(|user: user::User| {
let mut user = UserWithTokens::new(user);
let mut user = new_user_with_tokens(user);
user.tokens = user_to_tokens.remove(&user.userid).unwrap_or_default();
user
})
.collect()
} else {
iter.map(UserWithTokens::new)
iter.map(new_user_with_tokens)
.collect()
};

3
src/api2/admin/datastore.rs
View File

@ -26,13 +26,14 @@ use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
use pxar::accessor::aio::Accessor;
use pxar::EntryKind;
use pbs_client::pxar::create_zip;
use crate::api2::types::*;
use crate::api2::node::rrd::create_value_from_rrd;
use crate::api2::helpers;
use crate::backup::*;
use crate::config::datastore;
use crate::config::cached_user_info::CachedUserInfo;
use crate::pxar::create_zip;
use crate::server::{jobstate::Job, WorkerTask};
use crate::tools::{

1
src/api2/backup/mod.rs
View File

@ -13,6 +13,7 @@ use proxmox::api::router::SubdirMap;
use proxmox::api::schema::*;
use pbs_tools::fs::lock_dir_noblock_shared;
use pbs_datastore::PROXMOX_BACKUP_PROTOCOL_ID_V1;
use crate::tools;
use crate::server::{WorkerTask, H2Service};

3
src/api2/config/remote.rs
View File

@ -6,8 +6,9 @@ use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
use proxmox::http_err;
use proxmox::tools::fs::open_file_locked;
use pbs_client::{HttpClient, HttpClientOptions};
use crate::api2::types::*;
use crate::client::{HttpClient, HttpClientOptions};
use crate::config::cached_user_info::CachedUserInfo;
use crate::config::remote;
use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};

3
src/api2/pull.rs
View File

@ -7,9 +7,10 @@ use futures::{select, future::FutureExt};
use proxmox::api::api;
use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
use pbs_client::{HttpClient, BackupRepository};
use crate::server::{WorkerTask, jobstate::Job, pull::pull_store};
use crate::backup::DataStore;
use crate::client::{HttpClient, BackupRepository};
use crate::api2::types::{
DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid,
};

1
src/api2/reader/mod.rs
View File

@ -28,6 +28,7 @@ use proxmox::{
};
use pbs_tools::fs::lock_dir_noblock_shared;
use pbs_datastore::PROXMOX_BACKUP_READER_PROTOCOL_ID_V1;
use crate::{
api2::{

8
src/api2/types/mod.rs
View File

@ -107,14 +107,6 @@ pub const TAPE_ENCRYPTION_KEY_FINGERPRINT_SCHEMA: Schema = StringSchema::new(
.format(&FINGERPRINT_SHA256_FORMAT)
.schema();
pub const PROXMOX_CONFIG_DIGEST_SCHEMA: Schema = StringSchema::new(
"Prevent changes if current configuration file has different \
SHA256 digest. This can be used to prevent concurrent \
modifications."
)
.format(&PVE_CONFIG_DIGEST_FORMAT) .schema();
pub const CHUNK_DIGEST_SCHEMA: Schema = StringSchema::new("Chunk digest (SHA256).")
.format(&CHUNK_DIGEST_FORMAT)
.schema();

18
src/backup/mod.rs
View File

@ -5,20 +5,6 @@ use anyhow::{bail, Error};
// Note: .pcat1 => Proxmox Catalog Format version 1
pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
#[macro_export]
macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
() => {
"proxmox-backup-protocol-v1"
};
}
#[macro_export]
macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
() => {
"proxmox-backup-reader-protocol-v1"
};
}
/// Unix system user used by proxmox-backup-proxy
pub const BACKUP_USER_NAME: &str = "backup";
/// Unix system group used by proxmox-backup-proxy
@ -102,9 +88,5 @@ pub use datastore::*;
mod verify;
pub use verify::*;
// Move to client
mod catalog_shell;
pub use catalog_shell::*;
mod cached_chunk_reader;
pub use cached_chunk_reader::*;

2
src/bin/dump-catalog-shell-cli.rs
View File

@ -3,7 +3,7 @@ use anyhow::{Error};
use proxmox::api::format::*;
use proxmox::api::cli::*;
use proxmox_backup::backup::catalog_shell_cli;
use pbs_client::catalog_shell::catalog_shell_cli;
fn main() -> Result<(), Error> {

111
src/bin/proxmox-backup-client.rs
View File

@ -27,47 +27,24 @@ use proxmox::{
};
use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
use pbs_datastore::catalog::BackupCatalogWriter;
use pbs_tools::sync::StdChannelWriter;
use pbs_tools::tokio::TokioWriterAdapter;
use proxmox_backup::api2::types::*;
use proxmox_backup::api2::version;
use proxmox_backup::client::*;
use proxmox_backup::backup::{
archive_type,
decrypt_key,
rsa_encrypt_key_config,
verify_chunk_size,
ArchiveType,
AsyncReadChunk,
BackupDir,
BackupGroup,
BackupManifest,
BufferedDynamicReader,
CATALOG_NAME,
CatalogReader,
CatalogWriter,
ChunkStream,
CryptConfig,
CryptMode,
DynamicIndexReader,
ENCRYPTED_KEY_BLOB_NAME,
FixedChunkStream,
FixedIndexReader,
KeyConfig,
IndexFile,
MANIFEST_BLOB_NAME,
Shell,
PruneOptions,
use pbs_api_types::CryptMode;
use pbs_client::{
BACKUP_SOURCE_SCHEMA,
BackupReader,
BackupRepository,
BackupSpecificationType,
BackupStats,
BackupWriter,
HttpClient,
PxarBackupStream,
RemoteChunkReader,
UploadOptions,
delete_ticket_info,
parse_backup_specification,
view_task_result,
};
use proxmox_backup::tools;
mod proxmox_backup_client;
use proxmox_backup_client::*;
pub mod proxmox_client_tools;
use proxmox_client_tools::{
use pbs_client::catalog_shell::Shell;
use pbs_client::tools::{
complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
@ -78,6 +55,37 @@ use proxmox_client_tools::{
},
CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
};
use pbs_datastore::CryptConfig;
use pbs_datastore::backup_info::{BackupDir, BackupGroup};
use pbs_datastore::catalog::BackupCatalogWriter;
use pbs_datastore::dynamic_index::DynamicIndexReader;
use pbs_datastore::fixed_index::FixedIndexReader;
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type};
use pbs_datastore::read_chunk::AsyncReadChunk;
use pbs_tools::sync::StdChannelWriter;
use pbs_tools::tokio::TokioWriterAdapter;
use proxmox_backup::api2::types::*;
use proxmox_backup::api2::version;
use proxmox_backup::backup::{
decrypt_key,
rsa_encrypt_key_config,
verify_chunk_size,
BufferedDynamicReader,
CATALOG_NAME,
CatalogReader,
CatalogWriter,
ChunkStream,
ENCRYPTED_KEY_BLOB_NAME,
FixedChunkStream,
KeyConfig,
PruneOptions,
};
use proxmox_backup::tools;
mod proxmox_backup_client;
use proxmox_backup_client::*;
fn record_repository(repo: &BackupRepository) {
@ -172,7 +180,7 @@ async fn backup_directory<P: AsRef<Path>>(
archive_name: &str,
chunk_size: Option<usize>,
catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
pxar_create_options: pbs_client::pxar::PxarCreateOptions,
upload_options: UploadOptions,
) -> Result<BackupStats, Error> {
@ -589,7 +597,7 @@ fn spawn_catalog_upload(
type: Integer,
description: "Max number of entries to hold in memory.",
optional: true,
default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
},
"verbose": {
type: Boolean,
@ -633,7 +641,7 @@ async fn create_backup(
let include_dev = param["include-dev"].as_array();
let entries_max = param["entries-max"].as_u64()
.unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
.unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
let empty = Vec::new();
let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
@ -856,7 +864,7 @@ async fn create_backup(
println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
let pxar_options = pbs_client::pxar::PxarCreateOptions {
device_set: devices.clone(),
patterns: pattern_list.clone(),
entries_max: entries_max as usize,
@ -1168,7 +1176,7 @@ async fn restore(param: Value) -> Result<Value, Error> {
let mut reader = BufferedDynamicReader::new(index, chunk_reader);
let options = proxmox_backup::pxar::PxarExtractOptions {
let options = pbs_client::pxar::PxarExtractOptions {
match_list: &[],
extract_match_default: true,
allow_existing_dirs,
@ -1176,10 +1184,10 @@ async fn restore(param: Value) -> Result<Value, Error> {
};
if let Some(target) = target {
proxmox_backup::pxar::extract_archive(
pbs_client::pxar::extract_archive(
pxar::decoder::Decoder::from_std(reader)?,
Path::new(target),
proxmox_backup::pxar::Flags::DEFAULT,
pbs_client::pxar::Flags::DEFAULT,
|path| {
if verbose {
println!("{:?}", path);
@ -1377,7 +1385,6 @@ async fn status(param: Value) -> Result<Value, Error> {
Ok(Value::Null)
}
use proxmox_backup::client::RemoteChunkReader;
/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
/// async use!
///
@ -1424,13 +1431,13 @@ fn main() {
.arg_param(&["backupspec"])
.completion_cb("repository", complete_repository)
.completion_cb("backupspec", complete_backup_source)
.completion_cb("keyfile", tools::complete_file_name)
.completion_cb("master-pubkey-file", tools::complete_file_name)
.completion_cb("keyfile", pbs_tools::fs::complete_file_name)
.completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
.completion_cb("chunk-size", complete_chunk_size);
let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
.completion_cb("repository", complete_repository)
.completion_cb("keyfile", tools::complete_file_name);
.completion_cb("keyfile", pbs_tools::fs::complete_file_name);
let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
.completion_cb("repository", complete_repository);
@ -1443,7 +1450,7 @@ fn main() {
.completion_cb("repository", complete_repository)
.completion_cb("snapshot", complete_group_or_snapshot)
.completion_cb("archive-name", complete_archive_name)
.completion_cb("target", tools::complete_file_name);
.completion_cb("target", pbs_tools::fs::complete_file_name);
let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
.arg_param(&["group"])

2
src/bin/proxmox-backup-manager.rs
View File

@ -6,12 +6,12 @@ use serde_json::{json, Value};
use proxmox::api::{api, cli::*, RpcEnvironment};
use pbs_client::{connect_to_localhost, display_task_log, view_task_result};
use pbs_tools::percent_encoding::percent_encode_component;
use proxmox_backup::tools;
use proxmox_backup::config;
use proxmox_backup::api2::{self, types::* };
use proxmox_backup::client::*;
use proxmox_backup::server::wait_for_local_worker;
mod proxmox_backup_manager;

37
src/bin/proxmox-file-restore.rs
View File

@ -16,18 +16,10 @@ use proxmox::api::{
use pxar::accessor::aio::Accessor;
use pxar::decoder::aio::Decoder;
use proxmox_backup::api2::{helpers, types::ArchiveEntry};
use proxmox_backup::backup::{
decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode,
DirEntryAttribute, IndexFile, LocalDynamicReadAt, CATALOG_NAME,
};
use proxmox_backup::client::{BackupReader, RemoteChunkReader};
use proxmox_backup::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
use proxmox_backup::tools;
// use "pub" so rust doesn't complain about "unused" functions in the module
pub mod proxmox_client_tools;
use proxmox_client_tools::{
use pbs_datastore::index::IndexFile;
use pbs_client::{BackupReader, RemoteChunkReader};
use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
use pbs_client::tools::{
complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
key_source::{
crypto_parameters_keep_fd, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
@ -36,6 +28,13 @@ use proxmox_client_tools::{
REPO_URL_SCHEMA,
};
use proxmox_backup::api2::{helpers, types::ArchiveEntry};
use proxmox_backup::backup::{
decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode,
DirEntryAttribute, LocalDynamicReadAt, CATALOG_NAME,
};
use proxmox_backup::tools;
mod proxmox_file_restore;
use proxmox_file_restore::*;
@ -456,7 +455,7 @@ fn main() {
.arg_param(&["snapshot", "path", "target"])
.completion_cb("repository", complete_repository)
.completion_cb("snapshot", complete_group_or_snapshot)
.completion_cb("target", tools::complete_file_name);
.completion_cb("target", pbs_tools::fs::complete_file_name);
let status_cmd_def = CliCommand::new(&API_METHOD_STATUS);
let stop_cmd_def = CliCommand::new(&API_METHOD_STOP)
@ -476,3 +475,15 @@ fn main() {
Some(|future| pbs_runtime::main(future)),
);
}
/// Returns a runtime dir owned by the current user.
/// Note that XDG_RUNTIME_DIR is not always available, especially for non-login users like
/// "www-data", so we use a custom one in /run/proxmox-backup/<uid> instead.
pub fn get_user_run_dir() -> Result<std::path::PathBuf, Error> {
let uid = nix::unistd::Uid::current();
let mut path: std::path::PathBuf = pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR.into();
path.push(uid.to_string());
tools::create_run_dir()?;
std::fs::create_dir_all(&path)?;
Ok(path)
}

3
src/bin/proxmox-restore-daemon.rs
View File

@ -13,8 +13,9 @@ use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use pbs_client::DEFAULT_VSOCK_PORT;
use proxmox::api::RpcEnvironmentType;
use proxmox_backup::client::DEFAULT_VSOCK_PORT;
use proxmox_backup::server::{rest::*, ApiConfig};
mod proxmox_restore_daemon;

5
src/bin/proxmox-tape.rs
View File

@ -14,6 +14,7 @@ use proxmox::{
},
};
use pbs_client::{connect_to_localhost, view_task_result};
use pbs_tools::format::{
HumanByte,
render_epoch,
@ -21,10 +22,6 @@ use pbs_tools::format::{
};
use proxmox_backup::{
client::{
connect_to_localhost,
view_task_result,
},
api2::{
self,
types::{

7
src/bin/proxmox_backup_client/benchmark.rs
View File

@ -18,6 +18,9 @@ use proxmox::api::{
router::ReturnType,
};
use pbs_client::tools::key_source::get_encryption_key_password;
use pbs_client::{BackupRepository, BackupWriter};
use proxmox_backup::backup::{
load_and_decrypt_key,
CryptConfig,
@ -25,8 +28,6 @@ use proxmox_backup::backup::{
DataChunkBuilder,
};
use proxmox_backup::client::*;
use crate::{
KEYFILE_SCHEMA, REPO_URL_SCHEMA,
extract_repository_from_value,
@ -34,8 +35,6 @@ use crate::{
connect,
};
use crate::proxmox_client_tools::key_source::get_encryption_key_password;
#[api()]
#[derive(Copy, Clone, Serialize)]
/// Speed test result

11
src/bin/proxmox_backup_client/catalog.rs
View File

@ -7,9 +7,10 @@ use serde_json::Value;
use proxmox::api::{api, cli::*};
use proxmox_backup::tools;
use pbs_client::tools::key_source::get_encryption_key_password;
use pbs_client::{BackupReader, RemoteChunkReader};
use proxmox_backup::client::*;
use proxmox_backup::tools;
use crate::{
REPO_URL_SCHEMA,
@ -37,8 +38,6 @@ use crate::{
Shell,
};
use crate::proxmox_client_tools::key_source::get_encryption_key_password;
#[api(
input: {
properties: {
@ -219,9 +218,9 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), most_used);
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader: proxmox_backup::pxar::fuse::Reader =
let reader: pbs_client::pxar::fuse::Reader =
Arc::new(BufferedDynamicReadAt::new(reader));
let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
let decoder = pbs_client::pxar::fuse::Accessor::new(reader, archive_size).await?;
client.download(CATALOG_NAME, &mut tmpfile).await?;
let index = DynamicIndexReader::new(tmpfile)

29
src/bin/proxmox_backup_client/key.rs
View File

@ -14,19 +14,18 @@ use proxmox::sys::linux::tty;
use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
use pbs_datastore::{KeyInfo, Kdf};
use pbs_client::tools::key_source::{
find_default_encryption_key, find_default_master_pubkey, get_encryption_key_password,
place_default_encryption_key, place_default_master_pubkey,
};
use proxmox_backup::{
api2::types::{RsaPubKeyInfo, PASSWORD_HINT_SCHEMA},
backup::{rsa_decrypt_key_config, KeyConfig},
tools,
tools::paperkey::{generate_paper_key, PaperkeyFormat},
};
use crate::proxmox_client_tools::key_source::{
find_default_encryption_key, find_default_master_pubkey, get_encryption_key_password,
place_default_encryption_key, place_default_master_pubkey,
};
#[api(
input: {
properties: {
@ -458,35 +457,35 @@ fn paper_key(
pub fn cli() -> CliCommandMap {
let key_create_cmd_def = CliCommand::new(&API_METHOD_CREATE)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
let key_import_with_master_key_cmd_def = CliCommand::new(&API_METHOD_IMPORT_WITH_MASTER_KEY)
.arg_param(&["master-keyfile"])
.completion_cb("master-keyfile", tools::complete_file_name)
.completion_cb("master-keyfile", pbs_tools::fs::complete_file_name)
.arg_param(&["encrypted-keyfile"])
.completion_cb("encrypted-keyfile", tools::complete_file_name)
.completion_cb("encrypted-keyfile", pbs_tools::fs::complete_file_name)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_CHANGE_PASSPHRASE)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_CREATE_MASTER_KEY);
let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_IMPORT_MASTER_PUBKEY)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
let key_show_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_SHOW_MASTER_PUBKEY)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
let key_show_cmd_def = CliCommand::new(&API_METHOD_SHOW_KEY)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
let paper_key_cmd_def = CliCommand::new(&API_METHOD_PAPER_KEY)
.arg_param(&["path"])
.completion_cb("path", tools::complete_file_name);
.completion_cb("path", pbs_tools::fs::complete_file_name);
CliCommandMap::new()
.insert("create", key_create_cmd_def)

15
src/bin/proxmox_backup_client/mount.rs
View File

@ -17,6 +17,9 @@ use proxmox::{sortable, identity};
use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment, schema::*, cli::*};
use proxmox::tools::fd::Fd;
use pbs_client::tools::key_source::get_encryption_key_password;
use pbs_client::{BackupReader, RemoteChunkReader};
use proxmox_backup::tools;
use proxmox_backup::backup::{
load_and_decrypt_key,
@ -28,8 +31,6 @@ use proxmox_backup::backup::{
CachedChunkReader,
};
use proxmox_backup::client::*;
use crate::{
REPO_URL_SCHEMA,
extract_repository_from_value,
@ -43,8 +44,6 @@ use crate::{
BufferedDynamicReadAt,
};
use crate::proxmox_client_tools::key_source::get_encryption_key_password;
#[sortable]
const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&mount),
@ -98,7 +97,7 @@ pub fn mount_cmd_def() -> CliCommand {
.completion_cb("repository", complete_repository)
.completion_cb("snapshot", complete_group_or_snapshot)
.completion_cb("archive-name", complete_pxar_archive_name)
.completion_cb("target", tools::complete_file_name)
.completion_cb("target", pbs_tools::fs::complete_file_name)
}
pub fn map_cmd_def() -> CliCommand {
@ -257,11 +256,11 @@ async fn mount_do(param: Value, pipe: Option<Fd>) -> Result<Value, Error> {
let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader: proxmox_backup::pxar::fuse::Reader =
let reader: pbs_client::pxar::fuse::Reader =
Arc::new(BufferedDynamicReadAt::new(reader));
let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
let decoder = pbs_client::pxar::fuse::Accessor::new(reader, archive_size).await?;
let session = proxmox_backup::pxar::fuse::Session::mount(
let session = pbs_client::pxar::fuse::Session::mount(
decoder,
&options,
false,

8
src/bin/proxmox_backup_client/snapshot.rs
View File

@ -8,6 +8,8 @@ use proxmox::{
tools::fs::file_get_contents,
};
use pbs_client::tools::key_source::get_encryption_key_password;
use proxmox_backup::{
tools,
api2::types::*,
@ -35,8 +37,6 @@ use crate::{
record_repository,
};
use crate::proxmox_client_tools::key_source::get_encryption_key_password;
#[api(
input: {
properties: {
@ -412,8 +412,8 @@ pub fn snapshot_mgtm_cli() -> CliCommandMap {
CliCommand::new(&API_METHOD_UPLOAD_LOG)
.arg_param(&["snapshot", "logfile"])
.completion_cb("snapshot", complete_backup_snapshot)
.completion_cb("logfile", tools::complete_file_name)
.completion_cb("keyfile", tools::complete_file_name)
.completion_cb("logfile", pbs_tools::fs::complete_file_name)
.completion_cb("keyfile", pbs_tools::fs::complete_file_name)
.completion_cb("repository", complete_repository)
)
}

2
src/bin/proxmox_backup_client/task.rs
View File

@ -4,10 +4,10 @@ use serde_json::{json, Value};
use proxmox::api::{api, cli::*};
use pbs_tools::percent_encoding::percent_encode_component;
use pbs_client::display_task_log;
use proxmox_backup::tools;
use proxmox_backup::client::*;
use proxmox_backup::api2::types::UPID_SCHEMA;
use crate::{

6
src/bin/proxmox_backup_manager/datastore.rs
View File

@ -3,12 +3,10 @@ use serde_json::Value;
use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
use pbs_client::{connect_to_localhost, view_task_result};
use proxmox_backup::config;
use proxmox_backup::api2::{self, types::* };
use proxmox_backup::client::{
connect_to_localhost,
view_task_result,
};
use proxmox_backup::config::datastore::DIR_NAME_SCHEMA;
#[api(

15
src/bin/proxmox_file_restore/block_driver.rs
View File

@ -1,19 +1,20 @@
//! Abstraction layer over different methods of accessing a block backup
use anyhow::{bail, Error};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::future::Future;
use std::hash::BuildHasher;
use std::pin::Pin;
use proxmox_backup::backup::{BackupDir, BackupManifest};
use proxmox_backup::api2::types::ArchiveEntry;
use proxmox_backup::client::BackupRepository;
use anyhow::{bail, Error};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use proxmox::api::{api, cli::*};
use pbs_client::BackupRepository;
use proxmox_backup::backup::{BackupDir, BackupManifest};
use proxmox_backup::api2::types::ArchiveEntry;
use super::block_driver_qemu::QemuBlockDriver;
/// Contains details about a snapshot that is to be accessed by block file restore

14
src/bin/proxmox_file_restore/block_driver_qemu.rs
View File

@ -1,21 +1,23 @@
//! Block file access via a small QEMU restore VM using the PBS block driver in QEMU
use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::{prelude::*, SeekFrom};
use anyhow::{bail, Error};
use futures::FutureExt;
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::{prelude::*, SeekFrom};
use proxmox::tools::fs::lock_file;
use pbs_client::{DEFAULT_VSOCK_PORT, BackupRepository, VsockClient};
use proxmox_backup::api2::types::ArchiveEntry;
use proxmox_backup::backup::BackupDir;
use proxmox_backup::client::*;
use proxmox_backup::tools;
use super::block_driver::*;
use crate::proxmox_client_tools::get_user_run_dir;
use crate::get_user_run_dir;
const RESTORE_VM_MAP: &str = "restore-vm-map.json";

3
src/bin/proxmox_file_restore/qemu_helper.rs
View File

@ -13,8 +13,9 @@ use nix::unistd::Pid;
use proxmox::tools::fs::{create_path, file_read_string, make_tmp_file, CreateOptions};
use pbs_client::{VsockClient, DEFAULT_VSOCK_PORT};
use proxmox_backup::backup::backup_user;
use proxmox_backup::client::{VsockClient, DEFAULT_VSOCK_PORT};
use proxmox_backup::tools;
use super::SnapRestoreDetails;

5
src/bin/proxmox_restore_daemon/api.rs
View File

@ -19,12 +19,13 @@ use proxmox::api::{
};
use proxmox::{identity, list_subdirs_api_method, sortable};
use pbs_client::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES};
use pbs_tools::fs::read_subdir;
use pbs_tools::zip::zip_directory;
use proxmox_backup::api2::types::*;
use proxmox_backup::backup::DirEntryAttribute;
use proxmox_backup::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES};
use proxmox_backup::tools::{self, zip::zip_directory};
use proxmox_backup::tools;
use pxar::encoder::aio::TokioWriter;

6
src/bin/proxmox_tape/backup_job.rs
View File

@ -3,12 +3,10 @@ use serde_json::Value;
use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
use pbs_client::{connect_to_localhost, view_task_result};
use proxmox_backup::{
config,
client::{
connect_to_localhost,
view_task_result,
},
api2::{
self,
types::*,

27
src/bin/pxar.rs
View File

@ -12,13 +12,11 @@ use futures::select;
use tokio::signal::unix::{signal, SignalKind};
use pathpatterns::{MatchEntry, MatchType, PatternFlag};
use pbs_client::pxar::{fuse, format_single_line_entry, ENCODER_MAX_ENTRIES, Flags, PxarExtractOptions};
use proxmox::api::cli::*;
use proxmox::api::api;
use proxmox_backup::tools;
use proxmox_backup::pxar::{fuse, format_single_line_entry, ENCODER_MAX_ENTRIES, Flags, PxarExtractOptions};
fn extract_archive_from_reader<R: std::io::Read>(
reader: &mut R,
target: &str,
@ -26,8 +24,7 @@ fn extract_archive_from_reader<R: std::io::Read>(
verbose: bool,
options: PxarExtractOptions,
) -> Result<(), Error> {
proxmox_backup::pxar::extract_archive(
pbs_client::pxar::extract_archive(
pxar::decoder::Decoder::from_std(reader)?,
Path::new(target),
feature_flags,
@ -327,7 +324,7 @@ async fn create_archive(
Some(HashSet::new())
};
let options = proxmox_backup::pxar::PxarCreateOptions {
let options = pbs_client::pxar::PxarCreateOptions {
entries_max: entries_max as usize,
device_set,
patterns,
@ -372,7 +369,7 @@ async fn create_archive(
}
let writer = pxar::encoder::sync::StandardWriter::new(writer);
proxmox_backup::pxar::create_archive(
pbs_client::pxar::create_archive(
dir,
writer,
feature_flags,
@ -464,29 +461,29 @@ fn main() {
"create",
CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
.arg_param(&["archive", "source"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("source", tools::complete_file_name),
.completion_cb("archive", pbs_tools::fs::complete_file_name)
.completion_cb("source", pbs_tools::fs::complete_file_name),
)
.insert(
"extract",
CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
.arg_param(&["archive", "target"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("target", tools::complete_file_name)
.completion_cb("files-from", tools::complete_file_name),
.completion_cb("archive", pbs_tools::fs::complete_file_name)
.completion_cb("target", pbs_tools::fs::complete_file_name)
.completion_cb("files-from", pbs_tools::fs::complete_file_name),
)
.insert(
"mount",
CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
.arg_param(&["archive", "mountpoint"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("mountpoint", tools::complete_file_name),
.completion_cb("archive", pbs_tools::fs::complete_file_name)
.completion_cb("mountpoint", pbs_tools::fs::complete_file_name),
)
.insert(
"list",
CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
.arg_param(&["archive"])
.completion_cb("archive", tools::complete_file_name),
.completion_cb("archive", pbs_tools::fs::complete_file_name),
);
let rpcenv = CliEnvironment::new();

150
src/config/user.rs
View File

@ -3,10 +3,8 @@ use std::sync::{Arc, RwLock};
use anyhow::{bail, Error};
use lazy_static::lazy_static;
use serde::{Serialize, Deserialize};
use proxmox::api::{
api,
schema::*,
section_config::{
SectionConfig,
@ -17,154 +15,18 @@ use proxmox::api::{
use proxmox::tools::{fs::replace_file, fs::CreateOptions};
use crate::api2::types::*;
use pbs_api_types::{Authid, Userid};
pub use pbs_api_types::{ApiToken, User};
pub use pbs_api_types::{
EMAIL_SCHEMA, ENABLE_USER_SCHEMA, EXPIRE_USER_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA,
};
use crate::tools::Memcom;
lazy_static! {
pub static ref CONFIG: SectionConfig = init();
}
pub const ENABLE_USER_SCHEMA: Schema = BooleanSchema::new(
"Enable the account (default). You can set this to '0' to disable the account.")
.default(true)
.schema();
pub const EXPIRE_USER_SCHEMA: Schema = IntegerSchema::new(
"Account expiration date (seconds since epoch). '0' means no expiration date.")
.default(0)
.minimum(0)
.schema();
pub const FIRST_NAME_SCHEMA: Schema = StringSchema::new("First name.")
.format(&SINGLE_LINE_COMMENT_FORMAT)
.min_length(2)
.max_length(64)
.schema();
pub const LAST_NAME_SCHEMA: Schema = StringSchema::new("Last name.")
.format(&SINGLE_LINE_COMMENT_FORMAT)
.min_length(2)
.max_length(64)
.schema();
pub const EMAIL_SCHEMA: Schema = StringSchema::new("E-Mail Address.")
.format(&SINGLE_LINE_COMMENT_FORMAT)
.min_length(2)
.max_length(64)
.schema();
#[api(
properties: {
tokenid: {
schema: PROXMOX_TOKEN_ID_SCHEMA,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
enable: {
optional: true,
schema: ENABLE_USER_SCHEMA,
},
expire: {
optional: true,
schema: EXPIRE_USER_SCHEMA,
},
}
)]
#[derive(Serialize,Deserialize)]
/// ApiToken properties.
pub struct ApiToken {
pub tokenid: Authid,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub enable: Option<bool>,
#[serde(skip_serializing_if="Option::is_none")]
pub expire: Option<i64>,
}
impl ApiToken {
pub fn is_active(&self) -> bool {
if !self.enable.unwrap_or(true) {
return false;
}
if let Some(expire) = self.expire {
let now = proxmox::tools::time::epoch_i64();
if expire > 0 && expire <= now {
return false;
}
}
true
}
}
#[api(
properties: {
userid: {
type: Userid,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
enable: {
optional: true,
schema: ENABLE_USER_SCHEMA,
},
expire: {
optional: true,
schema: EXPIRE_USER_SCHEMA,
},
firstname: {
optional: true,
schema: FIRST_NAME_SCHEMA,
},
lastname: {
schema: LAST_NAME_SCHEMA,
optional: true,
},
email: {
schema: EMAIL_SCHEMA,
optional: true,
},
}
)]
#[derive(Serialize,Deserialize)]
/// User properties.
pub struct User {
pub userid: Userid,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub enable: Option<bool>,
#[serde(skip_serializing_if="Option::is_none")]
pub expire: Option<i64>,
#[serde(skip_serializing_if="Option::is_none")]
pub firstname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub lastname: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub email: Option<String>,
}
impl User {
pub fn is_active(&self) -> bool {
if !self.enable.unwrap_or(true) {
return false;
}
if let Some(expire) = self.expire {
let now = proxmox::tools::time::epoch_i64();
if expire > 0 && expire <= now {
return false;
}
}
true
}
}
fn init() -> SectionConfig {
let mut config = SectionConfig::new(&Authid::API_SCHEMA);

4
src/lib.rs
View File

@ -9,8 +9,6 @@ pub mod tools;
#[macro_use]
pub mod server;
pub mod pxar;
#[macro_use]
pub mod backup;
@ -18,8 +16,6 @@ pub mod config;
pub mod api2;
pub mod client;
pub mod auth_helpers;
pub mod auth;

2
src/server/pull.rs
View File

@ -22,10 +22,10 @@ use pbs_datastore::manifest::{
CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, FileInfo, archive_type
};
use pbs_tools::sha::sha256;
use pbs_client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader};
use crate::{
backup::DataStore,
client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader},
server::WorkerTask,
tools::ParallelHandler,
};

4
src/server/rest.rs
View File

@ -30,6 +30,8 @@ use proxmox::api::{
};
use proxmox::http_err;
use pbs_tools::compression::{DeflateEncoder, Level};
use super::auth::AuthError;
use super::environment::RestEnvironment;
use super::formatter::*;
@ -39,7 +41,7 @@ use crate::api2::types::{Authid, Userid};
use crate::auth_helpers::*;
use crate::config::cached_user_info::CachedUserInfo;
use crate::tools;
use crate::tools::compression::{CompressionMethod, DeflateEncoder, Level};
use crate::tools::compression::CompressionMethod;
use crate::tools::AsyncReaderStream;
use crate::tools::FileLogger;

193
src/tools/compression.rs
View File

@ -1,19 +1,5 @@
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use anyhow::{bail, Error};
use bytes::Bytes;
use flate2::{Compress, Compression, FlushCompress};
use futures::ready;
use futures::stream::Stream;
use hyper::header;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use proxmox::io_format_err;
use proxmox::tools::byte_buffer::ByteBuffer;
const BUFFER_SIZE: usize = 8192;
/// Possible Compression Methods, order determines preference (later is preferred)
#[derive(Eq, Ord, PartialEq, PartialOrd, Debug)]
@ -51,182 +37,3 @@ impl std::str::FromStr for CompressionMethod {
}
}
}
pub enum Level {
Fastest,
Best,
Default,
Precise(u32),
}
#[derive(Eq, PartialEq)]
enum EncoderState {
Reading,
Writing,
Flushing,
Finished,
}
pub struct DeflateEncoder<T> {
inner: T,
compressor: Compress,
buffer: ByteBuffer,
input_buffer: Bytes,
state: EncoderState,
}
impl<T> DeflateEncoder<T> {
pub fn new(inner: T) -> Self {
Self::with_quality(inner, Level::Default)
}
pub fn with_quality(inner: T, level: Level) -> Self {
let level = match level {
Level::Fastest => Compression::fast(),
Level::Best => Compression::best(),
Level::Default => Compression::new(3),
Level::Precise(val) => Compression::new(val),
};
Self {
inner,
compressor: Compress::new(level, false),
buffer: ByteBuffer::with_capacity(BUFFER_SIZE),
input_buffer: Bytes::new(),
state: EncoderState::Reading,
}
}
pub fn total_in(&self) -> u64 {
self.compressor.total_in()
}
pub fn total_out(&self) -> u64 {
self.compressor.total_out()
}
pub fn into_inner(self) -> T {
self.inner
}
fn encode(
&mut self,
inbuf: &[u8],
flush: FlushCompress,
) -> Result<(usize, flate2::Status), io::Error> {
let old_in = self.compressor.total_in();
let old_out = self.compressor.total_out();
let res = self
.compressor
.compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?;
let new_in = (self.compressor.total_in() - old_in) as usize;
let new_out = (self.compressor.total_out() - old_out) as usize;
self.buffer.add_size(new_out);
Ok((new_in, res))
}
}
impl DeflateEncoder<Vec<u8>> {
// assume small files
pub async fn compress_vec<R>(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error>
where
R: AsyncRead + Unpin,
{
let mut buffer = Vec::with_capacity(size_hint);
reader.read_to_end(&mut buffer).await?;
self.inner.reserve(size_hint); // should be enough since we want smalller files
self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?;
Ok(())
}
}
impl<T: AsyncWrite + Unpin> DeflateEncoder<T> {
pub async fn compress<R>(&mut self, reader: &mut R) -> Result<(), Error>
where
R: AsyncRead + Unpin,
{
let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE);
let mut eof = false;
loop {
if !eof && !buffer.is_full() {
let read = buffer.read_from_async(reader).await?;
if read == 0 {
eof = true;
}
}
let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?;
buffer.consume(read);
self.inner.write_all(&self.buffer[..]).await?;
self.buffer.clear();
if buffer.is_empty() && eof {
break;
}
}
loop {
let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?;
self.inner.write_all(&self.buffer[..]).await?;
self.buffer.clear();
if res == flate2::Status::StreamEnd {
break;
}
}
Ok(())
}
}
impl<T, O> Stream for DeflateEncoder<T>
where
T: Stream<Item = Result<O, io::Error>> + Unpin,
O: Into<Bytes>
{
type Item = Result<Bytes, io::Error>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
loop {
match this.state {
EncoderState::Reading => {
if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) {
let buf = res?;
this.input_buffer = buf.into();
this.state = EncoderState::Writing;
} else {
this.state = EncoderState::Flushing;
}
}
EncoderState::Writing => {
if this.input_buffer.is_empty() {
return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
}
let mut buf = this.input_buffer.split_off(0);
let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
this.input_buffer = buf.split_off(read);
if this.input_buffer.is_empty() {
this.state = EncoderState::Reading;
}
if this.buffer.is_full() || res == flate2::Status::BufError {
let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
return Poll::Ready(Some(Ok(bytes.into())));
}
}
EncoderState::Flushing => {
let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?;
if !this.buffer.is_empty() {
let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
return Poll::Ready(Some(Ok(bytes.into())));
}
if res == flate2::Status::StreamEnd {
this.state = EncoderState::Finished;
}
}
EncoderState::Finished => return Poll::Ready(None),
}
}
}
}

87
src/tools/mod.rs
View File

@ -2,8 +2,6 @@
//!
//! This is a collection of small and useful tools.
use std::any::Any;
use std::collections::HashMap;
use std::hash::BuildHasher;
use std::fs::File;
use std::io::{self, BufRead};
use std::os::unix::io::RawFd;
@ -29,7 +27,6 @@ pub use pbs_tools::process_locker::{
ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard
};
pub mod acl;
pub mod apt;
pub mod async_io;
pub mod compression;
@ -51,8 +48,6 @@ pub mod statistics;
pub mod subscription;
pub mod systemd;
pub mod ticket;
pub mod xattr;
pub mod zip;
pub mod sgutils2;
pub mod paperkey;
@ -69,6 +64,7 @@ mod file_logger;
pub use file_logger::{FileLogger, FileLogOptions};
pub use pbs_tools::broadcast_future::{BroadcastData, BroadcastFuture};
pub use pbs_tools::ops::ControlFlow;
/// The `BufferedRead` trait provides a single function
/// `buffered_read`. It returns a reference to an internal buffer. The
@ -122,65 +118,6 @@ pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [
}
}
pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
where
S: BuildHasher,
{
let mut result = vec![];
use nix::fcntl::AtFlags;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
Err(_) => false,
};
if !is_dir {
if let Some(parent) = dirname.parent() {
dirname = parent.to_owned();
}
}
let mut dir =
match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
Ok(d) => d,
Err(_) => return result,
};
for item in dir.iter() {
if let Ok(entry) = item {
if let Ok(name) = entry.file_name().to_str() {
if name == "." || name == ".." {
continue;
}
let mut newpath = dirname.clone();
newpath.push(name);
if let Ok(stat) =
nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
{
if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
newpath.push("");
if let Some(newpath) = newpath.to_str() {
result.push(newpath.to_owned());
}
continue;
}
}
if let Some(newpath) = newpath.to_str() {
result.push(newpath.to_owned());
}
}
}
}
result
}
/// Shortcut for md5 sums.
pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
hash(MessageDigest::md5(), data).map_err(Error::from)
@ -373,17 +310,6 @@ pub fn setup_safe_path_env() {
}
}
pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
Some(n) => &line[n..],
None => return &[],
};
match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
Some(n) => &line[..(line.len() - n)],
None => &[],
}
}
/// Create the base run-directory.
///
/// This exists to fixate the permissions for the run *base* directory while allowing intermediate
@ -396,14 +322,3 @@ pub fn create_run_dir() -> Result<(), Error> {
let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
Ok(())
}
/// Modeled after the nightly `std::ops::ControlFlow`.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ControlFlow<B, C = ()> {
Continue(C),
Break(B),
}
impl<B> ControlFlow<B> {
pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
}

5
tests/catar.rs
View File

@ -1,7 +1,8 @@
use std::process::Command;
use anyhow::{Error};
use std::process::Command;
use proxmox_backup::pxar::*;
use pbs_client::pxar::*;
fn run_test(dir_name: &str) -> Result<(), Error> {