tsarevvv/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add traffic control configuration config with API

Browse Source 
Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
This commit is contained in:
Dietmar Maurer 2021-11-06 18:46:58 +01:00
parent 0c136bfab1
commit bfd12e871f
11 changed files with 577 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pbs-api-types/Cargo.toml
View File

@ -16,7 +16,7 @@ serde = { version = "1.0", features = ["derive"] }
proxmox = "0.15.0"
proxmox-lang = "1.0.0"
proxmox-schema = { version = "1.0.0", features = [ "api-macro" ] }
proxmox-schema = { version = "1.0.1", features = [ "api-macro" ] }
proxmox-time = "1.0.0"
proxmox-uuid = { version = "1.0.0", features = [ "serde" ] }

7
pbs-api-types/src/lib.rs
View File

@ -7,6 +7,7 @@ use proxmox_schema::{
api, const_regex, ApiStringFormat, ApiType, ArraySchema, Schema, StringSchema, ReturnType,
};
use proxmox::{IPRE, IPRE_BRACKET, IPV4OCTET, IPV4RE, IPV6H16, IPV6LS32, IPV6RE};
use proxmox_systemd::daily_duration::parse_daily_duration;
#[rustfmt::skip]
#[macro_export]
@ -73,6 +74,9 @@ pub use remote::*;
mod tape;
pub use tape::*;
mod traffic_control;
pub use traffic_control::*;
mod zfs;
pub use zfs::*;
@ -152,6 +156,9 @@ pub const HOSTNAME_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&HOSTNAME_
pub const DNS_ALIAS_FORMAT: ApiStringFormat =
ApiStringFormat::Pattern(&DNS_ALIAS_REGEX);
pub const DAILY_DURATION_FORMAT: ApiStringFormat =
ApiStringFormat::VerifyFn(|s| parse_daily_duration(s).map(drop));
pub const SEARCH_DOMAIN_SCHEMA: Schema =
StringSchema::new("Search domain for host-name lookup.").schema();

96
pbs-api-types/src/traffic_control.rs Normal file
View File

@ -0,0 +1,96 @@
use serde::{Deserialize, Serialize};
use proxmox_schema::{api, Schema, IntegerSchema, StringSchema, Updater};
use crate::{
CIDR_SCHEMA, DAILY_DURATION_FORMAT,
PROXMOX_SAFE_ID_FORMAT, SINGLE_LINE_COMMENT_SCHEMA,
};
pub const TRAFFIC_CONTROL_TIMEFRAME_SCHEMA: Schema = StringSchema::new(
"Timeframe to specify when the rule is actice.")
.format(&DAILY_DURATION_FORMAT)
.schema();
pub const TRAFFIC_CONTROL_ID_SCHEMA: Schema = StringSchema::new("Rule ID.")
.format(&PROXMOX_SAFE_ID_FORMAT)
.min_length(3)
.max_length(32)
.schema();
pub const TRAFFIC_CONTROL_RATE_SCHEMA: Schema = IntegerSchema::new(
"Rate limit (for Token bucket filter) in bytes/second.")
.minimum(100_000)
.schema();
pub const TRAFFIC_CONTROL_BURST_SCHEMA: Schema = IntegerSchema::new(
"Size of the token bucket (for Token bucket filter) in bytes.")
.minimum(1000)
.schema();
#[api(
properties: {
name: {
schema: TRAFFIC_CONTROL_ID_SCHEMA,
},
comment: {
optional: true,
schema: SINGLE_LINE_COMMENT_SCHEMA,
},
"rate-in": {
schema: TRAFFIC_CONTROL_RATE_SCHEMA,
optional: true,
},
"burst-in": {
schema: TRAFFIC_CONTROL_BURST_SCHEMA,
optional: true,
},
"rate-out": {
schema: TRAFFIC_CONTROL_RATE_SCHEMA,
optional: true,
},
"burst-out": {
schema: TRAFFIC_CONTROL_BURST_SCHEMA,
optional: true,
},
network: {
type: Array,
items: {
schema: CIDR_SCHEMA,
},
},
timeframe: {
type: Array,
items: {
schema: TRAFFIC_CONTROL_TIMEFRAME_SCHEMA,
},
optional: true,
},
},
)]
#[derive(Serialize,Deserialize, Updater)]
#[serde(rename_all = "kebab-case")]
/// Traffic control rule
pub struct TrafficControlRule {
#[updater(skip)]
pub name: String,
#[serde(skip_serializing_if="Option::is_none")]
pub comment: Option<String>,
/// Rule applies to Source IPs within this networks
pub network: Vec<String>,
#[serde(skip_serializing_if="Option::is_none")]
pub rate_in: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub burst_in: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub rate_out: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub burst_out: Option<u64>,
// fixme: expose this?
// /// Bandwidth is shared accross all connections
// #[serde(skip_serializing_if="Option::is_none")]
// pub shared: Option<bool>,
/// Enable the rule at specific times
#[serde(skip_serializing_if="Option::is_none")]
pub timeframe: Option<Vec<String>>,
}

1
pbs-config/src/lib.rs
View File

@ -12,6 +12,7 @@ pub mod sync;
pub mod tape_encryption_keys;
pub mod tape_job;
pub mod token_shadow;
pub mod traffic_control;
pub mod user;
pub mod verify;

90
pbs-config/src/traffic_control.rs Normal file
View File

@ -0,0 +1,90 @@
//! Traffic Control Settings (Network rate limits)
use std::collections::HashMap;
use anyhow::Error;
use lazy_static::lazy_static;
use proxmox_schema::{ApiType, Schema};
use pbs_api_types::{TrafficControlRule, TRAFFIC_CONTROL_ID_SCHEMA};
use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
use crate::{open_backup_lockfile, replace_backup_config, BackupLockGuard};
lazy_static! {
/// Static [`SectionConfig`] to access parser/writer functions.
pub static ref CONFIG: SectionConfig = init();
}
fn init() -> SectionConfig {
let mut config = SectionConfig::new(&TRAFFIC_CONTROL_ID_SCHEMA);
let obj_schema = match TrafficControlRule::API_SCHEMA {
Schema::Object(ref obj_schema) => obj_schema,
_ => unreachable!(),
};
let plugin = SectionConfigPlugin::new("rule".to_string(), Some("name".to_string()), obj_schema);
config.register_plugin(plugin);
config
}
/// Configuration file name
pub const TRAFFIC_CONTROL_CFG_FILENAME: &str = "/etc/proxmox-backup/traffic-control.cfg";
/// Lock file name (used to prevent concurrent access)
pub const TRAFFIC_CONTROL_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.traffic-control.lck";
/// Get exclusive lock
pub fn lock_config() -> Result<BackupLockGuard, Error> {
open_backup_lockfile(TRAFFIC_CONTROL_CFG_LOCKFILE, None, true)
}
/// Read and parse the configuration file
pub fn config() -> Result<(SectionConfigData, [u8;32]), Error> {
let content = proxmox::tools::fs::file_read_optional_string(TRAFFIC_CONTROL_CFG_FILENAME)?
.unwrap_or_else(|| "".to_string());
let digest = openssl::sha::sha256(content.as_bytes());
let data = CONFIG.parse(TRAFFIC_CONTROL_CFG_FILENAME, &content)?;
Ok((data, digest))
}
/// Save the configuration file
pub fn save_config(config: &SectionConfigData) -> Result<(), Error> {
let raw = CONFIG.write(TRAFFIC_CONTROL_CFG_FILENAME, &config)?;
replace_backup_config(TRAFFIC_CONTROL_CFG_FILENAME, raw.as_bytes())
}
// shell completion helper
pub fn complete_traffic_control_name(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
match config() {
Ok((data, _digest)) => data.sections.iter().map(|(id, _)| id.to_string()).collect(),
Err(_) => return vec![],
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test1() -> Result<(), Error> {
let content = "rule: rule1
comment localnet at working hours
network 192.168.2.0/24
network 192.168.3.0/24
rate-in 500000
timeframe mon..wed 8:00-16:30
timeframe fri 9:00-12:00
";
let data = CONFIG.parse(TRAFFIC_CONTROL_CFG_FILENAME, &content)?;
eprintln!("GOT {:?}", data);
Ok(())
}
}

26
proxmox-backup-client/src/main.rs
View File

@ -20,8 +20,10 @@ use proxmox_time::{strftime_local, epoch_i64};
use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
use pbs_api_types::{
BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem,
PruneListItem, SnapshotListItem, StorageStatus, Fingerprint, PruneOptions,
BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA,
TRAFFIC_CONTROL_BURST_SCHEMA, TRAFFIC_CONTROL_RATE_SCHEMA,
Authid, CryptMode, Fingerprint, GroupListItem, PruneListItem, PruneOptions,
SnapshotListItem, StorageStatus,
};
use pbs_client::{
BACKUP_SOURCE_SCHEMA,
@ -583,16 +585,12 @@ fn spawn_catalog_upload(
optional: true,
},
rate: {
type: u64,
description: "Rate limit for TBF in bytes/second.",
schema: TRAFFIC_CONTROL_RATE_SCHEMA,
optional: true,
minimum: 1,
},
burst: {
type: u64,
description: "Size of the TBF bucket, in bytes.",
schema: TRAFFIC_CONTROL_BURST_SCHEMA,
optional: true,
minimum: 1,
},
"exclude": {
type: Array,
@ -1056,6 +1054,14 @@ We do not extract '.pxar' archives when writing to standard output.
"###
},
rate: {
schema: TRAFFIC_CONTROL_RATE_SCHEMA,
optional: true,
},
burst: {
schema: TRAFFIC_CONTROL_BURST_SCHEMA,
optional: true,
},
"allow-existing-dirs": {
type: Boolean,
description: "Do not fail if directories already exists.",
@ -1086,8 +1092,10 @@ async fn restore(param: Value) -> Result<Value, Error> {
let archive_name = json::required_string_param(&param, "archive-name")?;
let client = connect(&repo)?;
let rate_limit = param["rate"].as_u64();
let bucket_size = param["burst"].as_u64();
let client = connect_rate_limited(&repo, rate_limit, bucket_size)?;
record_repository(&repo);
let path = json::required_string_param(&param, "snapshot")?;

2
src/api2/config/mod.rs
View File

@ -14,6 +14,7 @@ pub mod changer;
pub mod media_pool;
pub mod tape_encryption_keys;
pub mod tape_backup_job;
pub mod traffic_control;
const SUBDIRS: SubdirMap = &[
("access", &access::ROUTER),
@ -26,6 +27,7 @@ const SUBDIRS: SubdirMap = &[
("sync", &sync::ROUTER),
("tape-backup-job", &tape_backup_job::ROUTER),
("tape-encryption-keys", &tape_encryption_keys::ROUTER),
("traffic-control", &traffic_control::ROUTER),
("verify", &verify::ROUTER),
];

253
src/api2/config/traffic_control.rs Normal file
View File

@ -0,0 +1,253 @@
use anyhow::{bail, Error};
use serde_json::Value;
use ::serde::{Deserialize, Serialize};
use proxmox_router::{ApiMethod, Router, RpcEnvironment, Permission};
use proxmox_schema::api;
use pbs_api_types::{
TrafficControlRule, TrafficControlRuleUpdater,
PROXMOX_CONFIG_DIGEST_SCHEMA, TRAFFIC_CONTROL_ID_SCHEMA,
PRIV_SYS_AUDIT, PRIV_SYS_MODIFY,
};
#[api(
input: {
properties: {},
},
returns: {
description: "The list of configured traffic control rules (with config digest).",
type: Array,
items: { type: TrafficControlRule },
},
access: {
permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
},
)]
/// List traffic control rules
pub fn list_traffic_controls(
_param: Value,
_info: &ApiMethod,
mut rpcenv: &mut dyn RpcEnvironment,
) -> Result<Vec<TrafficControlRule>, Error> {
let (config, digest) = pbs_config::traffic_control::config()?;
let list: Vec<TrafficControlRule> = config.convert_to_typed_array("rule")?;
rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
Ok(list)
}
#[api(
protected: true,
input: {
properties: {
config: {
type: TrafficControlRule,
flatten: true,
},
},
},
access: {
permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
},
)]
/// Create new traffic control rule.
pub fn create_traffic_control(config: TrafficControlRule) -> Result<(), Error> {
let _lock = pbs_config::traffic_control::lock_config()?;
let (mut section_config, _digest) = pbs_config::traffic_control::config()?;
if section_config.sections.get(&config.name).is_some() {
bail!("traffic control rule '{}' already exists.", config.name);
}
section_config.set_data(&config.name, "rule", &config)?;
pbs_config::traffic_control::save_config(&section_config)?;
Ok(())
}
#[api(
input: {
properties: {
name: {
schema: TRAFFIC_CONTROL_ID_SCHEMA,
},
},
},
returns: { type: TrafficControlRule },
access: {
permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
}
)]
/// Read traffic control configuration data.
pub fn read_traffic_control(
name: String,
_info: &ApiMethod,
mut rpcenv: &mut dyn RpcEnvironment,
) -> Result<TrafficControlRule, Error> {
let (config, digest) = pbs_config::traffic_control::config()?;
let data: TrafficControlRule = config.lookup("rule", &name)?;
rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
Ok(data)
}
#[api()]
#[derive(Serialize, Deserialize)]
#[allow(non_camel_case_types)]
/// Deletable property name
pub enum DeletableProperty {
/// Delete the rate_in property.
rate_in,
/// Delete the burst_in property.
burst_in,
/// Delete the rate_out property.
rate_out,
/// Delete the burst_out property.
burst_out,
/// Delete the comment property.
comment,
/// Delete the timeframe property
timeframe,
}
// fixme: use TrafficControlUpdater
#[api(
protected: true,
input: {
properties: {
name: {
schema: TRAFFIC_CONTROL_ID_SCHEMA,
},
update: {
type: TrafficControlRuleUpdater,
flatten: true,
},
delete: {
description: "List of properties to delete.",
type: Array,
optional: true,
items: {
type: DeletableProperty,
}
},
digest: {
optional: true,
schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
},
},
},
access: {
permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
},
)]
/// Update traffic control configuration.
pub fn update_traffic_control(
name: String,
update: TrafficControlRuleUpdater,
delete: Option<Vec<DeletableProperty>>,
digest: Option<String>,
) -> Result<(), Error> {
let _lock = pbs_config::traffic_control::lock_config()?;
let (mut config, expected_digest) = pbs_config::traffic_control::config()?;
if let Some(ref digest) = digest {
let digest = proxmox::tools::hex_to_digest(digest)?;
crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
}
let mut data: TrafficControlRule = config.lookup("rule", &name)?;
if let Some(delete) = delete {
for delete_prop in delete {
match delete_prop {
DeletableProperty::rate_in => { data.rate_in = None; },
DeletableProperty::rate_out => { data.rate_out = None; },
DeletableProperty::burst_in => { data.burst_in = None; },
DeletableProperty::burst_out => { data.burst_out = None; },
DeletableProperty::comment => { data.comment = None; },
DeletableProperty::timeframe => { data.timeframe = None; },
}
}
}
if let Some(comment) = update.comment {
let comment = comment.trim().to_string();
if comment.is_empty() {
data.comment = None;
} else {
data.comment = Some(comment);
}
}
if update.rate_in.is_some() { data.rate_in = update.rate_in; }
if update.rate_out.is_some() { data.rate_out = update.rate_out; }
if update.burst_in.is_some() { data.burst_in = update.burst_in; }
if update.burst_out.is_some() { data.burst_out = update.burst_out; }
if let Some(network) = update.network { data.network = network; }
if update.timeframe.is_some() { data.timeframe = update.timeframe; }
config.set_data(&name, "rule", &data)?;
pbs_config::traffic_control::save_config(&config)?;
Ok(())
}
#[api(
protected: true,
input: {
properties: {
name: {
schema: TRAFFIC_CONTROL_ID_SCHEMA,
},
digest: {
optional: true,
schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
},
},
},
access: {
permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
},
)]
/// Remove a traffic control rule from the configuration file.
pub fn delete_traffic_control(name: String, digest: Option<String>) -> Result<(), Error> {
let _lock = pbs_config::traffic_control::lock_config()?;
let (mut config, expected_digest) = pbs_config::traffic_control::config()?;
if let Some(ref digest) = digest {
let digest = proxmox::tools::hex_to_digest(digest)?;
crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
}
match config.sections.get(&name) {
Some(_) => { config.sections.remove(&name); },
None => bail!("traffic control rule '{}' does not exist.", name),
}
pbs_config::traffic_control::save_config(&config)?;
Ok(())
}
const ITEM_ROUTER: Router = Router::new()
.get(&API_METHOD_READ_TRAFFIC_CONTROL)
.put(&API_METHOD_UPDATE_TRAFFIC_CONTROL)
.delete(&API_METHOD_DELETE_TRAFFIC_CONTROL);
pub const ROUTER: Router = Router::new()
.get(&API_METHOD_LIST_TRAFFIC_CONTROLS)
.post(&API_METHOD_CREATE_TRAFFIC_CONTROL)
.match_all("name", &ITEM_ROUTER);

1
src/bin/proxmox-backup-manager.rs
View File

@ -374,6 +374,7 @@ async fn run() -> Result<(), Error> {
.insert("user", user_commands())
.insert("openid", openid_commands())
.insert("remote", remote_commands())
.insert("traffic-control", traffic_control_commands())
.insert("garbage-collection", garbage_collection_commands())
.insert("acme", acme_mgmt_cli())
.insert("cert", cert_mgmt_cli())

2
src/bin/proxmox_backup_manager/mod.rs
View File

@ -26,3 +26,5 @@ mod node;
pub use node::*;
mod openid;
pub use openid::*;
mod traffic_control;
pub use traffic_control::*;

107
src/bin/proxmox_backup_manager/traffic_control.rs Normal file
View File

@ -0,0 +1,107 @@
use anyhow::Error;
use serde_json::Value;
use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
use proxmox_schema::api;
use pbs_api_types::TRAFFIC_CONTROL_ID_SCHEMA;
use proxmox_backup::api2;
#[api(
input: {
properties: {
"output-format": {
schema: OUTPUT_FORMAT,
optional: true,
},
}
}
)]
/// List configured traffic control rules.
fn list_traffic_controls(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
let output_format = get_output_format(&param);
let info = &api2::config::traffic_control::API_METHOD_LIST_TRAFFIC_CONTROLS;
let mut data = match info.handler {
ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
_ => unreachable!(),
};
let options = default_table_format_options()
.column(ColumnConfig::new("name"))
.column(ColumnConfig::new("rate-in"))
.column(ColumnConfig::new("burst-in"))
.column(ColumnConfig::new("rate-out"))
.column(ColumnConfig::new("burst-out"))
.column(ColumnConfig::new("network"))
.column(ColumnConfig::new("timeframe"))
.column(ColumnConfig::new("comment"));
format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
Ok(Value::Null)
}
#[api(
input: {
properties: {
name: {
schema: TRAFFIC_CONTROL_ID_SCHEMA,
},
"output-format": {
schema: OUTPUT_FORMAT,
optional: true,
},
}
}
)]
/// Show traffic control configuration
fn show_traffic_control(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
let output_format = get_output_format(&param);
let info = &api2::config::traffic_control::API_METHOD_READ_TRAFFIC_CONTROL;
let mut data = match info.handler {
ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
_ => unreachable!(),
};
let options = default_table_format_options();
format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
Ok(Value::Null)
}
pub fn traffic_control_commands() -> CommandLineInterface {
let cmd_def = CliCommandMap::new()
.insert("list", CliCommand::new(&API_METHOD_LIST_TRAFFIC_CONTROLS))
.insert(
"show",
CliCommand::new(&API_METHOD_SHOW_TRAFFIC_CONTROL)
.arg_param(&["name"])
.completion_cb("name", pbs_config::traffic_control::complete_traffic_control_name)
)
.insert(
"create",
CliCommand::new(&api2::config::traffic_control::API_METHOD_CREATE_TRAFFIC_CONTROL)
.arg_param(&["name"])
)
.insert(
"update",
CliCommand::new(&api2::config::traffic_control::API_METHOD_UPDATE_TRAFFIC_CONTROL)
.arg_param(&["name"])
.completion_cb("name", pbs_config::traffic_control::complete_traffic_control_name)
)
.insert(
"remove",
CliCommand::new(&api2::config::traffic_control::API_METHOD_DELETE_TRAFFIC_CONTROL)
.arg_param(&["name"])
.completion_cb("name", pbs_config::traffic_control::complete_traffic_control_name)
);
cmd_def.into()
}