tsarevvv/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 3.1.4-1

Browse Source
This commit is contained in:
Andrew A. Vasilyev 2024-02-02 19:33:27 +03:00
commit 33961cbe87
61 changed files with 1416 additions and 2962 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.toml
View File

@ -1,5 +1,5 @@
[workspace.package]
version = "3.1.2"
version = "3.1.4"
authors = [
"Dietmar Maurer <dietmar@proxmox.com>",
"Dominik Csapak <d.csapak@proxmox.com>",
@ -43,7 +43,6 @@ members = [
"proxmox-backup-client",
"proxmox-file-restore",
"proxmox-restore-daemon",
"proxmox-rrd",
"pxar-bin",
]
@ -70,6 +69,7 @@ proxmox-openid = "0.10.0"
proxmox-rest-server = { version = "0.5.1", features = [ "templates" ] }
# some use "cli", some use "cli" and "server", pbs-config uses nothing
proxmox-router = { version = "2.0.0", default_features = false }
proxmox-rrd = { version = "0.1" }
# everything but pbs-config and pbs-client use "api-macro"
proxmox-schema = "2.0.0"
proxmox-section-config = "2"
@ -77,9 +77,9 @@ proxmox-serde = "0.1.1"
proxmox-shared-memory = "0.3.0"
proxmox-sortable-macro = "0.1.2"
proxmox-subscription = { version = "0.4.2", features = [ "api-types" ] }
proxmox-sys = "0.5.2"
proxmox-sys = "0.5.3"
proxmox-tfa = { version = "4.0.4", features = [ "api", "api-types" ] }
proxmox-time = "1.1.2"
proxmox-time = "1.1.6"
proxmox-uuid = "1"
# other proxmox crates
@ -98,7 +98,6 @@ pbs-key-config = { path = "pbs-key-config" }
pbs-pxar-fuse = { path = "pbs-pxar-fuse" }
pbs-tape = { path = "pbs-tape" }
pbs-tools = { path = "pbs-tools" }
proxmox-rrd = { path = "proxmox-rrd" }
# regular crates
anyhow = "1.0"
@ -133,7 +132,6 @@ pin-project-lite = "0.2"
regex = "1.5.5"
rustyline = "9"
serde = { version = "1.0", features = ["derive"] }
serde_cbor = "0.11.1"
serde_json = "1.0"
serde_plain = "1"
siphasher = "0.3"
@ -260,6 +258,7 @@ proxmox-rrd.workspace = true
#proxmox-openid = { path = "../proxmox/proxmox-openid" }
#proxmox-rest-server = { path = "../proxmox/proxmox-rest-server" }
#proxmox-router = { path = "../proxmox/proxmox-router" }
#proxmox-rrd = { path = "../proxmox/proxmox-rrd" }
#proxmox-schema = { path = "../proxmox/proxmox-schema" }
#proxmox-section-config = { path = "../proxmox/proxmox-section-config" }
#proxmox-serde = { path = "../proxmox/proxmox-serde" }
@ -271,7 +270,7 @@ proxmox-rrd.workspace = true
#proxmox-time = { path = "../proxmox/proxmox-time" }
#proxmox-uuid = { path = "../proxmox/proxmox-uuid" }
#proxmox-acme-rs = { path = "../proxmox-acme-rs" }
#proxmox-acme = { path = "../proxmox/proxmox-acme" }
#pathpatterns = {path = "../pathpatterns" }
#pxar = { path = "../pxar" }

73
debian/changelog vendored
View File

@ -1,3 +1,76 @@
rust-proxmox-backup (3.1.4-1) bookworm; urgency=medium
* api: acme: skip serializing empty 'api' and 'data' option
* tape: fix regression in restoring an encryption key from medium, avoid
trying to load the key to the drive, which cannot work in this special
case.
-- Proxmox Support Team <support@proxmox.com> Thu, 01 Feb 2024 16:30:18 +0100
rust-proxmox-backup (3.1.3-1) bookworm; urgency=medium
* improve efficiency of detecting if a block device is a partition
* acme: api: add option for external account binding to account registration
endpoint
* ui: datastore summary handle non-existent 'avail' and 'used' status values
* tape: allow configuring the time out for "wait until ready" to better cope
with the long initialization duration that happens on the first use of
LTO 9+ tapes
* tape: improve error on decode element status page
* tape: improve LTO 9 compatibility
* fix #4904: tape changer: add option to explicitly eject the tape before
unloading it
* docs: tape: replace use of 'export-media' with correct 'export-media-set'
option
* docs: tape: add LTO 9 considerations
* fix #5117: ui: node info: avoid invalid array access for certain foreign
kernels
* d/control: explicitly depend on gdisk package to ensure it's available
when installing on top of a plain Debian installation
* tape: work around buggy changer implementations when reading the element
status
* system report: include prune.cfg
* fix #4315: jobs: modify GroupFilter so include/exclude is tracked
* ui: show if Filter includes or excludes
* datastore: add additional context for a parsing error when getting the
owner of a backup group
* api: tape: optionally accept uuid for destroying or moving a media, so
that one can uniquely identify existing tapes with duplicate labels.
* api: tape: don't allow duplicate media label-texts anymore
* ui: tape inventory: use uuid as id
* ui: tape: add button to remove a medium from the inventory, while not
touching the data
* api: custom certificate upload: make key optional and use the existing
key, if it's not specified.
* close #4819: ui: allow usernames shorter than 4 characters
* tape: rework on-drive encryption key handling and ensure this key does not
gets unloaded to early
-- Proxmox Support Team <support@proxmox.com> Mon, 22 Jan 2024 15:20:45 +0100
rust-proxmox-backup (3.1.2-1) bookworm; urgency=medium
* sync: fix recent regression with recursive remote sync

17
debian/control vendored
View File

@ -48,7 +48,7 @@ Build-Depends: bash-completion,
librust-pathpatterns-0.3+default-dev,
librust-percent-encoding-2+default-dev (>= 2.1-~~),
librust-pin-project-lite-0.2+default-dev,
librust-proxmox-acme-rs-0.4+default-dev,
librust-proxmox-acme-0.5+default-dev,
librust-proxmox-apt-0.10+default-dev (>= 0.10.5-~~),
librust-proxmox-async-0.4+default-dev,
librust-proxmox-auth-api-0.3+api-dev,
@ -79,6 +79,7 @@ Build-Depends: bash-completion,
librust-proxmox-router-2+cli-dev,
librust-proxmox-router-2+default-dev,
librust-proxmox-router-2+server-dev,
librust-proxmox-rrd-0.1+default-dev,
librust-proxmox-schema-2+api-macro-dev,
librust-proxmox-schema-2+default-dev,
librust-proxmox-section-config-2+default-dev,
@ -88,15 +89,15 @@ Build-Depends: bash-completion,
librust-proxmox-sortable-macro-0.1+default-dev (>= 0.1.2-~~),
librust-proxmox-subscription-0.4+api-types-dev (>= 0.4.2-~~),
librust-proxmox-subscription-0.4+default-dev (>= 0.4.2-~~),
librust-proxmox-sys-0.5+acl-dev (>= 0.5.2-~~),
librust-proxmox-sys-0.5+crypt-dev (>= 0.5.2-~~),
librust-proxmox-sys-0.5+default-dev (>= 0.5.2-~~),
librust-proxmox-sys-0.5+logrotate-dev (>= 0.5.2-~~),
librust-proxmox-sys-0.5+timer-dev (>= 0.5.2-~~),
librust-proxmox-sys-0.5+acl-dev (>= 0.5.3-~~),
librust-proxmox-sys-0.5+crypt-dev (>= 0.5.3-~~),
librust-proxmox-sys-0.5+default-dev (>= 0.5.3-~~),
librust-proxmox-sys-0.5+logrotate-dev (>= 0.5.3-~~),
librust-proxmox-sys-0.5+timer-dev (>= 0.5.3-~~),
librust-proxmox-tfa-4+api-dev (>= 4.0.4-~~),
librust-proxmox-tfa-4+api-types-dev (>= 4.0.4-~~),
librust-proxmox-tfa-4+default-dev (>= 4.0.4-~~),
librust-proxmox-time-1+default-dev (>= 1.1.2-~~),
librust-proxmox-time-1+default-dev (>= 1.1.6-~~),
librust-proxmox-uuid-1+default-dev,
librust-proxmox-uuid-1+serde-dev,
librust-pxar-0.10+default-dev (>= 0.10.2-~~),
@ -104,7 +105,6 @@ Build-Depends: bash-completion,
librust-rustyline-9+default-dev,
librust-serde-1+default-dev,
librust-serde-1+derive-dev,
librust-serde-cbor-0.11+default-dev (>= 0.11.1-~~),
librust-serde-json-1+default-dev,
librust-serde-plain-1+default-dev,
librust-siphasher-0.3+default-dev,
@ -163,6 +163,7 @@ Rules-Requires-Root: binary-targets
Package: proxmox-backup-server
Architecture: any
Depends: fonts-font-awesome,
gdisk,
libjs-extjs (>= 7~),
libjs-qrcodejs (>= 1.20201119),
libproxmox-acme-plugins,

1
debian/lintian-overrides vendored
View File

@ -4,4 +4,5 @@ proxmox-backup-server: elevated-privileges 4755 root/root [usr/lib/x86_64-linux-
proxmox-backup-server: systemd-service-file-refers-to-unusual-wantedby-target getty.target [lib/systemd/system/proxmox-backup-banner.service]
proxmox-backup-server: uses-dpkg-database-directly [usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-api]
proxmox-backup-server: uses-dpkg-database-directly [usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-proxy]
proxmox-backup-server: uses-dpkg-database-directly [usr/sbin/pbs2to3]
proxmox-backup-server: uses-dpkg-database-directly [usr/sbin/proxmox-backup-debug]

14
docs/managing-remotes.rst
View File

@ -116,6 +116,20 @@ of the specified criteria are synced. The available criteria are:
The same filter is applied to local groups, for handling of the
``remove-vanished`` option.
A ``group-filter`` can be inverted by prepending ``exclude:`` to it.
* Regular expression example, excluding the match:
.. code-block:: console
# proxmox-backup-manager sync-job update ID --group-filter exclude:regex:'^vm/1\d{2,3}$'
For mixing include and exclude filter, following rules apply:
- no filters: all backup groups
- include: only those matching the include filters
- exclude: all but those matching the exclude filters
- both: those matching the include filters, but without those matching the exclude filters
.. note:: The ``protected`` flag of remote backup snapshots will not be synced.
Namespace Support

52
docs/tape-backup.rst
View File

@ -98,6 +98,31 @@ so it takes 33 hours to read the 12TB needed to fill up an LTO-8 tape. If you wa
to write to your tape at full speed, please make sure that the source
datastore is able to deliver that performance (for example, by using SSDs).
LTO-9+ considerations
~~~~~~~~~~~~~~~~~~~~~
Since LTO-9, it is necessary to initialize new media in your drives, this is
called `Media Optimization`. This usually takes between 40 and 120 minutes per
medium. It is recommended to initialize your media in this manner with the
tools provided by your hardware vendor of your drive or changer. Some tape
changers have a method to 'bulk' initialize your media.
Because of this, formatting tapes is handled differently in Proxmox Backup
Server to avoid re-optimizing on each format/labelling. If you want to format
your media for use with the Proxmox Backup Server the first time or after use
with another program, either use the functionality of your drive/changer, or
use the 'slow' format on the cli:
.. code-block:: console
# proxmox-tape format --drive your-drive --fast 0
This will completely remove all pre-existing data and trigger a `Media
Optimization` pass.
If you format a partitioned LTO-9 medium with the 'fast' method (the default or
by setting `--fast 1`), only the first partition will be formatted, so make
sure to use the 'slow' method.
Terminology
-----------
@ -326,6 +351,25 @@ the status output:
│ slot │ 14 │ │ │
└───────────────┴──────────┴────────────┴─────────────┘
Advanced options
^^^^^^^^^^^^^^^^
Since not all tape changer behave the same, there is sometimes the need
for configuring advanced options.
Currently there are the following:
* `eject-before-unload` : This is needed for some changers that require a tape
to be ejected before unloading from the drive.
You can set these options with `proxmox-tape` like this:
.. code-block:: console
# proxmox-tape changer update sl3 --eject-before-unload true
.. _tape_drive_config:
Tape drives
@ -515,8 +559,6 @@ a single media pool, so a job only uses tapes from that pool.
- Create a new set when the specified Calendar Event triggers.
.. _systemd.time manpage: https://manpages.debian.org/buster/systemd/systemd.time.7.en.html
This allows you to specify points in time by using systemd like
Calendar Event specifications (see `systemd.time manpage`_).
@ -664,16 +706,16 @@ dust protection than inside a drive:
.. Note:: For failed jobs, the tape remains in the drive.
For tape libraries, the ``export-media`` option moves all tapes from
For tape libraries, the ``export-media-set`` option moves all tapes from
the media set to an export slot, making sure that the following backup
cannot use the tapes. An operator can pick up those tapes and move them
to a vault.
.. code-block:: console
# proxmox-tape backup-job update job2 --export-media
# proxmox-tape backup-job update job2 --export-media-set
.. Note:: The ``export-media`` option can be used to force the start
.. Note:: The ``export-media-set`` option can be used to force the start
of a new media set, because tapes from the current set are no
longer online.

55
pbs-api-types/src/datastore.rs
View File

@ -10,9 +10,9 @@ use proxmox_schema::{
};
use crate::{
Authid, CryptMode, Fingerprint, MaintenanceMode, Userid, DATASTORE_NOTIFY_STRING_SCHEMA,
GC_SCHEDULE_SCHEMA, PROXMOX_SAFE_ID_FORMAT, PRUNE_SCHEDULE_SCHEMA, SHA256_HEX_REGEX,
SINGLE_LINE_COMMENT_SCHEMA, UPID,
Authid, CryptMode, Fingerprint, GroupFilter, MaintenanceMode, Userid,
DATASTORE_NOTIFY_STRING_SCHEMA, GC_SCHEDULE_SCHEMA, PROXMOX_SAFE_ID_FORMAT,
PRUNE_SCHEDULE_SCHEMA, SHA256_HEX_REGEX, SINGLE_LINE_COMMENT_SCHEMA, UPID,
};
const_regex! {
@ -843,19 +843,37 @@ impl BackupGroup {
}
pub fn matches(&self, filter: &crate::GroupFilter) -> bool {
use crate::GroupFilter;
match filter {
GroupFilter::Group(backup_group) => {
use crate::FilterType;
match &filter.filter_type {
FilterType::Group(backup_group) => {
match backup_group.parse::<BackupGroup>() {
Ok(group) => *self == group,
Err(_) => false, // shouldn't happen if value is schema-checked
}
}
GroupFilter::BackupType(ty) => self.ty == *ty,
GroupFilter::Regex(regex) => regex.is_match(&self.to_string()),
FilterType::BackupType(ty) => self.ty == *ty,
FilterType::Regex(regex) => regex.is_match(&self.to_string()),
}
}
pub fn apply_filters(&self, filters: &[GroupFilter]) -> bool {
// since there will only be view filter in the list, an extra iteration to get the umber of
// include filter should not be an issue
let is_included = if filters.iter().filter(|f| !f.is_exclude).count() == 0 {
true
} else {
filters
.iter()
.filter(|f| !f.is_exclude)
.any(|filter| self.matches(filter))
};
is_included
&& !filters
.iter()
.filter(|f| f.is_exclude)
.any(|filter| self.matches(filter))
}
}
impl AsRef<BackupGroup> for BackupGroup {
@ -1302,12 +1320,15 @@ pub struct DataStoreStatus {
/// Status of a Datastore
pub struct DataStoreStatusListItem {
pub store: String,
/// The Size of the underlying storage in bytes. (-1 on error)
pub total: i64,
/// The used bytes of the underlying storage. (-1 on error)
pub used: i64,
/// The Size of the underlying storage in bytes.
#[serde(skip_serializing_if = "Option::is_none")]
pub total: Option<u64>,
/// The used bytes of the underlying storage.
#[serde(skip_serializing_if = "Option::is_none")]
pub used: Option<u64>,
/// The available bytes of the underlying storage. (-1 on error)
pub avail: i64,
#[serde(skip_serializing_if = "Option::is_none")]
pub avail: Option<u64>,
/// A list of usages of the past (last Month).
#[serde(skip_serializing_if = "Option::is_none")]
pub history: Option<Vec<Option<f64>>>,
@ -1335,9 +1356,9 @@ impl DataStoreStatusListItem {
pub fn empty(store: &str, err: Option<String>) -> Self {
DataStoreStatusListItem {
store: store.to_owned(),
total: -1,
used: -1,
avail: -1,
total: None,
used: None,
avail: None,
history: None,
history_start: None,
history_delta: None,

73
pbs-api-types/src/jobs.rs
View File

@ -1,6 +1,6 @@
use anyhow::format_err;
use std::str::FromStr;
use anyhow::bail;
use regex::Regex;
use serde::{Deserialize, Serialize};
@ -388,7 +388,7 @@ pub struct TapeBackupJobStatus {
#[derive(Clone, Debug)]
/// Filter for matching `BackupGroup`s, for use with `BackupGroup::filter`.
pub enum GroupFilter {
pub enum FilterType {
/// BackupGroup type - either `vm`, `ct`, or `host`.
BackupType(BackupType),
/// Full identifier of BackupGroup, including type
@ -397,7 +397,7 @@ pub enum GroupFilter {
Regex(Regex),
}
impl PartialEq for GroupFilter {
impl PartialEq for FilterType {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::BackupType(a), Self::BackupType(b)) => a == b,
@ -408,28 +408,69 @@ impl PartialEq for GroupFilter {
}
}
impl std::str::FromStr for FilterType {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s.split_once(':') {
Some(("group", value)) => BACKUP_GROUP_SCHEMA.parse_simple_value(value).map(|_| FilterType::Group(value.to_string()))?,
Some(("type", value)) => FilterType::BackupType(value.parse()?),
Some(("regex", value)) => FilterType::Regex(Regex::new(value)?),
Some((ty, _value)) => bail!("expected 'group', 'type' or 'regex' prefix, got '{}'", ty),
None => bail!("input doesn't match expected format '<group:GROUP||type:<vm|ct|host>|regex:REGEX>'"),
})
}
}
// used for serializing below, caution!
impl std::fmt::Display for FilterType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FilterType::BackupType(backup_type) => write!(f, "type:{}", backup_type),
FilterType::Group(backup_group) => write!(f, "group:{}", backup_group),
FilterType::Regex(regex) => write!(f, "regex:{}", regex.as_str()),
}
}
}
#[derive(Clone, Debug)]
pub struct GroupFilter {
pub is_exclude: bool,
pub filter_type: FilterType,
}
impl PartialEq for GroupFilter {
fn eq(&self, other: &Self) -> bool {
self.filter_type == other.filter_type && self.is_exclude == other.is_exclude
}
}
impl Eq for GroupFilter {}
impl std::str::FromStr for GroupFilter {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.split_once(':') {
Some(("group", value)) => BACKUP_GROUP_SCHEMA.parse_simple_value(value).map(|_| GroupFilter::Group(value.to_string())),
Some(("type", value)) => Ok(GroupFilter::BackupType(value.parse()?)),
Some(("regex", value)) => Ok(GroupFilter::Regex(Regex::new(value)?)),
Some((ty, _value)) => Err(format_err!("expected 'group', 'type' or 'regex' prefix, got '{}'", ty)),
None => Err(format_err!("input doesn't match expected format '<group:GROUP||type:<vm|ct|host>|regex:REGEX>'")),
}.map_err(|err| format_err!("'{}' - {}", s, err))
let (is_exclude, type_str) = match s.split_once(':') {
Some(("include", value)) => (false, value),
Some(("exclude", value)) => (true, value),
_ => (false, s),
};
Ok(GroupFilter {
is_exclude,
filter_type: type_str.parse()?,
})
}
}
// used for serializing below, caution!
impl std::fmt::Display for GroupFilter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
GroupFilter::BackupType(backup_type) => write!(f, "type:{}", backup_type),
GroupFilter::Group(backup_group) => write!(f, "group:{}", backup_group),
GroupFilter::Regex(regex) => write!(f, "regex:{}", regex.as_str()),
if self.is_exclude {
f.write_str("exclude:")?;
}
std::fmt::Display::fmt(&self.filter_type, f)
}
}
@ -441,9 +482,9 @@ fn verify_group_filter(input: &str) -> Result<(), anyhow::Error> {
}
pub const GROUP_FILTER_SCHEMA: Schema = StringSchema::new(
"Group filter based on group identifier ('group:GROUP'), group type ('type:<vm|ct|host>'), or regex ('regex:RE').")
"Group filter based on group identifier ('group:GROUP'), group type ('type:<vm|ct|host>'), or regex ('regex:RE'). Can be inverted by prepending 'exclude:'.")
.format(&ApiStringFormat::VerifyFn(verify_group_filter))
.type_text("<type:<vm|ct|host>|group:GROUP|regex:RE>")
.type_text("[<exclude:|include:>]<type:<vm|ct|host>|group:GROUP|regex:RE>")
.schema();
pub const GROUP_FILTER_LIST_SCHEMA: Schema =

2
pbs-api-types/src/remote.rs
View File

@ -79,7 +79,7 @@ pub struct RemoteConfig {
pub struct Remote {
pub name: String,
// Note: The stored password is base64 encoded
#[serde(skip_serializing_if = "String::is_empty")]
#[serde(default, skip_serializing_if = "String::is_empty")]
#[serde(with = "proxmox_serde::string_as_base64")]
pub password: String,
#[serde(flatten)]

7
pbs-api-types/src/tape/changer.rs
View File

@ -51,6 +51,10 @@ Import/Export, i.e. any media in those slots are considered to be
schema: EXPORT_SLOT_LIST_SCHEMA,
optional: true,
},
"eject-before-unload": {
optional: true,
default: false,
}
},
)]
#[derive(Serialize, Deserialize, Updater)]
@ -62,6 +66,9 @@ pub struct ScsiTapeChanger {
pub path: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub export_slots: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
/// if set to true, tapes are ejected manually before unloading
pub eject_before_unload: Option<bool>,
}
#[api(

4
pbs-api-types/src/tape/drive.rs
View File

@ -59,7 +59,7 @@ pub struct VirtualTapeDrive {
},
}
)]
#[derive(Serialize, Deserialize, Updater)]
#[derive(Serialize, Deserialize, Updater, Clone)]
#[serde(rename_all = "kebab-case")]
/// Lto SCSI tape driver
pub struct LtoTapeDrive {
@ -108,7 +108,7 @@ pub struct MamAttribute {
}
#[api()]
#[derive(Serialize, Deserialize, Copy, Clone, Debug)]
#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialOrd, PartialEq)]
pub enum TapeDensity {
/// Unknown (no media loaded)
Unknown,

76
pbs-api-types/tests/group_filter_tests.rs Normal file
View File

@ -0,0 +1,76 @@
use pbs_api_types::{BackupGroup, BackupType, GroupFilter};
use std::str::FromStr;
#[test]
fn test_no_filters() {
let group_filters = vec![];
let do_backup = [
"vm/101", "vm/102", "vm/103", "vm/104", "vm/105", "vm/106", "vm/107", "vm/108", "vm/109",
];
for id in do_backup {
assert!(BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
}
#[test]
fn test_include_filters() {
let group_filters = vec![GroupFilter::from_str("regex:.*10[2-8]").unwrap()];
let do_backup = [
"vm/102", "vm/103", "vm/104", "vm/105", "vm/106", "vm/107", "vm/108",
];
let dont_backup = ["vm/101", "vm/109"];
for id in do_backup {
assert!(BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
for id in dont_backup {
assert!(!BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
}
#[test]
fn test_exclude_filters() {
let group_filters = [
GroupFilter::from_str("exclude:regex:.*10[1-3]").unwrap(),
GroupFilter::from_str("exclude:regex:.*10[5-7]").unwrap(),
];
let do_backup = ["vm/104", "vm/108", "vm/109"];
let dont_backup = ["vm/101", "vm/102", "vm/103", "vm/105", "vm/106", "vm/107"];
for id in do_backup {
assert!(BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
for id in dont_backup {
assert!(!BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
}
#[test]
fn test_include_and_exclude_filters() {
let group_filters = [
GroupFilter::from_str("exclude:regex:.*10[1-3]").unwrap(),
GroupFilter::from_str("regex:.*10[2-8]").unwrap(),
GroupFilter::from_str("exclude:regex:.*10[5-7]").unwrap(),
];
let do_backup = ["vm/104", "vm/108"];
let dont_backup = [
"vm/101", "vm/102", "vm/103", "vm/105", "vm/106", "vm/107", "vm/109",
];
for id in do_backup {
assert!(BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
for id in dont_backup {
assert!(!BackupGroup::new(BackupType::Vm, id).apply_filters(&group_filters));
}
}

62
pbs-client/src/catalog_shell.rs
View File

@ -304,68 +304,6 @@ async fn restore_command(target: String, pattern: Option<String>) -> Result<(),
/// The `Path` type's component iterator does not tell us anything about trailing slashes or
/// trailing `Component::CurDir` entries. Since we only support regular paths we'll roll our own
/// here:
enum PathComponent<'a> {
Root,
CurDir,
ParentDir,
Normal(&'a OsStr),
TrailingSlash,
}
struct PathComponentIter<'a> {
path: &'a [u8],
state: u8, // 0=beginning, 1=ongoing, 2=trailing, 3=finished (fused)
}
impl std::iter::FusedIterator for PathComponentIter<'_> {}
impl<'a> Iterator for PathComponentIter<'a> {
type Item = PathComponent<'a>;
fn next(&mut self) -> Option<Self::Item> {
if self.path.is_empty() {
return None;
}
if self.state == 0 {
self.state = 1;
if self.path[0] == b'/' {
// absolute path
self.path = &self.path[1..];
return Some(PathComponent::Root);
}
}
// skip slashes
let had_slashes = self.path[0] == b'/';
while self.path.first().copied() == Some(b'/') {
self.path = &self.path[1..];
}
Some(match self.path {
[] if had_slashes => PathComponent::TrailingSlash,
[] => return None,
[b'.'] | [b'.', b'/', ..] => {
self.path = &self.path[1..];
PathComponent::CurDir
}
[b'.', b'.'] | [b'.', b'.', b'/', ..] => {
self.path = &self.path[2..];
PathComponent::ParentDir
}
_ => {
let end = self
.path
.iter()
.position(|&b| b == b'/')
.unwrap_or(self.path.len());
let (out, rest) = self.path.split_at(end);
self.path = rest;
PathComponent::Normal(OsStr::from_bytes(out))
}
})
}
}
pub struct Shell {
/// Readline instance handling input and callbacks

8
pbs-config/src/lib.rs
View File

@ -44,12 +44,14 @@ pub fn backup_group() -> Result<nix::unistd::Group, Error> {
}
}
pub struct BackupLockGuard(Option<std::fs::File>);
pub struct BackupLockGuard {
_file: Option<std::fs::File>,
}
#[doc(hidden)]
/// Note: do not use for production code, this is only intended for tests
pub unsafe fn create_mocked_lock() -> BackupLockGuard {
BackupLockGuard(None)
BackupLockGuard { _file: None }
}
/// Open or create a lock file owned by user "backup" and lock it.
@ -73,7 +75,7 @@ pub fn open_backup_lockfile<P: AsRef<std::path::Path>>(
let timeout = timeout.unwrap_or(std::time::Duration::new(10, 0));
let file = proxmox_sys::fs::open_file_locked(&path, timeout, exclusive, options)?;
Ok(BackupLockGuard(Some(file)))
Ok(BackupLockGuard { _file: Some(file) })
}
/// Atomically write data to file owned by "root:backup" with permission "0640"

5
pbs-datastore/src/datastore.rs
View File

@ -602,7 +602,10 @@ impl DataStore {
) -> Result<Authid, Error> {
let full_path = self.owner_path(ns, backup_group);
let owner = proxmox_sys::fs::file_read_firstline(full_path)?;
owner.trim_end().parse() // remove trailing newline
owner
.trim_end() // remove trailing newline
.parse()
.map_err(|err| format_err!("parsing owner for {backup_group} failed: {err}"))
}
pub fn owns_backup(

1
pbs-tape/Cargo.toml
View File

@ -14,6 +14,7 @@ lazy_static.workspace = true
libc.workspace = true
log.workspace = true
nix.workspace = true
openssl.workspace = true
regex.workspace = true
serde.workspace = true
serde_json.workspace = true

11
pbs-tape/src/sg_pt_changer.rs
View File

@ -247,7 +247,7 @@ pub fn transfer_medium<F: AsRawFd>(
Ok(())
}
#[derive(Clone, Copy)]
#[derive(Clone, Copy, Debug)]
enum ElementType {
MediumTransport,
Storage,
@ -326,7 +326,9 @@ fn get_element<F: AsRawFd>(
let data = execute_scsi_command(sg_raw, &cmd, "read element status (B8h)", retry)?;
let page = decode_element_status_page(&data, start_element_address)?;
let page = decode_element_status_page(&data, start_element_address).map_err(|err| {
format_err!("decode element status for {element_type:?} on {start_element_address} failed - {err}")
})?;
retry = false; // only retry the first command
@ -367,7 +369,7 @@ pub fn read_element_status<F: AsRawFd>(file: &mut F) -> Result<MtxStatus, Error>
// first, request address assignment (used for sanity checks)
let setup = read_element_address_assignment(file)?;
let allocation_len: u32 = 0x10000;
let allocation_len: u32 = 0xFFFF; // some changer only use the lower 2 bytes
let mut sg_raw = SgRaw::new(file, allocation_len as usize)?;
sg_raw.set_timeout(SCSI_CHANGER_DEFAULT_TIMEOUT);
@ -679,7 +681,6 @@ fn decode_element_status_page(
data: &[u8],
start_element_address: u16,
) -> Result<DecodedStatusPage, Error> {
proxmox_lang::try_block!({
let mut result = DecodedStatusPage {
last_element_address: None,
transports: Vec::new(),
@ -820,8 +821,6 @@ fn decode_element_status_page(
}
Ok(result)
})
.map_err(|err: Error| format_err!("decode element status failed - {}", err))
}
/// Open the device for read/write, returns the file handle

103
pbs-tape/src/sg_tape.rs
View File

@ -9,9 +9,10 @@ use endian_trait::Endian;
use nix::fcntl::{fcntl, FcntlArg, OFlag};
mod encryption;
pub use encryption::*;
pub use encryption::{drive_get_encryption, drive_set_encryption};
mod volume_statistics;
use proxmox_uuid::Uuid;
pub use volume_statistics::*;
mod tape_alert_flags;
@ -26,8 +27,11 @@ pub use report_density::*;
use proxmox_io::{ReadExt, WriteExt};
use proxmox_sys::error::SysResult;
use pbs_api_types::{Lp17VolumeStatistics, LtoDriveAndMediaStatus, MamAttribute};
use pbs_api_types::{
Lp17VolumeStatistics, LtoDriveAndMediaStatus, LtoTapeDrive, MamAttribute, TapeDensity,
};
use crate::linux_list_drives::open_lto_tape_device;
use crate::{
sgutils2::{
alloc_page_aligned_buffer, scsi_cmd_mode_select10, scsi_cmd_mode_select6, scsi_inquiry,
@ -102,7 +106,6 @@ pub struct SgTape {
file: File,
locate_offset: Option<i64>,
info: InquiryInfo,
encryption_key_loaded: bool,
}
impl SgTape {
@ -124,11 +127,47 @@ impl SgTape {
Ok(Self {
file,
info,
encryption_key_loaded: false,
locate_offset: None,
})
}
/// Open a tape device
///
/// This does additional checks:
///
/// - check if it is a non-rewinding tape device
/// - check if drive is ready (tape loaded)
/// - check block size
/// - for autoloader only, try to reload ejected tapes
pub fn open_lto_drive(config: &LtoTapeDrive) -> Result<Self, Error> {
proxmox_lang::try_block!({
let file = open_lto_tape_device(&config.path)?;
let mut handle = SgTape::new(file)?;
if handle.test_unit_ready().is_err() {
// for autoloader only, try to reload ejected tapes
if config.changer.is_some() {
let _ = handle.load(); // just try, ignore error
}
}
handle.wait_until_ready(None)?;
handle.set_default_options()?;
Ok(handle)
})
.map_err(|err: Error| {
format_err!(
"open drive '{}' ({}) failed - {}",
config.name,
config.path,
err
)
})
}
/// Access to file descriptor - useful for testing
pub fn file_mut(&mut self) -> &mut File {
&mut self.file
@ -197,16 +236,17 @@ impl SgTape {
/// Format media, single partition
pub fn format_media(&mut self, fast: bool) -> Result<(), Error> {
// try to get info about loaded media first
let (has_format, is_worm) = match self.read_medium_configuration_page() {
let (density, is_worm) = match self.read_medium_configuration_page() {
Ok((_head, block_descriptor, page)) => {
// FORMAT requires LTO5 or newer
let has_format = block_descriptor.density_code >= 0x58;
let density: TapeDensity = TapeDensity::try_from(block_descriptor.density_code)
.unwrap_or(TapeDensity::Unknown);
let is_worm = page.is_worm();
(has_format, is_worm)
(density, is_worm)
}
Err(_) => {
// LTO3 and older do not support medium configuration mode page
(false, false)
(TapeDensity::Unknown, false)
}
};
@ -227,14 +267,21 @@ impl SgTape {
sg_raw.set_timeout(Self::SCSI_TAPE_DEFAULT_TIMEOUT);
let mut cmd = Vec::new();
if has_format {
if density >= TapeDensity::LTO5 && density <= TapeDensity::LTO8 {
cmd.extend([0x04, 0, 0, 0, 0, 0]); // FORMAT
sg_raw.do_command(&cmd)?;
if !fast {
self.erase_media(false)?; // overwrite everything
}
} else if density >= TapeDensity::LTO9 && !fast {
cmd.extend([0x04, 0x01, 0, 0, 0, 0]); // FORMAT, set IMMED
sg_raw.do_command(&cmd)?;
self.wait_until_ready(Some(60 * 60 * 2)) // 2 hours, max. initialization time
.map_err(|err| format_err!("error waiting for LTO9+ initialization: {err}"))?;
self.erase_media(false)?; // overwrite everything
} else {
// try rewind/erase instead
// we also do this for LTO9+ to avoid reinitialization on FORMAT(04h)
self.erase_media(fast)?
}
@ -571,9 +618,10 @@ impl SgTape {
}
}
pub fn wait_until_ready(&mut self) -> Result<(), Error> {
pub fn wait_until_ready(&mut self, timeout: Option<u64>) -> Result<(), Error> {
let start = SystemTime::now();
let max_wait = std::time::Duration::new(Self::SCSI_TAPE_DEFAULT_TIMEOUT as u64, 0);
let timeout = timeout.unwrap_or(Self::SCSI_TAPE_DEFAULT_TIMEOUT as u64);
let max_wait = std::time::Duration::new(timeout, 0);
loop {
match self.test_unit_ready() {
@ -603,10 +651,28 @@ impl SgTape {
read_volume_statistics(&mut self.file)
}
pub fn set_encryption(&mut self, key: Option<[u8; 32]>) -> Result<(), Error> {
self.encryption_key_loaded = key.is_some();
pub fn set_encryption(&mut self, key_data: Option<([u8; 32], Uuid)>) -> Result<(), Error> {
let key = if let Some((ref key, ref uuid)) = key_data {
// derive specialized key for each media-set
set_encryption(&mut self.file, key)
let mut tape_key = [0u8; 32];
let uuid_bytes: [u8; 16] = *uuid.as_bytes();
openssl::pkcs5::pbkdf2_hmac(
key,
&uuid_bytes,
10,
openssl::hash::MessageDigest::sha256(),
&mut tape_key,
)?;
Some(tape_key)
} else {
None
};
drive_set_encryption(&mut self.file, key)
}
// Note: use alloc_page_aligned_buffer to alloc data transfer buffer
@ -960,15 +1026,6 @@ impl SgTape {
}
}
impl Drop for SgTape {
fn drop(&mut self) {
// For security reasons, clear the encryption key
if self.encryption_key_loaded {
let _ = self.set_encryption(None);
}
}
}
pub struct SgTapeReader<'a> {
sg_tape: &'a mut SgTape,
end_of_file: bool,

36
pbs-tape/src/sg_tape/encryption.rs
View File

@ -8,21 +8,10 @@ use proxmox_io::{ReadExt, WriteExt};
use crate::sgutils2::{alloc_page_aligned_buffer, SgRaw};
/// Test if drive supports hardware encryption
///
/// We search for AES_GCM algorithm with 256bits key.
pub fn has_encryption<F: AsRawFd>(file: &mut F) -> bool {
let data = match sg_spin_data_encryption_caps(file) {
Ok(data) => data,
Err(_) => return false,
};
decode_spin_data_encryption_caps(&data).is_ok()
}
/// Set or clear encryption key
///
/// We always use mixed mode,
pub fn set_encryption<F: AsRawFd>(file: &mut F, key: Option<[u8; 32]>) -> Result<(), Error> {
pub fn drive_set_encryption<F: AsRawFd>(file: &mut F, key: Option<[u8; 32]>) -> Result<(), Error> {
let data = match sg_spin_data_encryption_caps(file) {
Ok(data) => data,
Err(_) if key.is_none() => {
@ -57,6 +46,27 @@ pub fn set_encryption<F: AsRawFd>(file: &mut F, key: Option<[u8; 32]>) -> Result
bail!("got unexpected encryption mode {:?}", status.mode);
}
/// Returns if encryption is enabled on the drive
pub fn drive_get_encryption<F: AsRawFd>(file: &mut F) -> Result<bool, Error> {
let data = match sg_spin_data_encryption_status(file) {
Ok(data) => data,
Err(_) => {
// Assume device does not support HW encryption
return Ok(false);
}
};
let status = decode_spin_data_encryption_status(&data)?;
match status.mode {
// these three below have all encryption enabled, and only differ in how decryption is
// handled
DataEncryptionMode::On => Ok(true),
DataEncryptionMode::Mixed => Ok(true),
DataEncryptionMode::RawRead => Ok(true),
// currently, the mode below is the only one that has encryption actually disabled
DataEncryptionMode::Off => Ok(false),
}
}
#[derive(Endian)]
#[repr(C, packed)]
struct SspSetDataEncryptionPage {
@ -187,7 +197,7 @@ fn sg_spin_data_encryption_caps<F: AsRawFd>(file: &mut F) -> Result<Vec<u8>, Err
.map(|v| v.to_vec())
}
#[derive(Debug)]
#[derive(Debug, PartialEq, Eq)]
enum DataEncryptionMode {
On,
Mixed,

25
proxmox-rrd/Cargo.toml
View File

@ -1,25 +0,0 @@
[package]
name = "proxmox-rrd"
version = "0.1.0"
authors.workspace = true
edition.workspace = true
license.workspace = true
description = "Simple RRD database implementation."
[dev-dependencies]
proxmox-router = { workspace = true, features = ["cli", "server"] }
[dependencies]
anyhow.workspace = true
bitflags.workspace = true
crossbeam-channel.workspace = true
libc.workspace = true
log.workspace = true
nix.workspace = true
serde.workspace = true
serde_cbor.workspace = true
serde_json.workspace = true
proxmox-schema = { workspace = true, features = [ "api-macro" ] }
proxmox-sys.workspace = true
proxmox-time.workspace = true

390
proxmox-rrd/examples/prrd.rs
View File

@ -1,390 +0,0 @@
//! RRD toolkit - create/manage/update proxmox RRD (v2) file
use std::path::PathBuf;
use anyhow::{bail, Error};
use serde::{Deserialize, Serialize};
use serde_json::json;
use proxmox_router::cli::{
complete_file_name, run_cli_command, CliCommand, CliCommandMap, CliEnvironment,
};
use proxmox_router::RpcEnvironment;
use proxmox_schema::{api, ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema};
use proxmox_sys::fs::CreateOptions;
use proxmox_rrd::rrd::{CF, DST, RRA, RRD};
pub const RRA_INDEX_SCHEMA: Schema = IntegerSchema::new("Index of the RRA.").minimum(0).schema();
pub const RRA_CONFIG_STRING_SCHEMA: Schema = StringSchema::new("RRA configuration")
.format(&ApiStringFormat::PropertyString(&RRAConfig::API_SCHEMA))
.schema();
#[api(
properties: {},
default_key: "cf",
)]
#[derive(Debug, Serialize, Deserialize)]
/// RRA configuration
pub struct RRAConfig {
/// Time resolution
pub r: u64,
pub cf: CF,
/// Number of data points
pub n: u64,
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
},
},
)]
/// Dump the RRD file in JSON format
pub fn dump_rrd(path: String) -> Result<(), Error> {
let rrd = RRD::load(&PathBuf::from(path), false)?;
serde_json::to_writer_pretty(std::io::stdout(), &rrd)?;
println!();
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
},
},
)]
/// RRD file information
pub fn rrd_info(path: String) -> Result<(), Error> {
let rrd = RRD::load(&PathBuf::from(path), false)?;
println!("DST: {:?}", rrd.source.dst);
for (i, rra) in rrd.rra_list.iter().enumerate() {
// use RRAConfig property string format
println!(
"RRA[{}]: {:?},r={},n={}",
i,
rra.cf,
rra.resolution,
rra.data.len()
);
}
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
time: {
description: "Update time.",
optional: true,
},
value: {
description: "Update value.",
},
},
},
)]
/// Update the RRD database
pub fn update_rrd(path: String, time: Option<u64>, value: f64) -> Result<(), Error> {
let path = PathBuf::from(path);
let time = time
.map(|v| v as f64)
.unwrap_or_else(proxmox_time::epoch_f64);
let mut rrd = RRD::load(&path, false)?;
rrd.update(time, value);
rrd.save(&path, CreateOptions::new(), false)?;
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
cf: {
type: CF,
},
resolution: {
description: "Time resolution",
},
start: {
description: "Start time. If not specified, we simply extract 10 data points.",
optional: true,
},
end: {
description: "End time (Unix Epoch). Default is the last update time.",
optional: true,
},
},
},
)]
/// Fetch data from the RRD file
pub fn fetch_rrd(
path: String,
cf: CF,
resolution: u64,
start: Option<u64>,
end: Option<u64>,
) -> Result<(), Error> {
let rrd = RRD::load(&PathBuf::from(path), false)?;
let data = rrd.extract_data(cf, resolution, start, end)?;
println!("{}", serde_json::to_string_pretty(&data)?);
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
"rra-index": {
schema: RRA_INDEX_SCHEMA,
},
},
},
)]
/// Return the Unix timestamp of the first time slot inside the
/// specified RRA (slot start time)
pub fn first_update_time(path: String, rra_index: usize) -> Result<(), Error> {
let rrd = RRD::load(&PathBuf::from(path), false)?;
if rra_index >= rrd.rra_list.len() {
bail!("rra-index is out of range");
}
let rra = &rrd.rra_list[rra_index];
let duration = (rra.data.len() as u64) * rra.resolution;
let first = rra.slot_start_time((rrd.source.last_update as u64).saturating_sub(duration));
println!("{}", first);
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
},
},
)]
/// Return the Unix timestamp of the last update
pub fn last_update_time(path: String) -> Result<(), Error> {
let rrd = RRD::load(&PathBuf::from(path), false)?;
println!("{}", rrd.source.last_update);
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
},
},
)]
/// Return the time and value from the last update
pub fn last_update(path: String) -> Result<(), Error> {
let rrd = RRD::load(&PathBuf::from(path), false)?;
let result = json!({
"time": rrd.source.last_update,
"value": rrd.source.last_value,
});
println!("{}", serde_json::to_string_pretty(&result)?);
Ok(())
}
#[api(
input: {
properties: {
dst: {
type: DST,
},
path: {
description: "The filename to create."
},
rra: {
description: "Configuration of contained RRAs.",
type: Array,
items: {
schema: RRA_CONFIG_STRING_SCHEMA,
}
},
},
},
)]
/// Create a new RRD file
pub fn create_rrd(dst: DST, path: String, rra: Vec<String>) -> Result<(), Error> {
let mut rra_list = Vec::new();
for item in rra.iter() {
let rra: RRAConfig =
serde_json::from_value(RRAConfig::API_SCHEMA.parse_property_string(item)?)?;
println!("GOT {:?}", rra);
rra_list.push(RRA::new(rra.cf, rra.r, rra.n as usize));
}
let path = PathBuf::from(path);
let rrd = RRD::new(dst, rra_list);
rrd.save(&path, CreateOptions::new(), false)?;
Ok(())
}
#[api(
input: {
properties: {
path: {
description: "The filename."
},
"rra-index": {
schema: RRA_INDEX_SCHEMA,
},
slots: {
description: "The number of slots you want to add or remove.",
type: i64,
},
},
},
)]
/// Resize. Change the number of data slots for the specified RRA.
pub fn resize_rrd(path: String, rra_index: usize, slots: i64) -> Result<(), Error> {
let path = PathBuf::from(&path);
let mut rrd = RRD::load(&path, false)?;
if rra_index >= rrd.rra_list.len() {
bail!("rra-index is out of range");
}
let rra = &rrd.rra_list[rra_index];
let new_slots = (rra.data.len() as i64) + slots;
if new_slots < 1 {
bail!("number of new slots is too small ('{}' < 1)", new_slots);
}
if new_slots > 1024 * 1024 {
bail!("number of new slots is too big ('{}' > 1M)", new_slots);
}
let rra_end = rra.slot_end_time(rrd.source.last_update as u64);
let rra_start = rra_end - rra.resolution * (rra.data.len() as u64);
let (start, reso, data) = rra
.extract_data(rra_start, rra_end, rrd.source.last_update)
.into();
let mut new_rra = RRA::new(rra.cf, rra.resolution, new_slots as usize);
new_rra.last_count = rra.last_count;
new_rra.insert_data(start, reso, data)?;
rrd.rra_list[rra_index] = new_rra;
rrd.save(&path, CreateOptions::new(), false)?;
Ok(())
}
fn main() -> Result<(), Error> {
let uid = nix::unistd::Uid::current();
let username = match nix::unistd::User::from_uid(uid)? {
Some(user) => user.name,
None => bail!("unable to get user name"),
};
let cmd_def = CliCommandMap::new()
.insert(
"create",
CliCommand::new(&API_METHOD_CREATE_RRD)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"dump",
CliCommand::new(&API_METHOD_DUMP_RRD)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"fetch",
CliCommand::new(&API_METHOD_FETCH_RRD)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"first",
CliCommand::new(&API_METHOD_FIRST_UPDATE_TIME)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"info",
CliCommand::new(&API_METHOD_RRD_INFO)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"last",
CliCommand::new(&API_METHOD_LAST_UPDATE_TIME)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"lastupdate",
CliCommand::new(&API_METHOD_LAST_UPDATE)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"resize",
CliCommand::new(&API_METHOD_RESIZE_RRD)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
)
.insert(
"update",
CliCommand::new(&API_METHOD_UPDATE_RRD)
.arg_param(&["path"])
.completion_cb("path", complete_file_name),
);
let mut rpcenv = CliEnvironment::new();
rpcenv.set_auth_id(Some(format!("{}@pam", username)));
run_cli_command(cmd_def, rpcenv, None);
Ok(())
}

448
proxmox-rrd/src/cache.rs
View File

@ -1,448 +0,0 @@
use std::collections::BTreeSet;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::thread::spawn;
use std::time::SystemTime;
use anyhow::{bail, format_err, Error};
use crossbeam_channel::{bounded, TryRecvError};
use proxmox_sys::fs::{create_path, CreateOptions};
use crate::rrd::{CF, DST, RRA, RRD};
use crate::Entry;
mod journal;
use journal::*;
mod rrd_map;
use rrd_map::*;
/// RRD cache - keep RRD data in RAM, but write updates to disk
///
/// This cache is designed to run as single instance (no concurrent
/// access from other processes).
pub struct RRDCache {
config: Arc<CacheConfig>,
state: Arc<RwLock<JournalState>>,
rrd_map: Arc<RwLock<RRDMap>>,
}
pub(crate) struct CacheConfig {
apply_interval: f64,
basedir: PathBuf,
file_options: CreateOptions,
dir_options: CreateOptions,
}
impl RRDCache {
/// Creates a new instance
///
/// `basedir`: All files are stored relative to this path.
///
/// `file_options`: Files are created with this options.
///
/// `dir_options`: Directories are created with this options.
///
/// `apply_interval`: Commit journal after `apply_interval` seconds.
///
/// `load_rrd_cb`; The callback function is used to load RRD files,
/// and should return a newly generated RRD if the file does not
/// exists (or is unreadable). This may generate RRDs with
/// different configurations (dependent on `rel_path`).
pub fn new<P: AsRef<Path>>(
basedir: P,
file_options: Option<CreateOptions>,
dir_options: Option<CreateOptions>,
apply_interval: f64,
load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD,
) -> Result<Self, Error> {
let basedir = basedir.as_ref().to_owned();
let file_options = file_options.unwrap_or_else(CreateOptions::new);
let dir_options = dir_options.unwrap_or_else(CreateOptions::new);
create_path(
&basedir,
Some(dir_options.clone()),
Some(dir_options.clone()),
)
.map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?;
let config = Arc::new(CacheConfig {
basedir,
file_options,
dir_options,
apply_interval,
});
let state = JournalState::new(Arc::clone(&config))?;
let rrd_map = RRDMap::new(Arc::clone(&config), load_rrd_cb);
Ok(Self {
config: Arc::clone(&config),
state: Arc::new(RwLock::new(state)),
rrd_map: Arc::new(RwLock::new(rrd_map)),
})
}
/// Create a new RRD as used by the proxmox backup server
///
/// It contains the following RRAs:
///
/// * cf=average,r=60,n=1440 => 1day
/// * cf=maximum,r=60,n=1440 => 1day
/// * cf=average,r=30*60,n=1440 => 1month
/// * cf=maximum,r=30*60,n=1440 => 1month
/// * cf=average,r=6*3600,n=1440 => 1year
/// * cf=maximum,r=6*3600,n=1440 => 1year
/// * cf=average,r=7*86400,n=570 => 10years
/// * cf=maximum,r=7*86400,n=570 => 10year
///
/// The resulting data file size is about 80KB.
pub fn create_proxmox_backup_default_rrd(dst: DST) -> RRD {
let rra_list = vec![
// 1 min * 1440 => 1 day
RRA::new(CF::Average, 60, 1440),
RRA::new(CF::Maximum, 60, 1440),
// 30 min * 1440 => 30 days ~ 1 month
RRA::new(CF::Average, 30 * 60, 1440),
RRA::new(CF::Maximum, 30 * 60, 1440),
// 6 h * 1440 => 360 days ~ 1 year
RRA::new(CF::Average, 6 * 3600, 1440),
RRA::new(CF::Maximum, 6 * 3600, 1440),
// 1 week * 570 => 10 years
RRA::new(CF::Average, 7 * 86400, 570),
RRA::new(CF::Maximum, 7 * 86400, 570),
];
RRD::new(dst, rra_list)
}
/// Sync the journal data to disk (using `fdatasync` syscall)
pub fn sync_journal(&self) -> Result<(), Error> {
self.state.read().unwrap().sync_journal()
}
/// Apply and commit the journal. Should be used at server startup.
pub fn apply_journal(&self) -> Result<bool, Error> {
let config = Arc::clone(&self.config);
let state = Arc::clone(&self.state);
let rrd_map = Arc::clone(&self.rrd_map);
let mut state_guard = self.state.write().unwrap();
let journal_applied = state_guard.journal_applied;
if let Some(ref recv) = state_guard.apply_thread_result {
match recv.try_recv() {
Ok(Ok(())) => {
// finished without errors, OK
state_guard.apply_thread_result = None;
}
Ok(Err(err)) => {
// finished with errors, log them
log::error!("{}", err);
state_guard.apply_thread_result = None;
}
Err(TryRecvError::Empty) => {
// still running
return Ok(journal_applied);
}
Err(TryRecvError::Disconnected) => {
// crashed, start again
log::error!("apply journal thread crashed - try again");
state_guard.apply_thread_result = None;
}
}
}
let now = proxmox_time::epoch_f64();
let wants_commit = (now - state_guard.last_journal_flush) > self.config.apply_interval;
if journal_applied && !wants_commit {
return Ok(journal_applied);
}
state_guard.last_journal_flush = proxmox_time::epoch_f64();
let (sender, receiver) = bounded(1);
state_guard.apply_thread_result = Some(receiver);
spawn(move || {
let result = apply_and_commit_journal_thread(config, state, rrd_map, journal_applied)
.map_err(|err| err.to_string());
sender.send(result).unwrap();
});
Ok(journal_applied)
}
/// Update data in RAM and write file back to disk (journal)
pub fn update_value(
&self,
rel_path: &str,
time: f64,
value: f64,
dst: DST,
) -> Result<(), Error> {
let journal_applied = self.apply_journal()?;
self.state
.write()
.unwrap()
.append_journal_entry(time, value, dst, rel_path)?;
if journal_applied {
self.rrd_map
.write()
.unwrap()
.update(rel_path, time, value, dst, false)?;
}
Ok(())
}
/// Extract data from cached RRD
///
/// `start`: Start time. If not specified, we simply extract 10 data points.
///
/// `end`: End time. Default is to use the current time.
pub fn extract_cached_data(
&self,
base: &str,
name: &str,
cf: CF,
resolution: u64,
start: Option<u64>,
end: Option<u64>,
) -> Result<Option<Entry>, Error> {
self.rrd_map
.read()
.unwrap()
.extract_cached_data(base, name, cf, resolution, start, end)
}
}
fn apply_and_commit_journal_thread(
config: Arc<CacheConfig>,
state: Arc<RwLock<JournalState>>,
rrd_map: Arc<RwLock<RRDMap>>,
commit_only: bool,
) -> Result<(), Error> {
if commit_only {
state.write().unwrap().rotate_journal()?; // start new journal, keep old one
} else {
let start_time = SystemTime::now();
log::debug!("applying rrd journal");
match apply_journal_impl(Arc::clone(&state), Arc::clone(&rrd_map)) {
Ok(entries) => {
let elapsed = start_time.elapsed().unwrap().as_secs_f64();
log::info!(
"applied rrd journal ({} entries in {:.3} seconds)",
entries,
elapsed
);
}
Err(err) => bail!("apply rrd journal failed - {}", err),
}
}
let start_time = SystemTime::now();
log::debug!("commit rrd journal");
match commit_journal_impl(config, state, rrd_map) {
Ok(rrd_file_count) => {
let elapsed = start_time.elapsed().unwrap().as_secs_f64();
log::info!(
"rrd journal successfully committed ({} files in {:.3} seconds)",
rrd_file_count,
elapsed
);
}
Err(err) => bail!("rrd journal commit failed: {}", err),
}
Ok(())
}
fn apply_journal_lines(
state: Arc<RwLock<JournalState>>,
rrd_map: Arc<RwLock<RRDMap>>,
journal_name: &str, // used for logging
reader: &mut BufReader<File>,
lock_read_line: bool,
) -> Result<usize, Error> {
let mut linenr = 0;
loop {
linenr += 1;
let mut line = String::new();
let len = if lock_read_line {
let _lock = state.read().unwrap(); // make sure we read entire lines
reader.read_line(&mut line)?
} else {
reader.read_line(&mut line)?
};
if len == 0 {
break;
}
let entry: JournalEntry = match line.parse() {
Ok(entry) => entry,
Err(err) => {
log::warn!(
"unable to parse rrd journal '{}' line {} (skip) - {}",
journal_name,
linenr,
err,
);
continue; // skip unparsable lines
}
};
rrd_map.write().unwrap().update(
&entry.rel_path,
entry.time,
entry.value,
entry.dst,
true,
)?;
}
Ok(linenr)
}
fn apply_journal_impl(
state: Arc<RwLock<JournalState>>,
rrd_map: Arc<RwLock<RRDMap>>,
) -> Result<usize, Error> {
let mut lines = 0;
// Apply old journals first
let journal_list = state.read().unwrap().list_old_journals()?;
for entry in journal_list {
log::info!("apply old journal log {}", entry.name);
let file = std::fs::OpenOptions::new().read(true).open(&entry.path)?;
let mut reader = BufReader::new(file);
lines += apply_journal_lines(
Arc::clone(&state),
Arc::clone(&rrd_map),
&entry.name,
&mut reader,
false,
)?;
}
let mut journal = state.read().unwrap().open_journal_reader()?;
lines += apply_journal_lines(
Arc::clone(&state),
Arc::clone(&rrd_map),
"rrd.journal",
&mut journal,
true,
)?;
{
let mut state_guard = state.write().unwrap(); // block other writers
lines += apply_journal_lines(
Arc::clone(&state),
Arc::clone(&rrd_map),
"rrd.journal",
&mut journal,
false,
)?;
state_guard.rotate_journal()?; // start new journal, keep old one
// We need to apply the journal only once, because further updates
// are always directly applied.
state_guard.journal_applied = true;
}
Ok(lines)
}
fn fsync_file_or_dir(path: &Path) -> Result<(), Error> {
let file = std::fs::File::open(path)?;
nix::unistd::fsync(file.as_raw_fd())?;
Ok(())
}
pub(crate) fn fsync_file_and_parent(path: &Path) -> Result<(), Error> {
let file = std::fs::File::open(path)?;
nix::unistd::fsync(file.as_raw_fd())?;
if let Some(parent) = path.parent() {
fsync_file_or_dir(parent)?;
}
Ok(())
}
fn rrd_parent_dir(basedir: &Path, rel_path: &str) -> PathBuf {
let mut path = basedir.to_owned();
let rel_path = Path::new(rel_path);
if let Some(parent) = rel_path.parent() {
path.push(parent);
}
path
}
fn commit_journal_impl(
config: Arc<CacheConfig>,
state: Arc<RwLock<JournalState>>,
rrd_map: Arc<RwLock<RRDMap>>,
) -> Result<usize, Error> {
let files = rrd_map.read().unwrap().file_list();
let mut rrd_file_count = 0;
let mut errors = 0;
let mut dir_set = BTreeSet::new();
log::info!("write rrd data back to disk");
// save all RRDs - we only need a read lock here
// Note: no fsync here (we do it afterwards)
for rel_path in files.iter() {
let parent_dir = rrd_parent_dir(&config.basedir, rel_path);
dir_set.insert(parent_dir);
rrd_file_count += 1;
if let Err(err) = rrd_map.read().unwrap().flush_rrd_file(rel_path) {
errors += 1;
log::error!("unable to save rrd {}: {}", rel_path, err);
}
}
if errors != 0 {
bail!("errors during rrd flush - unable to commit rrd journal");
}
// Important: We fsync files after writing all data! This increase
// the likelihood that files are already synced, so this is
// much faster (although we need to re-open the files).
log::info!("starting rrd data sync");
for rel_path in files.iter() {
let mut path = config.basedir.clone();
path.push(rel_path);
fsync_file_or_dir(&path)
.map_err(|err| format_err!("fsync rrd file {} failed - {}", rel_path, err))?;
}
// also fsync directories
for dir_path in dir_set {
fsync_file_or_dir(&dir_path)
.map_err(|err| format_err!("fsync rrd dir {:?} failed - {}", dir_path, err))?;
}
// if everything went ok, remove the old journal files
state.write().unwrap().remove_old_journals()?;
Ok(rrd_file_count)
}

200
proxmox-rrd/src/cache/journal.rs vendored
View File

@ -1,200 +0,0 @@
use std::ffi::OsStr;
use std::fs::File;
use std::io::{BufReader, Write};
use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::Arc;
use anyhow::{bail, format_err, Error};
use crossbeam_channel::Receiver;
use nix::fcntl::OFlag;
use proxmox_sys::fs::atomic_open_or_create_file;
const RRD_JOURNAL_NAME: &str = "rrd.journal";
use crate::cache::CacheConfig;
use crate::rrd::DST;
// shared state behind RwLock
pub struct JournalState {
config: Arc<CacheConfig>,
journal: File,
pub last_journal_flush: f64,
pub journal_applied: bool,
pub apply_thread_result: Option<Receiver<Result<(), String>>>,
}
pub struct JournalEntry {
pub time: f64,
pub value: f64,
pub dst: DST,
pub rel_path: String,
}
impl FromStr for JournalEntry {
type Err = Error;
fn from_str(line: &str) -> Result<Self, Self::Err> {
let line = line.trim();
let parts: Vec<&str> = line.splitn(4, ':').collect();
if parts.len() != 4 {
bail!("wrong numper of components");
}
let time: f64 = parts[0]
.parse()
.map_err(|_| format_err!("unable to parse time"))?;
let value: f64 = parts[1]
.parse()
.map_err(|_| format_err!("unable to parse value"))?;
let dst: u8 = parts[2]
.parse()
.map_err(|_| format_err!("unable to parse data source type"))?;
let dst = match dst {
0 => DST::Gauge,
1 => DST::Derive,
_ => bail!("got strange value for data source type '{}'", dst),
};
let rel_path = parts[3].to_string();
Ok(JournalEntry {
time,
value,
dst,
rel_path,
})
}
}
pub struct JournalFileInfo {
pub time: u64,
pub name: String,
pub path: PathBuf,
}
impl JournalState {
pub(crate) fn new(config: Arc<CacheConfig>) -> Result<Self, Error> {
let journal = JournalState::open_journal_writer(&config)?;
Ok(Self {
config,
journal,
last_journal_flush: 0.0,
journal_applied: false,
apply_thread_result: None,
})
}
pub fn sync_journal(&self) -> Result<(), Error> {
nix::unistd::fdatasync(self.journal.as_raw_fd())?;
Ok(())
}
pub fn append_journal_entry(
&mut self,
time: f64,
value: f64,
dst: DST,
rel_path: &str,
) -> Result<(), Error> {
let journal_entry = format!("{}:{}:{}:{}\n", time, value, dst as u8, rel_path);
self.journal.write_all(journal_entry.as_bytes())?;
Ok(())
}
pub fn open_journal_reader(&self) -> Result<BufReader<File>, Error> {
// fixme : dup self.journal instead??
let mut journal_path = self.config.basedir.clone();
journal_path.push(RRD_JOURNAL_NAME);
let flags = OFlag::O_CLOEXEC | OFlag::O_RDONLY;
let journal = atomic_open_or_create_file(
&journal_path,
flags,
&[],
self.config.file_options.clone(),
false,
)?;
Ok(BufReader::new(journal))
}
fn open_journal_writer(config: &CacheConfig) -> Result<File, Error> {
let mut journal_path = config.basedir.clone();
journal_path.push(RRD_JOURNAL_NAME);
let flags = OFlag::O_CLOEXEC | OFlag::O_WRONLY | OFlag::O_APPEND;
let journal = atomic_open_or_create_file(
&journal_path,
flags,
&[],
config.file_options.clone(),
false,
)?;
Ok(journal)
}
pub fn rotate_journal(&mut self) -> Result<(), Error> {
let mut journal_path = self.config.basedir.clone();
journal_path.push(RRD_JOURNAL_NAME);
let mut new_name = journal_path.clone();
let now = proxmox_time::epoch_i64();
new_name.set_extension(format!("journal-{:08x}", now));
std::fs::rename(journal_path, &new_name)?;
self.journal = Self::open_journal_writer(&self.config)?;
// make sure the old journal data landed on the disk
super::fsync_file_and_parent(&new_name)?;
Ok(())
}
pub fn remove_old_journals(&self) -> Result<(), Error> {
let journal_list = self.list_old_journals()?;
for entry in journal_list {
std::fs::remove_file(entry.path)?;
}
Ok(())
}
pub fn list_old_journals(&self) -> Result<Vec<JournalFileInfo>, Error> {
let mut list = Vec::new();
for entry in std::fs::read_dir(&self.config.basedir)? {
let entry = entry?;
let path = entry.path();
if !path.is_file() {
continue;
}
match path.file_stem() {
None => continue,
Some(stem) if stem != OsStr::new("rrd") => continue,
Some(_) => (),
}
if let Some(extension) = path.extension() {
if let Some(extension) = extension.to_str() {
if let Some(rest) = extension.strip_prefix("journal-") {
if let Ok(time) = u64::from_str_radix(rest, 16) {
list.push(JournalFileInfo {
time,
name: format!("rrd.{}", extension),
path: path.to_owned(),
});
}
}
}
}
}
list.sort_unstable_by_key(|entry| entry.time);
Ok(list)
}
}

97
proxmox-rrd/src/cache/rrd_map.rs vendored
View File

@ -1,97 +0,0 @@
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use anyhow::{bail, Error};
use proxmox_sys::fs::create_path;
use crate::rrd::{CF, DST, RRD};
use super::CacheConfig;
use crate::Entry;
pub struct RRDMap {
config: Arc<CacheConfig>,
map: HashMap<String, RRD>,
load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD,
}
impl RRDMap {
pub(crate) fn new(
config: Arc<CacheConfig>,
load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD,
) -> Self {
Self {
config,
map: HashMap::new(),
load_rrd_cb,
}
}
pub fn update(
&mut self,
rel_path: &str,
time: f64,
value: f64,
dst: DST,
new_only: bool,
) -> Result<(), Error> {
if let Some(rrd) = self.map.get_mut(rel_path) {
if !new_only || time > rrd.last_update() {
rrd.update(time, value);
}
} else {
let mut path = self.config.basedir.clone();
path.push(rel_path);
create_path(
path.parent().unwrap(),
Some(self.config.dir_options.clone()),
Some(self.config.dir_options.clone()),
)?;
let mut rrd = (self.load_rrd_cb)(&path, rel_path, dst);
if !new_only || time > rrd.last_update() {
rrd.update(time, value);
}
self.map.insert(rel_path.to_string(), rrd);
}
Ok(())
}
pub fn file_list(&self) -> Vec<String> {
let mut list = Vec::new();
for rel_path in self.map.keys() {
list.push(rel_path.clone());
}
list
}
pub fn flush_rrd_file(&self, rel_path: &str) -> Result<(), Error> {
if let Some(rrd) = self.map.get(rel_path) {
let mut path = self.config.basedir.clone();
path.push(rel_path);
rrd.save(&path, self.config.file_options.clone(), true)
} else {
bail!("rrd file {} not loaded", rel_path);
}
}
pub fn extract_cached_data(
&self,
base: &str,
name: &str,
cf: CF,
resolution: u64,
start: Option<u64>,
end: Option<u64>,
) -> Result<Option<Entry>, Error> {
match self.map.get(&format!("{}/{}", base, name)) {
Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)),
None => Ok(None),
}
}
}

16
proxmox-rrd/src/lib.rs
View File

@ -1,16 +0,0 @@
//! # Round Robin Database files
//!
//! ## Features
//!
//! * One file stores a single data source
//! * Stores data for different time resolution
//! * Simple cache implementation with journal support
mod rrd_v1;
pub mod rrd;
#[doc(inline)]
pub use rrd::Entry;
mod cache;
pub use cache::*;

694
proxmox-rrd/src/rrd.rs
View File

@ -1,694 +0,0 @@
//! # Proxmox RRD format version 2
//!
//! The new format uses
//! [CBOR](https://datatracker.ietf.org/doc/html/rfc8949) as storage
//! format. This way we can use the serde serialization framework,
//! which make our code more flexible, much nicer and type safe.
//!
//! ## Features
//!
//! * Well defined data format [CBOR](https://datatracker.ietf.org/doc/html/rfc8949)
//! * Platform independent (big endian f64, hopefully a standard format?)
//! * Arbitrary number of RRAs (dynamically changeable)
use std::io::{Read, Write};
use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
use std::path::Path;
use anyhow::{bail, format_err, Error};
use serde::{Deserialize, Serialize};
use proxmox_schema::api;
use proxmox_sys::fs::{make_tmp_file, CreateOptions};
use crate::rrd_v1;
/// Proxmox RRD v2 file magic number
// openssl::sha::sha256(b"Proxmox Round Robin Database file v2.0")[0..8];
pub const PROXMOX_RRD_MAGIC_2_0: [u8; 8] = [224, 200, 228, 27, 239, 112, 122, 159];
#[api()]
#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
/// RRD data source type
pub enum DST {
/// Gauge values are stored unmodified.
Gauge,
/// Stores the difference to the previous value.
Derive,
/// Stores the difference to the previous value (like Derive), but
/// detect counter overflow (and ignores that value)
Counter,
}
#[api()]
#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
/// Consolidation function
pub enum CF {
/// Average
Average,
/// Maximum
Maximum,
/// Minimum
Minimum,
/// Use the last value
Last,
}
#[derive(Serialize, Deserialize)]
/// Data source specification
pub struct DataSource {
/// Data source type
pub dst: DST,
/// Last update time (epoch)
pub last_update: f64,
/// Stores the last value, used to compute differential value for
/// derive/counters
pub last_value: f64,
}
/// An RRD entry.
///
/// Serializes as a tuple.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(
from = "(u64, u64, Vec<Option<f64>>)",
into = "(u64, u64, Vec<Option<f64>>)"
)]
pub struct Entry {
pub start: u64,
pub resolution: u64,
pub data: Vec<Option<f64>>,
}
impl Entry {
pub const fn new(start: u64, resolution: u64, data: Vec<Option<f64>>) -> Self {
Self {
start,
resolution,
data,
}
}
/// Get a data point at a specific index which also does bound checking and returns `None` for
/// out of bounds indices.
pub fn get(&self, idx: usize) -> Option<f64> {
self.data.get(idx).copied().flatten()
}
}
impl From<Entry> for (u64, u64, Vec<Option<f64>>) {
fn from(entry: Entry) -> (u64, u64, Vec<Option<f64>>) {
(entry.start, entry.resolution, entry.data)
}
}
impl From<(u64, u64, Vec<Option<f64>>)> for Entry {
fn from(data: (u64, u64, Vec<Option<f64>>)) -> Self {
Self::new(data.0, data.1, data.2)
}
}
impl DataSource {
/// Create a new Instance
pub fn new(dst: DST) -> Self {
Self {
dst,
last_update: 0.0,
last_value: f64::NAN,
}
}
fn compute_new_value(&mut self, time: f64, mut value: f64) -> Result<f64, Error> {
if time < 0.0 {
bail!("got negative time");
}
if time <= self.last_update {
bail!("time in past ({} < {})", time, self.last_update);
}
if value.is_nan() {
bail!("new value is NAN");
}
// derive counter value
let is_counter = self.dst == DST::Counter;
if is_counter || self.dst == DST::Derive {
let time_diff = time - self.last_update;
let diff = if self.last_value.is_nan() {
0.0
} else if is_counter && value < 0.0 {
bail!("got negative value for counter");
} else if is_counter && value < self.last_value {
// Note: We do not try automatic overflow corrections, but
// we update last_value anyways, so that we can compute the diff
// next time.
self.last_value = value;
bail!("counter overflow/reset detected");
} else {
value - self.last_value
};
self.last_value = value;
value = diff / time_diff;
} else {
self.last_value = value;
}
Ok(value)
}
}
#[derive(Serialize, Deserialize)]
/// Round Robin Archive
pub struct RRA {
/// Number of seconds spaned by a single data entry.
pub resolution: u64,
/// Consolitation function.
pub cf: CF,
/// Count values computed inside this update interval.
pub last_count: u64,
/// The actual data entries.
pub data: Vec<f64>,
}
impl RRA {
/// Creates a new instance
pub fn new(cf: CF, resolution: u64, points: usize) -> Self {
Self {
cf,
resolution,
last_count: 0,
data: vec![f64::NAN; points],
}
}
/// Data slot end time
pub fn slot_end_time(&self, time: u64) -> u64 {
self.resolution * (time / self.resolution + 1)
}
/// Data slot start time
pub fn slot_start_time(&self, time: u64) -> u64 {
self.resolution * (time / self.resolution)
}
/// Data slot index
pub fn slot(&self, time: u64) -> usize {
((time / self.resolution) as usize) % self.data.len()
}
/// Directly overwrite data slots.
///
/// The caller need to set `last_update` value on the [DataSource] manually.
pub fn insert_data(
&mut self,
start: u64,
resolution: u64,
data: Vec<Option<f64>>,
) -> Result<(), Error> {
if resolution != self.resolution {
bail!("inser_data failed: got wrong resolution");
}
let mut index = self.slot(start);
for item in data {
if let Some(v) = item {
self.data[index] = v;
}
index += 1;
if index >= self.data.len() {
index = 0;
}
}
Ok(())
}
fn delete_old_slots(&mut self, time: f64, last_update: f64) {
let epoch = time as u64;
let last_update = last_update as u64;
let reso = self.resolution;
let num_entries = self.data.len() as u64;
let min_time = epoch.saturating_sub(num_entries * reso);
let min_time = self.slot_end_time(min_time);
let mut t = last_update.saturating_sub(num_entries * reso);
let mut index = self.slot(t);
for _ in 0..num_entries {
t += reso;
index += 1;
if index >= self.data.len() {
index = 0;
}
if t < min_time {
self.data[index] = f64::NAN;
} else {
break;
}
}
}
fn compute_new_value(&mut self, time: f64, last_update: f64, value: f64) {
let epoch = time as u64;
let last_update = last_update as u64;
let reso = self.resolution;
let index = self.slot(epoch);
let last_index = self.slot(last_update);
if (epoch - last_update) > reso || index != last_index {
self.last_count = 0;
}
let last_value = self.data[index];
if last_value.is_nan() {
self.last_count = 0;
}
let new_count = self.last_count.saturating_add(1);
if self.last_count == 0 {
self.data[index] = value;
self.last_count = 1;
} else {
let new_value = match self.cf {
CF::Maximum => {
if last_value > value {
last_value
} else {
value
}
}
CF::Minimum => {
if last_value < value {
last_value
} else {
value
}
}
CF::Last => value,
CF::Average => {
(last_value * (self.last_count as f64)) / (new_count as f64)
+ value / (new_count as f64)
}
};
self.data[index] = new_value;
self.last_count = new_count;
}
}
/// Extract data
///
/// Extract data from `start` to `end`. The RRA itself does not
/// store the `last_update` time, so you need to pass this a
/// parameter (see [DataSource]).
pub fn extract_data(&self, start: u64, end: u64, last_update: f64) -> Entry {
let last_update = last_update as u64;
let reso = self.resolution;
let num_entries = self.data.len() as u64;
let mut list = Vec::new();
let rrd_end = self.slot_end_time(last_update);
let rrd_start = rrd_end.saturating_sub(reso * num_entries);
let mut t = start;
let mut index = self.slot(t);
for _ in 0..num_entries {
if t > end {
break;
};
if t < rrd_start || t >= rrd_end {
list.push(None);
} else {
let value = self.data[index];
if value.is_nan() {
list.push(None);
} else {
list.push(Some(value));
}
}
t += reso;
index += 1;
if index >= self.data.len() {
index = 0;
}
}
Entry::new(start, reso, list)
}
}
#[derive(Serialize, Deserialize)]
/// Round Robin Database
pub struct RRD {
/// The data source definition
pub source: DataSource,
/// List of round robin archives
pub rra_list: Vec<RRA>,
}
impl RRD {
/// Creates a new Instance
pub fn new(dst: DST, rra_list: Vec<RRA>) -> RRD {
let source = DataSource::new(dst);
RRD { source, rra_list }
}
fn from_raw(raw: &[u8]) -> Result<Self, Error> {
if raw.len() < 8 {
bail!("not an rrd file - file is too small ({})", raw.len());
}
let rrd = if raw[0..8] == rrd_v1::PROXMOX_RRD_MAGIC_1_0 {
let v1 = rrd_v1::RRDv1::from_raw(raw)?;
v1.to_rrd_v2()
.map_err(|err| format_err!("unable to convert from old V1 format - {}", err))?
} else if raw[0..8] == PROXMOX_RRD_MAGIC_2_0 {
serde_cbor::from_slice(&raw[8..])
.map_err(|err| format_err!("unable to decode RRD file - {}", err))?
} else {
bail!("not an rrd file - unknown magic number");
};
if rrd.source.last_update < 0.0 {
bail!("rrd file has negative last_update time");
}
Ok(rrd)
}
/// Load data from a file
///
/// Setting `avoid_page_cache` uses
/// `fadvise(..,POSIX_FADV_DONTNEED)` to avoid keeping the data in
/// the linux page cache.
pub fn load(path: &Path, avoid_page_cache: bool) -> Result<Self, std::io::Error> {
let mut file = std::fs::File::open(path)?;
let buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0);
let mut raw = Vec::with_capacity(buffer_size);
file.read_to_end(&mut raw)?;
if avoid_page_cache {
nix::fcntl::posix_fadvise(
file.as_raw_fd(),
0,
buffer_size as i64,
nix::fcntl::PosixFadviseAdvice::POSIX_FADV_DONTNEED,
)
.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))?;
}
match Self::from_raw(&raw) {
Ok(rrd) => Ok(rrd),
Err(err) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
err.to_string(),
)),
}
}
/// Store data into a file (atomic replace file)
///
/// Setting `avoid_page_cache` uses
/// `fadvise(..,POSIX_FADV_DONTNEED)` to avoid keeping the data in
/// the linux page cache.
pub fn save(
&self,
path: &Path,
options: CreateOptions,
avoid_page_cache: bool,
) -> Result<(), Error> {
let (fd, tmp_path) = make_tmp_file(path, options)?;
let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
let mut try_block = || -> Result<(), Error> {
let mut data: Vec<u8> = Vec::new();
data.extend(PROXMOX_RRD_MAGIC_2_0);
serde_cbor::to_writer(&mut data, self)?;
file.write_all(&data)?;
if avoid_page_cache {
nix::fcntl::posix_fadvise(
file.as_raw_fd(),
0,
data.len() as i64,
nix::fcntl::PosixFadviseAdvice::POSIX_FADV_DONTNEED,
)?;
}
Ok(())
};
match try_block() {
Ok(()) => (),
error => {
let _ = nix::unistd::unlink(&tmp_path);
return error;
}
}
if let Err(err) = std::fs::rename(&tmp_path, path) {
let _ = nix::unistd::unlink(&tmp_path);
bail!("Atomic rename failed - {}", err);
}
Ok(())
}
/// Returns the last update time.
pub fn last_update(&self) -> f64 {
self.source.last_update
}
/// Update the value (in memory)
///
/// Note: This does not call [Self::save].
pub fn update(&mut self, time: f64, value: f64) {
let value = match self.source.compute_new_value(time, value) {
Ok(value) => value,
Err(err) => {
log::error!("rrd update failed: {}", err);
return;
}
};
let last_update = self.source.last_update;
self.source.last_update = time;
for rra in self.rra_list.iter_mut() {
rra.delete_old_slots(time, last_update);
rra.compute_new_value(time, last_update, value);
}
}
/// Extract data from the archive
///
/// This selects the RRA with specified [CF] and (minimum)
/// resolution, and extract data from `start` to `end`.
///
/// `start`: Start time. If not specified, we simply extract 10 data points.
/// `end`: End time. Default is to use the current time.
pub fn extract_data(
&self,
cf: CF,
resolution: u64,
start: Option<u64>,
end: Option<u64>,
) -> Result<Entry, Error> {
let mut rra: Option<&RRA> = None;
for item in self.rra_list.iter() {
if item.cf != cf {
continue;
}
if item.resolution > resolution {
continue;
}
if let Some(current) = rra {
if item.resolution > current.resolution {
rra = Some(item);
}
} else {
rra = Some(item);
}
}
match rra {
Some(rra) => {
let end = end.unwrap_or_else(|| proxmox_time::epoch_f64() as u64);
let start = start.unwrap_or_else(|| end.saturating_sub(10 * rra.resolution));
Ok(rra.extract_data(start, end, self.source.last_update))
}
None => bail!("unable to find RRA suitable ({:?}:{})", cf, resolution),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn basic_rra_maximum_gauge_test() -> Result<(), Error> {
let rra = RRA::new(CF::Maximum, 60, 5);
let mut rrd = RRD::new(DST::Gauge, vec![rra]);
for i in 2..10 {
rrd.update((i as f64) * 30.0, i as f64);
}
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Maximum, 60, Some(0), Some(5 * 60))?;
assert_eq!(start, 0);
assert_eq!(resolution, 60);
assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]);
Ok(())
}
#[test]
fn basic_rra_minimum_gauge_test() -> Result<(), Error> {
let rra = RRA::new(CF::Minimum, 60, 5);
let mut rrd = RRD::new(DST::Gauge, vec![rra]);
for i in 2..10 {
rrd.update((i as f64) * 30.0, i as f64);
}
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Minimum, 60, Some(0), Some(5 * 60))?;
assert_eq!(start, 0);
assert_eq!(resolution, 60);
assert_eq!(data, [None, Some(2.0), Some(4.0), Some(6.0), Some(8.0)]);
Ok(())
}
#[test]
fn basic_rra_last_gauge_test() -> Result<(), Error> {
let rra = RRA::new(CF::Last, 60, 5);
let mut rrd = RRD::new(DST::Gauge, vec![rra]);
for i in 2..10 {
rrd.update((i as f64) * 30.0, i as f64);
}
assert!(
rrd.extract_data(CF::Average, 60, Some(0), Some(5 * 60))
.is_err(),
"CF::Average should not exist"
);
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Last, 60, Some(0), Some(20 * 60))?;
assert_eq!(start, 0);
assert_eq!(resolution, 60);
assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]);
Ok(())
}
#[test]
fn basic_rra_average_derive_test() -> Result<(), Error> {
let rra = RRA::new(CF::Average, 60, 5);
let mut rrd = RRD::new(DST::Derive, vec![rra]);
for i in 2..10 {
rrd.update((i as f64) * 30.0, (i * 60) as f64);
}
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?;
assert_eq!(start, 60);
assert_eq!(resolution, 60);
assert_eq!(data, [Some(1.0), Some(2.0), Some(2.0), Some(2.0), None]);
Ok(())
}
#[test]
fn basic_rra_average_gauge_test() -> Result<(), Error> {
let rra = RRA::new(CF::Average, 60, 5);
let mut rrd = RRD::new(DST::Gauge, vec![rra]);
for i in 2..10 {
rrd.update((i as f64) * 30.0, i as f64);
}
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?;
assert_eq!(start, 60);
assert_eq!(resolution, 60);
assert_eq!(data, [Some(2.5), Some(4.5), Some(6.5), Some(8.5), None]);
for i in 10..14 {
rrd.update((i as f64) * 30.0, i as f64);
}
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?;
assert_eq!(start, 60);
assert_eq!(resolution, 60);
assert_eq!(data, [None, Some(4.5), Some(6.5), Some(8.5), Some(10.5)]);
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Average, 60, Some(3 * 60), Some(8 * 60))?;
assert_eq!(start, 3 * 60);
assert_eq!(resolution, 60);
assert_eq!(data, [Some(6.5), Some(8.5), Some(10.5), Some(12.5), None]);
// add much newer value (should delete all previous/outdated value)
let i = 100;
rrd.update((i as f64) * 30.0, i as f64);
println!("TEST {:?}", serde_json::to_string_pretty(&rrd));
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(100 * 30 + 5 * 60))?;
assert_eq!(start, 100 * 30);
assert_eq!(resolution, 60);
assert_eq!(data, [Some(100.0), None, None, None, None]);
// extract with end time smaller than start time
let Entry {
start,
resolution,
data,
} = rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(60))?;
assert_eq!(start, 100 * 30);
assert_eq!(resolution, 60);
assert_eq!(data, []);
Ok(())
}
}

295
proxmox-rrd/src/rrd_v1.rs
View File

@ -1,295 +0,0 @@
use std::io::Read;
use anyhow::Error;
use bitflags::bitflags;
/// The number of data entries per RRA
pub const RRD_DATA_ENTRIES: usize = 70;
/// Proxmox RRD file magic number
// openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8];
pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186];
use crate::rrd::{DataSource, CF, DST, RRA, RRD};
bitflags! {
/// Flags to specify the data source type and consolidation function
pub struct RRAFlags: u64 {
// Data Source Types
const DST_GAUGE = 1;
const DST_DERIVE = 2;
const DST_COUNTER = 4;
const DST_MASK = 255; // first 8 bits
// Consolidation Functions
const CF_AVERAGE = 1 << 8;
const CF_MAX = 2 << 8;
const CF_MASK = 255 << 8;
}
}
/// Round Robin Archive with [RRD_DATA_ENTRIES] data slots.
///
/// This data structure is used inside [RRD] and directly written to the
/// RRD files.
#[repr(C)]
pub struct RRAv1 {
/// Defined the data source type and consolidation function
pub flags: RRAFlags,
/// Resolution (seconds)
pub resolution: u64,
/// Last update time (epoch)
pub last_update: f64,
/// Count values computed inside this update interval
pub last_count: u64,
/// Stores the last value, used to compute differential value for derive/counters
pub counter_value: f64,
/// Data slots
pub data: [f64; RRD_DATA_ENTRIES],
}
impl RRAv1 {
fn extract_data(&self) -> (u64, u64, Vec<Option<f64>>) {
let reso = self.resolution;
let mut list = Vec::new();
let rra_end = reso * ((self.last_update as u64) / reso);
let rra_start = rra_end - reso * (RRD_DATA_ENTRIES as u64);
let mut t = rra_start;
let mut index = ((t / reso) % (RRD_DATA_ENTRIES as u64)) as usize;
for _ in 0..RRD_DATA_ENTRIES {
let value = self.data[index];
if value.is_nan() {
list.push(None);
} else {
list.push(Some(value));
}
t += reso;
index = (index + 1) % RRD_DATA_ENTRIES;
}
(rra_start, reso, list)
}
}
/// Round Robin Database file format with fixed number of [RRA]s
#[repr(C)]
// Note: Avoid alignment problems by using 8byte types only
pub struct RRDv1 {
/// The magic number to identify the file type
pub magic: [u8; 8],
/// Hourly data (average values)
pub hour_avg: RRAv1,
/// Hourly data (maximum values)
pub hour_max: RRAv1,
/// Dayly data (average values)
pub day_avg: RRAv1,
/// Dayly data (maximum values)
pub day_max: RRAv1,
/// Weekly data (average values)
pub week_avg: RRAv1,
/// Weekly data (maximum values)
pub week_max: RRAv1,
/// Monthly data (average values)
pub month_avg: RRAv1,
/// Monthly data (maximum values)
pub month_max: RRAv1,
/// Yearly data (average values)
pub year_avg: RRAv1,
/// Yearly data (maximum values)
pub year_max: RRAv1,
}
impl RRDv1 {
pub fn from_raw(mut raw: &[u8]) -> Result<Self, std::io::Error> {
let expected_len = std::mem::size_of::<RRDv1>();
if raw.len() != expected_len {
let msg = format!("wrong data size ({} != {})", raw.len(), expected_len);
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
let mut rrd: RRDv1 = unsafe { std::mem::zeroed() };
unsafe {
let rrd_slice =
std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len);
raw.read_exact(rrd_slice)?;
}
if rrd.magic != PROXMOX_RRD_MAGIC_1_0 {
let msg = "wrong magic number".to_string();
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
}
Ok(rrd)
}
pub fn to_rrd_v2(&self) -> Result<RRD, Error> {
let mut rra_list = Vec::new();
// old format v1:
//
// hour 1 min, 70 points
// day 30 min, 70 points
// week 3 hours, 70 points
// month 12 hours, 70 points
// year 1 week, 70 points
//
// new default for RRD v2:
//
// day 1 min, 1440 points
// month 30 min, 1440 points
// year 365 min (6h), 1440 points
// decade 1 week, 570 points
// Linear extrapolation
fn extrapolate_data(
start: u64,
reso: u64,
factor: u64,
data: Vec<Option<f64>>,
) -> (u64, u64, Vec<Option<f64>>) {
let mut new = Vec::new();
for i in 0..data.len() {
let mut next = i + 1;
if next >= data.len() {
next = 0
};
let v = data[i];
let v1 = data[next];
match (v, v1) {
(Some(v), Some(v1)) => {
let diff = (v1 - v) / (factor as f64);
for j in 0..factor {
new.push(Some(v + diff * (j as f64)));
}
}
(Some(v), None) => {
new.push(Some(v));
for _ in 0..factor - 1 {
new.push(None);
}
}
(None, Some(v1)) => {
for _ in 0..factor - 1 {
new.push(None);
}
new.push(Some(v1));
}
(None, None) => {
for _ in 0..factor {
new.push(None);
}
}
}
}
(start, reso / factor, new)
}
// Try to convert to new, higher capacity format
// compute daily average (merge old self.day_avg and self.hour_avg
let mut day_avg = RRA::new(CF::Average, 60, 1440);
let (start, reso, data) = self.day_avg.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 30, data);
day_avg.insert_data(start, reso, data)?;
let (start, reso, data) = self.hour_avg.extract_data();
day_avg.insert_data(start, reso, data)?;
// compute daily maximum (merge old self.day_max and self.hour_max
let mut day_max = RRA::new(CF::Maximum, 60, 1440);
let (start, reso, data) = self.day_max.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 30, data);
day_max.insert_data(start, reso, data)?;
let (start, reso, data) = self.hour_max.extract_data();
day_max.insert_data(start, reso, data)?;
// compute monthly average (merge old self.month_avg,
// self.week_avg and self.day_avg)
let mut month_avg = RRA::new(CF::Average, 30 * 60, 1440);
let (start, reso, data) = self.month_avg.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 24, data);
month_avg.insert_data(start, reso, data)?;
let (start, reso, data) = self.week_avg.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 6, data);
month_avg.insert_data(start, reso, data)?;
let (start, reso, data) = self.day_avg.extract_data();
month_avg.insert_data(start, reso, data)?;
// compute monthly maximum (merge old self.month_max,
// self.week_max and self.day_max)
let mut month_max = RRA::new(CF::Maximum, 30 * 60, 1440);
let (start, reso, data) = self.month_max.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 24, data);
month_max.insert_data(start, reso, data)?;
let (start, reso, data) = self.week_max.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 6, data);
month_max.insert_data(start, reso, data)?;
let (start, reso, data) = self.day_max.extract_data();
month_max.insert_data(start, reso, data)?;
// compute yearly average (merge old self.year_avg)
let mut year_avg = RRA::new(CF::Average, 6 * 3600, 1440);
let (start, reso, data) = self.year_avg.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 28, data);
year_avg.insert_data(start, reso, data)?;
// compute yearly maximum (merge old self.year_avg)
let mut year_max = RRA::new(CF::Maximum, 6 * 3600, 1440);
let (start, reso, data) = self.year_max.extract_data();
let (start, reso, data) = extrapolate_data(start, reso, 28, data);
year_max.insert_data(start, reso, data)?;
// compute decade average (merge old self.year_avg)
let mut decade_avg = RRA::new(CF::Average, 7 * 86400, 570);
let (start, reso, data) = self.year_avg.extract_data();
decade_avg.insert_data(start, reso, data)?;
// compute decade maximum (merge old self.year_max)
let mut decade_max = RRA::new(CF::Maximum, 7 * 86400, 570);
let (start, reso, data) = self.year_max.extract_data();
decade_max.insert_data(start, reso, data)?;
rra_list.push(day_avg);
rra_list.push(day_max);
rra_list.push(month_avg);
rra_list.push(month_max);
rra_list.push(year_avg);
rra_list.push(year_max);
rra_list.push(decade_avg);
rra_list.push(decade_max);
// use values from hour_avg for source (all RRAv1 must have the same config)
let dst = if self.hour_avg.flags.contains(RRAFlags::DST_COUNTER) {
DST::Counter
} else if self.hour_avg.flags.contains(RRAFlags::DST_DERIVE) {
DST::Derive
} else {
DST::Gauge
};
let source = DataSource {
dst,
last_value: f64::NAN,
last_update: self.hour_avg.last_update, // IMPORTANT!
};
Ok(RRD { source, rra_list })
}
}

56
proxmox-rrd/tests/file_format_test.rs
View File

@ -1,56 +0,0 @@
use std::path::Path;
use std::process::Command;
use anyhow::{bail, Error};
use proxmox_rrd::rrd::RRD;
use proxmox_sys::fs::CreateOptions;
fn compare_file(fn1: &str, fn2: &str) -> Result<(), Error> {
let status = Command::new("/usr/bin/cmp")
.arg(fn1)
.arg(fn2)
.status()
.expect("failed to execute process");
if !status.success() {
bail!("file compare failed");
}
Ok(())
}
const RRD_V1_FN: &str = "./tests/testdata/cpu.rrd_v1";
const RRD_V2_FN: &str = "./tests/testdata/cpu.rrd_v2";
// make sure we can load and convert RRD v1
#[test]
fn upgrade_from_rrd_v1() -> Result<(), Error> {
let rrd = RRD::load(Path::new(RRD_V1_FN), true)?;
const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.upgraded";
let new_path = Path::new(RRD_V2_NEW_FN);
rrd.save(new_path, CreateOptions::new(), true)?;
let result = compare_file(RRD_V2_FN, RRD_V2_NEW_FN);
let _ = std::fs::remove_file(RRD_V2_NEW_FN);
result?;
Ok(())
}
// make sure we can load and save RRD v2
#[test]
fn load_and_save_rrd_v2() -> Result<(), Error> {
let rrd = RRD::load(Path::new(RRD_V2_FN), true)?;
const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.saved";
let new_path = Path::new(RRD_V2_NEW_FN);
rrd.save(new_path, CreateOptions::new(), true)?;
let result = compare_file(RRD_V2_FN, RRD_V2_NEW_FN);
let _ = std::fs::remove_file(RRD_V2_NEW_FN);
result?;
Ok(())
}

BIN
proxmox-rrd/tests/testdata/cpu.rrd_v1 vendored
View File

Binary file not shown.

BIN
proxmox-rrd/tests/testdata/cpu.rrd_v2 vendored
View File

Binary file not shown.

9
src/acme/client.rs
View File

@ -116,6 +116,7 @@ impl AcmeClient {
tos_agreed: bool,
contact: Vec<String>,
rsa_bits: Option<u32>,
eab_creds: Option<(String, String)>,
) -> Result<&'a Account, anyhow::Error> {
self.tos = if tos_agreed {
self.terms_of_service_url().await?.map(str::to_owned)
@ -123,10 +124,14 @@ impl AcmeClient {
None
};
let account = Account::creator()
let mut account = Account::creator()
.set_contacts(contact)
.agree_to_tos(tos_agreed);
if let Some((eab_kid, eab_hmac_key)) = eab_creds {
account = account.set_eab_credentials(eab_kid, eab_hmac_key)?;
}
let account = if let Some(bits) = rsa_bits {
account.generate_rsa_key(bits)?
} else {
@ -572,7 +577,7 @@ impl AcmeClient {
Self::execute(&mut self.http_client, request, &mut self.nonce).await
}
async fn directory(&mut self) -> Result<&Directory, Error> {
pub async fn directory(&mut self) -> Result<&Directory, Error> {
Ok(Self::get_directory(
&mut self.http_client,
&self.directory_url,

37
src/api2/config/acme.rs
View File

@ -182,6 +182,16 @@ fn account_contact_from_string(s: &str) -> Vec<String> {
description: "The ACME Directory.",
optional: true,
},
eab_kid: {
type: String,
description: "Key Identifier for External Account Binding.",
optional: true,
},
eab_hmac_key: {
type: String,
description: "HMAC Key for External Account Binding.",
optional: true,
}
},
},
access: {
@ -196,6 +206,8 @@ fn register_account(
contact: String,
tos_url: Option<String>,
directory: Option<String>,
eab_kid: Option<String>,
eab_hmac_key: Option<String>,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@ -204,6 +216,15 @@ fn register_account(
AcmeAccountName::from_string_unchecked("default".to_string())
});
// TODO: this should be done via the api definition, but
// the api schema currently lacks this ability (2023-11-06)
if eab_kid.is_some() != eab_hmac_key.is_some() {
http_bail!(
BAD_REQUEST,
"either both or none of 'eab_kid' and 'eab_hmac_key' have to be set."
);
}
if Path::new(&crate::config::acme::account_path(&name)).exists() {
http_bail!(BAD_REQUEST, "account {} already exists", name);
}
@ -224,8 +245,15 @@ fn register_account(
task_log!(worker, "Registering ACME account '{}'...", &name);
let account =
do_register_account(&mut client, &name, tos_url.is_some(), contact, None).await?;
let account = do_register_account(
&mut client,
&name,
tos_url.is_some(),
contact,
None,
eab_kid.zip(eab_hmac_key),
)
.await?;
task_log!(
worker,
@ -244,10 +272,11 @@ pub async fn do_register_account<'a>(
agree_to_tos: bool,
contact: String,
rsa_bits: Option<u32>,
eab_creds: Option<(String, String)>,
) -> Result<&'a Account, Error> {
let contact = account_contact_from_string(&contact);
client
.new_account(name, agree_to_tos, contact, rsa_bits)
.new_account(name, agree_to_tos, contact, rsa_bits, eab_creds)
.await
}
@ -460,9 +489,11 @@ pub struct PluginConfig {
ty: String,
/// DNS Api name.
#[serde(skip_serializing_if = "Option::is_none", default)]
api: Option<String>,
/// Plugin configuration data.
#[serde(skip_serializing_if = "Option::is_none", default)]
data: Option<String>,
/// Extra delay in seconds to wait before requesting validation.

9
src/api2/config/changer.rs
View File

@ -138,6 +138,8 @@ pub fn list_changers(
pub enum DeletableProperty {
/// Delete export-slots.
ExportSlots,
/// Delete eject-before-unload.
EjectBeforeUnload,
}
#[api(
@ -194,6 +196,9 @@ pub fn update_changer(
DeletableProperty::ExportSlots => {
data.export_slots = None;
}
DeletableProperty::EjectBeforeUnload => {
data.eject_before_unload = None;
}
}
}
}
@ -222,6 +227,10 @@ pub fn update_changer(
}
}
if let Some(eject_before_unload) = update.eject_before_unload {
data.eject_before_unload = Some(eject_before_unload);
}
config.set_data(&name, "changer", &data)?;
pbs_config::drive::save_config(&config)?;

92
src/api2/node/certificates.rs
View File

@ -64,71 +64,39 @@ const ACME_SUBDIRS: SubdirMap = &[(
#[serde(rename_all = "kebab-case")]
pub struct CertificateInfo {
/// Certificate file name.
#[serde(skip_serializing_if = "Option::is_none")]
filename: Option<String>,
pub filename: String,
/// Certificate subject name.
subject: String,
pub subject: String,
/// List of certificate's SubjectAlternativeName entries.
san: Vec<String>,
pub san: Vec<String>,
/// Certificate issuer name.
issuer: String,
pub issuer: String,
/// Certificate's notBefore timestamp (UNIX epoch).
#[serde(skip_serializing_if = "Option::is_none")]
notbefore: Option<i64>,
pub notbefore: Option<i64>,
/// Certificate's notAfter timestamp (UNIX epoch).
#[serde(skip_serializing_if = "Option::is_none")]
notafter: Option<i64>,
pub notafter: Option<i64>,
/// Certificate in PEM format.
#[serde(skip_serializing_if = "Option::is_none")]
pem: Option<String>,
pub pem: Option<String>,
/// Certificate's public key algorithm.
public_key_type: String,
pub public_key_type: String,
/// Certificate's public key size if available.
#[serde(skip_serializing_if = "Option::is_none")]
public_key_bits: Option<u32>,
pub public_key_bits: Option<u32>,
/// The SSL Fingerprint.
fingerprint: Option<String>,
}
impl TryFrom<&cert::CertInfo> for CertificateInfo {
type Error = Error;
fn try_from(info: &cert::CertInfo) -> Result<Self, Self::Error> {
let pubkey = info.public_key()?;
Ok(Self {
filename: None,
subject: info.subject_name()?,
san: info
.subject_alt_names()
.map(|san| {
san.into_iter()
// FIXME: Support `.ipaddress()`?
.filter_map(|name| name.dnsname().map(str::to_owned))
.collect()
})
.unwrap_or_default(),
issuer: info.issuer_name()?,
notbefore: info.not_before_unix().ok(),
notafter: info.not_after_unix().ok(),
pem: None,
public_key_type: openssl::nid::Nid::from_raw(pubkey.id().as_raw())
.long_name()
.unwrap_or("<unsupported key type>")
.to_owned(),
public_key_bits: Some(pubkey.bits()),
fingerprint: Some(info.fingerprint()?),
})
}
#[serde(skip_serializing_if = "Option::is_none")]
pub fingerprint: Option<String>,
}
fn get_certificate_pem() -> Result<String, Error> {
@ -162,12 +130,31 @@ fn pem_to_cert_info(pem: &[u8]) -> Result<cert::CertInfo, Error> {
/// Get certificate info.
pub fn get_info() -> Result<Vec<CertificateInfo>, Error> {
let cert_pem = get_certificate_pem()?;
let cert = pem_to_cert_info(cert_pem.as_bytes())?;
let info = pem_to_cert_info(cert_pem.as_bytes())?;
let pubkey = info.public_key()?;
Ok(vec![CertificateInfo {
filename: Some("proxy.pem".to_string()), // we only have the one
filename: "proxy.pem".to_string(), // we only have the one
pem: Some(cert_pem),
..CertificateInfo::try_from(&cert)?
subject: info.subject_name()?,
san: info
.subject_alt_names()
.map(|san| {
san.into_iter()
// FIXME: Support `.ipaddress()`?
.filter_map(|name| name.dnsname().map(str::to_owned))
.collect()
})
.unwrap_or_default(),
issuer: info.issuer_name()?,
notbefore: info.not_before_unix().ok(),
notafter: info.not_after_unix().ok(),
public_key_type: openssl::nid::Nid::from_raw(pubkey.id().as_raw())
.long_name()
.unwrap_or("<unsupported key type>")
.to_owned(),
public_key_bits: Some(pubkey.bits()),
fingerprint: Some(info.fingerprint()?),
}])
}
@ -176,7 +163,10 @@ pub fn get_info() -> Result<Vec<CertificateInfo>, Error> {
properties: {
node: { schema: NODE_SCHEMA },
certificates: { description: "PEM encoded certificate (chain)." },
key: { description: "PEM encoded private key." },
key: {
description: "PEM encoded private key.",
optional: true,
},
// FIXME: widget-toolkit should have an option to disable using these 2 parameters...
restart: {
description: "UI compatibility parameter, ignored",
@ -205,10 +195,16 @@ pub fn get_info() -> Result<Vec<CertificateInfo>, Error> {
/// Upload a custom certificate.
pub async fn upload_custom_certificate(
certificates: String,
key: String,
key: Option<String>,
) -> Result<Vec<CertificateInfo>, Error> {
let certificates = X509::stack_from_pem(certificates.as_bytes())
.map_err(|err| format_err!("failed to decode certificate chain: {}", err))?;
let key = match key {
Some(key) => key,
None => proxmox_sys::fs::file_read_string(configdir!("/proxy.key"))?,
};
let key = PKey::private_key_from_pem(key.as_bytes())
.map_err(|err| format_err!("failed to parse private key: {}", err))?;

6
src/api2/status.rs
View File

@ -68,9 +68,9 @@ pub async fn datastore_status(
let mut entry = DataStoreStatusListItem {
store: store.clone(),
total: status.total as i64,
used: status.used as i64,
avail: status.available as i64,
total: Some(status.total),
used: Some(status.used),
avail: Some(status.available),
history: None,
history_start: None,
history_delta: None,

31
src/api2/tape/backup.rs
View File

@ -9,13 +9,13 @@ use proxmox_schema::api;
use proxmox_sys::{task_log, task_warn, WorkerTaskContext};
use pbs_api_types::{
print_ns_and_snapshot, print_store_and_ns, Authid, GroupFilter, MediaPoolConfig, Operation,
print_ns_and_snapshot, print_store_and_ns, Authid, MediaPoolConfig, Operation,
TapeBackupJobConfig, TapeBackupJobSetup, TapeBackupJobStatus, Userid, JOB_ID_SCHEMA,
PRIV_DATASTORE_READ, PRIV_TAPE_AUDIT, PRIV_TAPE_WRITE, UPID_SCHEMA,
};
use pbs_config::CachedUserInfo;
use pbs_datastore::backup_info::{BackupDir, BackupGroup, BackupInfo};
use pbs_datastore::backup_info::{BackupDir, BackupInfo};
use pbs_datastore::{DataStore, StoreProgress};
use proxmox_rest_server::WorkerTask;
@ -411,31 +411,24 @@ fn backup_worker(
group_list.sort_unstable_by(|a, b| a.group().cmp(b.group()));
let (group_list, group_count) = if let Some(group_filters) = &setup.group_filter {
let filter_fn = |group: &BackupGroup, group_filters: &[GroupFilter]| {
group_filters.iter().any(|filter| group.matches(filter))
let group_count_full = group_list.len();
let group_list = match &setup.group_filter {
Some(f) => group_list
.into_iter()
.filter(|group| group.group().apply_filters(f))
.collect(),
None => group_list,
};
let group_count_full = group_list.len();
let list: Vec<BackupGroup> = group_list
.into_iter()
.filter(|group| filter_fn(group, group_filters))
.collect();
let group_count = list.len();
task_log!(
worker,
"found {} groups (out of {} total)",
group_count,
group_list.len(),
group_count_full
);
(list, group_count)
} else {
let group_count = group_list.len();
task_log!(worker, "found {} groups", group_count);
(group_list, group_count)
};
let mut progress = StoreProgress::new(group_count as u64);
let mut progress = StoreProgress::new(group_list.len() as u64);
let latest_only = setup.latest_only.unwrap_or(false);

82
src/api2/tape/drive.rs
View File

@ -16,7 +16,7 @@ use proxmox_uuid::Uuid;
use pbs_api_types::{
Authid, DriveListEntry, LabelUuidMap, Lp17VolumeStatistics, LtoDriveAndMediaStatus,
LtoTapeDrive, MamAttribute, MediaIdFlat, CHANGER_NAME_SCHEMA, DRIVE_NAME_SCHEMA,
LtoTapeDrive, MamAttribute, MediaIdFlat, TapeDensity, CHANGER_NAME_SCHEMA, DRIVE_NAME_SCHEMA,
MEDIA_LABEL_SCHEMA, MEDIA_POOL_NAME_SCHEMA, UPID_SCHEMA,
};
@ -36,8 +36,7 @@ use crate::{
changer::update_changer_online_status,
drive::{
get_tape_device_state, lock_tape_device, media_changer, open_drive,
open_lto_tape_drive, required_media_changer, set_tape_device_state, LtoTapeHandle,
TapeDriver,
required_media_changer, set_tape_device_state, LtoTapeHandle, TapeDriver,
},
encryption_keys::insert_key,
file_formats::{MediaLabel, MediaSetLabel},
@ -309,6 +308,21 @@ pub fn format_media(
let mut handle = open_drive(&config, &drive)?;
if !fast.unwrap_or(true) {
let drive_config: LtoTapeDrive = config.lookup("lto", &drive)?;
let file = open_lto_tape_device(&drive_config.path)?;
let mut handle = LtoTapeHandle::new(file)?;
if let Ok(status) = handle.get_drive_and_media_status() {
if status.density >= TapeDensity::LTO9 {
task_log!(worker, "Slow formatting LTO9+ media.");
task_log!(
worker,
"This can take a very long time due to media optimization."
);
}
}
}
match handle.read_label() {
Err(err) => {
if let Some(label) = label_text {
@ -524,6 +538,14 @@ fn write_media_label(
label: MediaLabel,
pool: Option<String>,
) -> Result<(), Error> {
let mut inventory = Inventory::new(TAPE_STATUS_DIR);
inventory.reload()?;
if inventory
.find_media_by_label_text(&label.label_text)?
.is_some()
{
bail!("Media with label '{}' already exists", label.label_text);
}
drive.label_tape(&label)?;
if let Some(ref pool) = pool {
task_log!(
@ -547,8 +569,6 @@ fn write_media_label(
// Create the media catalog
MediaCatalog::overwrite(TAPE_STATUS_DIR, &media_id, false)?;
let mut inventory = Inventory::new(TAPE_STATUS_DIR);
inventory.store(media_id.clone(), false)?;
drive.rewind()?;
@ -610,7 +630,7 @@ pub async fn restore_key(drive: String, password: String) -> Result<(), Error> {
run_drive_blocking_task(drive.clone(), "restore key".to_string(), move |config| {
let mut drive = open_drive(&config, &drive)?;
let (_media_id, key_config) = drive.read_label()?;
let (_media_id, key_config) = drive.read_label_without_loading_key()?;
if let Some(key_config) = key_config {
let password_fn = || Ok(password.as_bytes().to_vec());
@ -657,9 +677,6 @@ pub async fn read_label(drive: String, inventorize: Option<bool>) -> Result<Medi
let label = if let Some(ref set) = media_id.media_set_label {
let key = &set.encryption_key_fingerprint;
if let Err(err) = drive.set_encryption(key.clone().map(|fp| (fp, set.uuid.clone()))) {
eprintln!("unable to load encryption key: {}", err); // best-effort only
}
MediaIdFlat {
ctime: media_id.label.ctime,
encryption_key_fingerprint: key.as_ref().map(|fp| fp.signature()),
@ -813,17 +830,27 @@ pub async fn inventory(drive: String) -> Result<Vec<LabelUuidMap>, Error> {
let label_text = label_text.to_string();
if let Some(media_id) = inventory.find_media_by_label_text(&label_text) {
match inventory.find_media_by_label_text(&label_text) {
Ok(Some(media_id)) => {
list.push(LabelUuidMap {
label_text,
uuid: Some(media_id.label.uuid.clone()),
});
} else {
}
Ok(None) => {
list.push(LabelUuidMap {
label_text,
uuid: None,
});
}
Err(err) => {
log::warn!("error getting unique media label: {err}");
list.push(LabelUuidMap {
label_text,
uuid: None,
});
}
};
}
Ok(list)
@ -901,12 +928,22 @@ pub fn update_inventory(
let label_text = label_text.to_string();
if !read_all_labels {
if let Some(media_id) = inventory.find_media_by_label_text(&label_text) {
if !catalog || MediaCatalog::exists(TAPE_STATUS_DIR, &media_id.label.uuid) {
match inventory.find_media_by_label_text(&label_text) {
Ok(Some(media_id)) => {
if !catalog
|| MediaCatalog::exists(TAPE_STATUS_DIR, &media_id.label.uuid)
{
task_log!(worker, "media '{}' already inventoried", label_text);
continue;
}
}
Err(err) => {
task_warn!(worker, "error getting media by unique label: {err}");
// we can't be sure which uuid it is
continue;
}
Ok(None) => {} // ok to inventorize
}
}
if let Err(err) = changer.load_media(&label_text) {
@ -1064,7 +1101,8 @@ fn barcode_label_media_worker(
}
inventory.reload()?;
if inventory.find_media_by_label_text(&label_text).is_some() {
match inventory.find_media_by_label_text(&label_text) {
Ok(Some(_)) => {
task_log!(
worker,
"media '{}' already inventoried (already labeled)",
@ -1072,6 +1110,12 @@ fn barcode_label_media_worker(
);
continue;
}
Err(err) => {
task_warn!(worker, "error getting media by unique label: {err}",);
continue;
}
Ok(None) => {} // ok to label
}
task_log!(worker, "checking/loading media '{}'", label_text);
@ -1144,7 +1188,7 @@ pub async fn cartridge_memory(drive: String) -> Result<Vec<MamAttribute>, Error>
"reading cartridge memory".to_string(),
move |config| {
let drive_config: LtoTapeDrive = config.lookup("lto", &drive)?;
let mut handle = open_lto_tape_drive(&drive_config)?;
let mut handle = LtoTapeHandle::open_lto_drive(&drive_config)?;
handle.cartridge_memory()
},
@ -1174,7 +1218,7 @@ pub async fn volume_statistics(drive: String) -> Result<Lp17VolumeStatistics, Er
"reading volume statistics".to_string(),
move |config| {
let drive_config: LtoTapeDrive = config.lookup("lto", &drive)?;
let mut handle = open_lto_tape_drive(&drive_config)?;
let mut handle = LtoTapeHandle::open_lto_drive(&drive_config)?;
handle.volume_statistics()
},
@ -1311,12 +1355,6 @@ pub fn catalog_media(
inventory.store(media_id.clone(), false)?;
return Ok(());
}
let encrypt_fingerprint = set
.encryption_key_fingerprint
.clone()
.map(|fp| (fp, set.uuid.clone()));
drive.set_encryption(encrypt_fingerprint)?;
let _pool_lock = lock_media_pool(TAPE_STATUS_DIR, &set.pool)?;
let media_set_lock = lock_media_set(TAPE_STATUS_DIR, &set.uuid, None)?;

84
src/api2/tape/media.rs
View File

@ -3,7 +3,7 @@ use std::collections::HashSet;
use anyhow::{bail, format_err, Error};
use proxmox_router::{list_subdirs_api_method, Permission, Router, RpcEnvironment, SubdirMap};
use proxmox_schema::api;
use proxmox_schema::{api, param_bail};
use proxmox_uuid::Uuid;
use pbs_api_types::{
@ -290,6 +290,11 @@ pub async fn list_media(
properties: {
"label-text": {
schema: MEDIA_LABEL_SCHEMA,
optional: true,
},
uuid: {
schema: MEDIA_UUID_SCHEMA,
optional: true,
},
"vault-name": {
schema: VAULT_NAME_SCHEMA,
@ -299,15 +304,33 @@ pub async fn list_media(
},
)]
/// Change Tape location to vault (if given), or offline.
pub fn move_tape(label_text: String, vault_name: Option<String>) -> Result<(), Error> {
pub fn move_tape(
label_text: Option<String>,
uuid: Option<Uuid>,
vault_name: Option<String>,
) -> Result<(), Error> {
let mut inventory = Inventory::load(TAPE_STATUS_DIR)?;
let uuid = inventory
.find_media_by_label_text(&label_text)
.ok_or_else(|| format_err!("no such media '{}'", label_text))?
.label
.uuid
.clone();
let uuid = match (uuid, label_text) {
(Some(_), Some(_)) => {
param_bail!(
"format-text",
format_err!("A uuid is given, no label-text is expected.")
);
}
(None, None) => {
param_bail!(
"uuid",
format_err!("No label-text is given, a uuid is required.")
);
}
(Some(uuid), None) => uuid,
(None, Some(label_text)) => match inventory.find_media_by_label_text(&label_text) {
Ok(Some(media_id)) => media_id.label.uuid.clone(),
Ok(None) => bail!("no such media '{}'", label_text),
Err(err) => bail!("error getting media from unique label: {err}"),
},
};
if let Some(vault_name) = vault_name {
inventory.set_media_location_vault(&uuid, &vault_name)?;
@ -323,6 +346,11 @@ pub fn move_tape(label_text: String, vault_name: Option<String>) -> Result<(), E
properties: {
"label-text": {
schema: MEDIA_LABEL_SCHEMA,
optional: true,
},
uuid: {
schema: MEDIA_UUID_SCHEMA,
optional: true,
},
force: {
description: "Force removal (even if media is used in a media set).",
@ -333,22 +361,46 @@ pub fn move_tape(label_text: String, vault_name: Option<String>) -> Result<(), E
},
)]
/// Destroy media (completely remove from database)
pub fn destroy_media(label_text: String, force: Option<bool>) -> Result<(), Error> {
pub fn destroy_media(
label_text: Option<String>,
uuid: Option<Uuid>,
force: Option<bool>,
) -> Result<(), Error> {
let force = force.unwrap_or(false);
let mut inventory = Inventory::load(TAPE_STATUS_DIR)?;
let media_id = inventory
.find_media_by_label_text(&label_text)
.ok_or_else(|| format_err!("no such media '{}'", label_text))?;
let (media_id, text) = match (uuid, label_text) {
(Some(_), Some(_)) => {
param_bail!(
"format-text",
format_err!("A uuid is given, no label-text is expected.")
);
}
(None, None) => {
param_bail!(
"uuid",
format_err!("No label-text is given, a uuid is required.")
);
}
(Some(uuid), None) => (
inventory
.lookup_media(&uuid)
.ok_or_else(|| format_err!("no such media '{}'", uuid))?,
uuid.to_string(),
),
(None, Some(label_text)) => (
inventory
.find_media_by_label_text(&label_text)?
.ok_or_else(|| format_err!("no such media '{}'", label_text))?,
label_text,
),
};
if !force {
if let Some(ref set) = media_id.media_set_label {
if !set.unassigned() {
bail!(
"media '{}' contains data (please use 'force' flag to remove.",
label_text
);
bail!("media '{text}' contains data (please use 'force' flag to remove.");
}
}
}

12
src/api2/tape/restore.rs
View File

@ -1029,12 +1029,6 @@ fn restore_snapshots_to_tmpdir(
media_set_uuid
);
}
let encrypt_fingerprint = set.encryption_key_fingerprint.clone().map(|fp| {
task_log!(worker, "Encryption key fingerprint: {}", fp);
(fp, set.uuid.clone())
});
drive.set_encryption(encrypt_fingerprint)?;
}
}
@ -1279,12 +1273,6 @@ pub fn request_and_restore_media(
media_set_uuid
);
}
let encrypt_fingerprint = set
.encryption_key_fingerprint
.clone()
.map(|fp| (fp, set.uuid.clone()));
drive.set_encryption(encrypt_fingerprint)?;
}
}

59
src/bin/proxmox_backup_manager/acme.rs
View File

@ -103,8 +103,8 @@ async fn register_account(
contact: String,
directory: Option<String>,
) -> Result<(), Error> {
let directory = match directory {
Some(directory) => directory,
let (directory_url, custom_directory) = match directory {
Some(directory) => (directory, true),
None => {
println!("Directory endpoints:");
for (i, dir) in KNOWN_ACME_DIRECTORIES.iter().enumerate() {
@ -122,12 +122,12 @@ async fn register_account(
match input.trim().parse::<usize>() {
Ok(n) if n < KNOWN_ACME_DIRECTORIES.len() => {
break KNOWN_ACME_DIRECTORIES[n].url.to_owned();
break (KNOWN_ACME_DIRECTORIES[n].url.to_owned(), false);
}
Ok(n) if n == KNOWN_ACME_DIRECTORIES.len() => {
input.clear();
std::io::stdin().read_line(&mut input)?;
break input.trim().to_owned();
break (input.trim().to_owned(), true);
}
_ => eprintln!("Invalid selection."),
}
@ -140,9 +140,13 @@ async fn register_account(
}
};
println!("Attempting to fetch Terms of Service from {:?}", directory);
let mut client = AcmeClient::new(directory.clone());
let tos_agreed = if let Some(tos_url) = client.terms_of_service_url().await? {
println!(
"Attempting to fetch Terms of Service from {:?}",
directory_url
);
let mut client = AcmeClient::new(directory_url.clone());
let directory = client.directory().await?;
let tos_agreed = if let Some(tos_url) = directory.terms_of_service_url() {
println!("Terms of Service: {}", tos_url);
print!("Do you agree to the above terms? [y|N]: ");
std::io::stdout().flush()?;
@ -154,10 +158,45 @@ async fn register_account(
true
};
println!("Attempting to register account with {:?}...", directory);
let mut eab_enabled = directory.external_account_binding_required();
if !eab_enabled && custom_directory {
print!("Do you want to use external account binding? [y|N]: ");
std::io::stdout().flush()?;
let mut input = String::new();
std::io::stdin().read_line(&mut input)?;
eab_enabled = input.trim().eq_ignore_ascii_case("y");
} else if eab_enabled {
println!("The CA requires external account binding.");
}
let account =
api2::config::acme::do_register_account(&mut client, &name, tos_agreed, contact, None)
let eab_creds = if eab_enabled {
println!("You should have received a key id and a key from your CA.");
print!("Enter EAB key id: ");
std::io::stdout().flush()?;
let mut eab_kid = String::new();
std::io::stdin().read_line(&mut eab_kid)?;
print!("Enter EAB key: ");
std::io::stdout().flush()?;
let mut eab_hmac_key = String::new();
std::io::stdin().read_line(&mut eab_hmac_key)?;
Some((eab_kid.trim().to_owned(), eab_hmac_key.trim().to_owned()))
} else {
None
};
println!("Attempting to register account with {:?}...", directory_url);
let account = api2::config::acme::do_register_account(
&mut client,
&name,
tos_agreed,
contact,
None,
eab_creds,
)
.await?;
println!("Registration successful, account URL: {}", account.location);

1
src/bin/proxmox_tape/changer.rs
View File

@ -161,6 +161,7 @@ fn get_config(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error
let options = default_table_format_options()
.column(ColumnConfig::new("name"))
.column(ColumnConfig::new("path"))
.column(ColumnConfig::new("eject-before-unload"))
.column(ColumnConfig::new("export-slots"));
format_and_print_result_full(&mut data, &info.returns, &output_format, &options);

19
src/bin/sg-tape-cmd.rs
View File

@ -6,6 +6,8 @@ use std::fs::File;
use std::os::unix::io::{AsRawFd, FromRawFd};
use anyhow::{bail, Error};
use pbs_tape::sg_tape::SgTape;
use proxmox_backup::tape::encryption_keys::load_key;
use serde_json::Value;
use proxmox_router::{cli::*, RpcEnvironment};
@ -19,28 +21,26 @@ use pbs_api_types::{
use pbs_tape::linux_list_drives::{check_tape_is_lto_tape_device, open_lto_tape_device};
use proxmox_backup::tape::drive::{open_lto_tape_drive, LtoTapeHandle, TapeDriver};
fn get_tape_handle(param: &Value) -> Result<LtoTapeHandle, Error> {
fn get_tape_handle(param: &Value) -> Result<SgTape, Error> {
let handle = if let Some(name) = param["drive"].as_str() {
let (config, _digest) = pbs_config::drive::config()?;
let drive: LtoTapeDrive = config.lookup("lto", name)?;
log::info!("using device {}", drive.path);
open_lto_tape_drive(&drive)?
SgTape::open_lto_drive(&drive)?
} else if let Some(device) = param["device"].as_str() {
log::info!("using device {}", device);
LtoTapeHandle::new(open_lto_tape_device(device)?)?
SgTape::new(open_lto_tape_device(device)?)?
} else if let Some(true) = param["stdin"].as_bool() {
log::info!("using stdin");
let fd = std::io::stdin().as_raw_fd();
let file = unsafe { File::from_raw_fd(fd) };
check_tape_is_lto_tape_device(&file)?;
LtoTapeHandle::new(file)?
SgTape::new(file)?
} else if let Ok(name) = std::env::var("PROXMOX_TAPE_DRIVE") {
let (config, _digest) = pbs_config::drive::config()?;
let drive: LtoTapeDrive = config.lookup("lto", &name)?;
log::info!("using device {}", drive.path);
open_lto_tape_drive(&drive)?
SgTape::open_lto_drive(&drive)?
} else {
let (config, _digest) = pbs_config::drive::config()?;
@ -56,7 +56,7 @@ fn get_tape_handle(param: &Value) -> Result<LtoTapeHandle, Error> {
let name = drive_names[0];
let drive: LtoTapeDrive = config.lookup("lto", name)?;
log::info!("using device {}", drive.path);
open_lto_tape_drive(&drive)?
SgTape::open_lto_drive(&drive)?
} else {
bail!("no drive/device specified");
}
@ -103,7 +103,8 @@ fn set_encryption(
match (fingerprint, uuid) {
(Some(fingerprint), Some(uuid)) => {
handle.set_encryption(Some((fingerprint, uuid)))?;
let key = load_key(&fingerprint)?;
handle.set_encryption(Some((key, uuid)))?;
}
(Some(_), None) => {
bail!("missing media set uuid");

10
src/config/mod.rs
View File

@ -8,7 +8,7 @@ use nix::sys::stat::Mode;
use openssl::pkey::PKey;
use openssl::rsa::Rsa;
use openssl::x509::X509Builder;
use std::path::PathBuf;
use std::path::Path;
use proxmox_lang::try_block;
@ -84,8 +84,8 @@ pub fn create_configdir() -> Result<(), Error> {
/// Update self signed node certificate.
pub fn update_self_signed_cert(force: bool) -> Result<(), Error> {
let key_path = PathBuf::from(configdir!("/proxy.key"));
let cert_path = PathBuf::from(configdir!("/proxy.pem"));
let key_path = Path::new(configdir!("/proxy.key"));
let cert_path = Path::new(configdir!("/proxy.pem"));
if key_path.exists() && cert_path.exists() && !force {
return Ok(());
@ -183,8 +183,8 @@ pub fn update_self_signed_cert(force: bool) -> Result<(), Error> {
}
pub(crate) fn set_proxy_certificate(cert_pem: &[u8], key_pem: &[u8]) -> Result<(), Error> {
let key_path = PathBuf::from(configdir!("/proxy.key"));
let cert_path = PathBuf::from(configdir!("/proxy.pem"));
let key_path = Path::new(configdir!("/proxy.key"));
let cert_path = Path::new(configdir!("/proxy.pem"));
create_configdir()?;
pbs_config::replace_backup_config(key_path, key_pem)

26
src/rrd_cache.rs
View File

@ -9,8 +9,8 @@ use std::path::Path;
use anyhow::{format_err, Error};
use once_cell::sync::OnceCell;
use proxmox_rrd::rrd::{CF, DST, RRD};
use proxmox_rrd::RRDCache;
use proxmox_rrd::rrd::{AggregationFn, DataSourceType, Database};
use proxmox_rrd::Cache;
use proxmox_sys::fs::CreateOptions;
use pbs_api_types::{RRDMode, RRDTimeFrame};
@ -18,10 +18,10 @@ use pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR_M;
const RRD_CACHE_BASEDIR: &str = concat!(PROXMOX_BACKUP_STATE_DIR_M!(), "/rrdb");
static RRD_CACHE: OnceCell<RRDCache> = OnceCell::new();
static RRD_CACHE: OnceCell<Cache> = OnceCell::new();
/// Get the RRD cache instance
pub fn get_rrd_cache() -> Result<&'static RRDCache, Error> {
pub fn get_rrd_cache() -> Result<&'static Cache, Error> {
RRD_CACHE
.get()
.ok_or_else(|| format_err!("RRD cache not initialized!"))
@ -30,7 +30,7 @@ pub fn get_rrd_cache() -> Result<&'static RRDCache, Error> {
/// Initialize the RRD cache instance
///
/// Note: Only a single process must do this (proxmox-backup-proxy)
pub fn initialize_rrd_cache() -> Result<&'static RRDCache, Error> {
pub fn initialize_rrd_cache() -> Result<&'static Cache, Error> {
let backup_user = pbs_config::backup_user()?;
let file_options = CreateOptions::new()
@ -43,7 +43,7 @@ pub fn initialize_rrd_cache() -> Result<&'static RRDCache, Error> {
let apply_interval = 30.0 * 60.0; // 30 minutes
let cache = RRDCache::new(
let cache = Cache::new(
RRD_CACHE_BASEDIR,
Some(file_options),
Some(dir_options),
@ -58,8 +58,8 @@ pub fn initialize_rrd_cache() -> Result<&'static RRDCache, Error> {
Ok(RRD_CACHE.get().unwrap())
}
fn load_callback(path: &Path, _rel_path: &str, dst: DST) -> RRD {
match RRD::load(path, true) {
fn load_callback(path: &Path, _rel_path: &str, dst: DataSourceType) -> Database {
match Database::load(path, true) {
Ok(rrd) => rrd,
Err(err) => {
if err.kind() != std::io::ErrorKind::NotFound {
@ -69,7 +69,7 @@ fn load_callback(path: &Path, _rel_path: &str, dst: DST) -> RRD {
err
);
}
RRDCache::create_proxmox_backup_default_rrd(dst)
Cache::create_proxmox_backup_default_rrd(dst)
}
}
}
@ -93,8 +93,8 @@ pub fn extract_rrd_data(
};
let cf = match mode {
RRDMode::Max => CF::Maximum,
RRDMode::Average => CF::Average,
RRDMode::Max => AggregationFn::Maximum,
RRDMode::Average => AggregationFn::Average,
};
let rrd_cache = get_rrd_cache()?;
@ -114,7 +114,7 @@ pub fn rrd_sync_journal() {
pub fn rrd_update_gauge(name: &str, value: f64) {
if let Ok(rrd_cache) = get_rrd_cache() {
let now = proxmox_time::epoch_f64();
if let Err(err) = rrd_cache.update_value(name, now, value, DST::Gauge) {
if let Err(err) = rrd_cache.update_value(name, now, value, DataSourceType::Gauge) {
log::error!("rrd::update_value '{}' failed - {}", name, err);
}
}
@ -124,7 +124,7 @@ pub fn rrd_update_gauge(name: &str, value: f64) {
pub fn rrd_update_derive(name: &str, value: f64) {
if let Ok(rrd_cache) = get_rrd_cache() {
let now = proxmox_time::epoch_f64();
if let Err(err) = rrd_cache.update_value(name, now, value, DST::Derive) {
if let Err(err) = rrd_cache.update_value(name, now, value, DataSourceType::Derive) {
log::error!("rrd::update_value '{}' failed - {}", name, err);
}
}

21
src/server/pull.rs
View File

@ -486,7 +486,7 @@ pub(crate) struct PullParameters {
/// How many levels of sub-namespaces to pull (0 == no recursion, None == maximum recursion)
max_depth: Option<usize>,
/// Filters for reducing the pull scope
group_filter: Option<Vec<GroupFilter>>,
group_filter: Vec<GroupFilter>,
/// How many snapshots should be transferred at most (taking the newest N snapshots)
transfer_last: Option<usize>,
}
@ -539,6 +539,8 @@ impl PullParameters {
ns,
};
let group_filter = group_filter.unwrap_or_default();
Ok(Self {
source,
target,
@ -1358,7 +1360,6 @@ pub(crate) async fn pull_ns(
) -> Result<(StoreProgress, bool), Error> {
let mut list: Vec<BackupGroup> = params.source.list_groups(namespace, &params.owner).await?;
let total_count = list.len();
list.sort_unstable_by(|a, b| {
let type_order = a.ty.cmp(&b.ty);
if type_order == std::cmp::Ordering::Equal {
@ -1368,15 +1369,10 @@ pub(crate) async fn pull_ns(
}
});
let apply_filters = |group: &BackupGroup, filters: &[GroupFilter]| -> bool {
filters.iter().any(|filter| group.matches(filter))
};
let list = if let Some(ref group_filter) = &params.group_filter {
let unfiltered_count = list.len();
let list: Vec<BackupGroup> = list
.into_iter()
.filter(|group| apply_filters(group, group_filter))
.filter(|group| group.apply_filters(&params.group_filter))
.collect();
task_log!(
worker,
@ -1384,11 +1380,6 @@ pub(crate) async fn pull_ns(
list.len(),
unfiltered_count
);
list
} else {
task_log!(worker, "found {} groups to sync", total_count);
list
};
let mut errors = false;
@ -1457,11 +1448,9 @@ pub(crate) async fn pull_ns(
if check_backup_owner(&owner, &params.owner).is_err() {
continue;
}
if let Some(ref group_filter) = &params.group_filter {
if !apply_filters(local_group, group_filter) {
if !local_group.apply_filters(&params.group_filter) {
continue;
}
}
task_log!(worker, "delete vanished group '{local_group}'",);
match params
.target

5
src/server/report.rs
View File

@ -42,6 +42,7 @@ fn files() -> Vec<(&'static str, Vec<&'static str>)> {
"Jobs",
vec![
"/etc/proxmox-backup/sync.cfg",
"/etc/proxmox-backup/prune.cfg",
"/etc/proxmox-backup/verification.cfg",
],
),
@ -103,7 +104,7 @@ fn function_calls() -> Vec<FunctionMapping> {
for store in config.sections.keys() {
list.push(store.as_str());
}
list.join(", ")
format!("```\n{}\n```", list.join(", "))
}),
("System Load & Uptime", get_top_processes),
]
@ -212,7 +213,7 @@ pub fn generate_report() -> String {
.iter()
.map(|(desc, function)| {
let output = function();
format!("#### {desc}\n```\n{}\n```", output.trim_end())
format!("#### {desc}\n{}\n", output.trim_end())
})
.collect::<Vec<String>>()
.join("\n\n");

27
src/tape/changer/mod.rs
View File

@ -13,7 +13,9 @@ use proxmox_sys::fs::{file_read_optional_string, replace_file, CreateOptions};
use pbs_api_types::{LtoTapeDrive, ScsiTapeChanger};
use pbs_tape::{sg_pt_changer, ElementStatus, MtxStatus};
use pbs_tape::{linux_list_drives::open_lto_tape_device, sg_pt_changer, ElementStatus, MtxStatus};
use crate::tape::drive::{LtoTapeHandle, TapeDriver};
/// Interface to SCSI changer devices
pub trait ScsiMediaChange {
@ -384,8 +386,7 @@ fn load_changer_state_cache(changer: &str) -> Result<Option<MtxStatus>, Error> {
/// Implements MediaChange using 'mtx' linux cli tool
pub struct MtxMediaChanger {
drive_name: String, // used for error messages
drive_number: u64,
drive: LtoTapeDrive,
config: ScsiTapeChanger,
}
@ -398,8 +399,7 @@ impl MtxMediaChanger {
};
Ok(Self {
drive_name: drive_config.name.clone(),
drive_number: drive_config.changer_drivenum.unwrap_or(0),
drive: drive_config.clone(),
config: changer_config,
})
}
@ -407,11 +407,11 @@ impl MtxMediaChanger {
impl MediaChange for MtxMediaChanger {
fn drive_number(&self) -> u64 {
self.drive_number
self.drive.changer_drivenum.unwrap_or(0)
}
fn drive_name(&self) -> &str {
&self.drive_name
&self.drive.name
}
fn status(&mut self) -> Result<MtxStatus, Error> {
@ -423,12 +423,21 @@ impl MediaChange for MtxMediaChanger {
}
fn load_media_from_slot(&mut self, slot: u64) -> Result<MtxStatus, Error> {
self.config.load_slot(slot, self.drive_number)
self.config.load_slot(slot, self.drive_number())
}
fn unload_media(&mut self, target_slot: Option<u64>) -> Result<MtxStatus, Error> {
if self.config.eject_before_unload.unwrap_or(false) {
let file = open_lto_tape_device(&self.drive.path)?;
let mut handle = LtoTapeHandle::new(file)?;
if handle.medium_present() {
handle.eject_media()?;
}
}
if let Some(target_slot) = target_slot {
self.config.unload(target_slot, self.drive_number)
self.config.unload(target_slot, self.drive_number())
} else {
let status = self.status()?;
self.unload_to_free_slot(status)

26
src/tape/changer/online_status_map.rs
View File

@ -87,6 +87,16 @@ impl OnlineStatusMap {
}
}
fn insert_into_online_set(inventory: &Inventory, label_text: &str, online_set: &mut HashSet<Uuid>) {
match inventory.find_media_by_label_text(&label_text) {
Ok(Some(media_id)) => {
online_set.insert(media_id.label.uuid.clone());
}
Ok(None) => {}
Err(err) => log::warn!("error getting media by unique label: {err}"),
}
}
/// Extract the list of online media from MtxStatus
///
/// Returns a HashSet containing all found media Uuid. This only
@ -96,9 +106,7 @@ pub fn mtx_status_to_online_set(status: &MtxStatus, inventory: &Inventory) -> Ha
for drive_status in status.drives.iter() {
if let ElementStatus::VolumeTag(ref label_text) = drive_status.status {
if let Some(media_id) = inventory.find_media_by_label_text(label_text) {
online_set.insert(media_id.label.uuid.clone());
}
insert_into_online_set(inventory, label_text, &mut online_set);
}
}
@ -107,9 +115,7 @@ pub fn mtx_status_to_online_set(status: &MtxStatus, inventory: &Inventory) -> Ha
continue;
}
if let ElementStatus::VolumeTag(ref label_text) = slot_info.status {
if let Some(media_id) = inventory.find_media_by_label_text(label_text) {
online_set.insert(media_id.label.uuid.clone());
}
insert_into_online_set(inventory, label_text, &mut online_set);
}
}
@ -174,9 +180,7 @@ pub fn update_online_status<P: AsRef<Path>>(
let mut online_set = HashSet::new();
for label_text in media_list {
if let Some(media_id) = inventory.find_media_by_label_text(&label_text) {
online_set.insert(media_id.label.uuid.clone());
}
insert_into_online_set(&inventory, &label_text, &mut online_set);
}
map.update_online_status(&vtape.name, online_set)?;
}
@ -205,9 +209,7 @@ pub fn update_changer_online_status(
let mut online_map = OnlineStatusMap::new(drive_config)?;
let mut online_set = HashSet::new();
for label_text in label_text_list.iter() {
if let Some(media_id) = inventory.find_media_by_label_text(label_text) {
online_set.insert(media_id.label.uuid.clone());
}
insert_into_online_set(inventory, label_text, &mut online_set)
}
online_map.update_online_status(changer_name, online_set)?;
inventory.update_online_status(&online_map)?;

165
src/tape/drive/lto/mod.rs
View File

@ -16,6 +16,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use anyhow::{bail, format_err, Error};
use pbs_tape::sg_tape::drive_get_encryption;
use proxmox_uuid::Uuid;
use pbs_api_types::{
@ -23,7 +24,6 @@ use pbs_api_types::{
};
use pbs_key_config::KeyConfig;
use pbs_tape::{
linux_list_drives::open_lto_tape_device,
sg_tape::{SgTape, TapeAlertFlags},
BlockReadError, MediaContentHeader, TapeRead, TapeWrite,
};
@ -34,75 +34,47 @@ use crate::tape::{
file_formats::{MediaSetLabel, PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0},
};
/// Open a tape device
///
/// This does additional checks:
///
/// - check if it is a non-rewinding tape device
/// - check if drive is ready (tape loaded)
/// - check block size
/// - for autoloader only, try to reload ejected tapes
pub fn open_lto_tape_drive(config: &LtoTapeDrive) -> Result<LtoTapeHandle, Error> {
proxmox_lang::try_block!({
let file = open_lto_tape_device(&config.path)?;
let mut handle = LtoTapeHandle::new(file)?;
if handle.sg_tape.test_unit_ready().is_err() {
// for autoloader only, try to reload ejected tapes
if config.changer.is_some() {
let _ = handle.sg_tape.load(); // just try, ignore error
impl Drop for LtoTapeHandle {
fn drop(&mut self) {
// always unload the encryption key when the handle is dropped for security
// but only log an error if we set one in the first place
if let Err(err) = self.set_encryption(None) {
if self.encryption_key_loaded {
log::error!("could not unload encryption key from drive: {err}");
}
}
}
handle.sg_tape.wait_until_ready()?;
handle.set_default_options()?;
Ok(handle)
})
.map_err(|err: Error| {
format_err!(
"open drive '{}' ({}) failed - {}",
config.name,
config.path,
err
)
})
}
/// Lto Tape device handle
pub struct LtoTapeHandle {
sg_tape: SgTape,
encryption_key_loaded: bool,
}
impl LtoTapeHandle {
/// Creates a new instance
pub fn new(file: File) -> Result<Self, Error> {
let sg_tape = SgTape::new(file)?;
Ok(Self { sg_tape })
Ok(Self {
sg_tape,
encryption_key_loaded: false,
})
}
/// Set all options we need/want
pub fn set_default_options(&mut self) -> Result<(), Error> {
self.sg_tape.set_default_options()?;
Ok(())
}
/// Open a tape device
///
/// since this calls [SgTape::open_lto_drive], it does some internal checks.
/// See [SgTape] docs for details.
pub fn open_lto_drive(config: &LtoTapeDrive) -> Result<Self, Error> {
let sg_tape = SgTape::open_lto_drive(config)?;
/// Set driver options
pub fn set_drive_options(
&mut self,
compression: Option<bool>,
block_length: Option<u32>,
buffer_mode: Option<bool>,
) -> Result<(), Error> {
self.sg_tape
.set_drive_options(compression, block_length, buffer_mode)
}
let handle = Self {
sg_tape,
encryption_key_loaded: false,
};
/// Write a single EOF mark without flushing buffers
pub fn write_filemarks(&mut self, count: usize) -> Result<(), std::io::Error> {
self.sg_tape.write_filemarks(count, false)
Ok(handle)
}
/// Get Tape and Media status
@ -118,27 +90,11 @@ impl LtoTapeHandle {
self.sg_tape.space_filemarks(-count.try_into()?)
}
pub fn forward_space_count_records(&mut self, count: usize) -> Result<(), Error> {
self.sg_tape.space_blocks(count.try_into()?)
}
pub fn backward_space_count_records(&mut self, count: usize) -> Result<(), Error> {
self.sg_tape.space_blocks(-count.try_into()?)
}
/// Position the tape after filemark count. Count 0 means BOT.
pub fn locate_file(&mut self, position: u64) -> Result<(), Error> {
self.sg_tape.locate_file(position)
}
pub fn erase_media(&mut self, fast: bool) -> Result<(), Error> {
self.sg_tape.erase_media(fast)
}
pub fn load(&mut self) -> Result<(), Error> {
self.sg_tape.load()
}
/// Read Cartridge Memory (MAM Attributes)
pub fn cartridge_memory(&mut self) -> Result<Vec<MamAttribute>, Error> {
self.sg_tape.cartridge_memory()
@ -149,18 +105,9 @@ impl LtoTapeHandle {
self.sg_tape.volume_statistics()
}
/// Lock the drive door
pub fn lock(&mut self) -> Result<(), Error> {
self.sg_tape
.set_medium_removal(false)
.map_err(|err| format_err!("lock door failed - {}", err))
}
/// Unlock the drive door
pub fn unlock(&mut self) -> Result<(), Error> {
self.sg_tape
.set_medium_removal(true)
.map_err(|err| format_err!("unlock door failed - {}", err))
/// Returns if a medium is present
pub fn medium_present(&mut self) -> bool {
self.sg_tape.test_unit_ready().is_ok()
}
}
@ -271,6 +218,13 @@ impl TapeDriver for LtoTapeHandle {
self.sync()?; // sync data to tape
let encrypt_fingerprint = media_set_label
.encryption_key_fingerprint
.clone()
.map(|fp| (fp, media_set_label.uuid.clone()));
self.set_encryption(encrypt_fingerprint)?;
Ok(())
}
@ -292,46 +246,27 @@ impl TapeDriver for LtoTapeHandle {
&mut self,
key_fingerprint: Option<(Fingerprint, Uuid)>,
) -> Result<(), Error> {
if nix::unistd::Uid::effective().is_root() {
if let Some((ref key_fingerprint, ref uuid)) = key_fingerprint {
let (key_map, _digest) = crate::tape::encryption_keys::load_keys()?;
match key_map.get(key_fingerprint) {
Some(item) => {
// derive specialized key for each media-set
let mut tape_key = [0u8; 32];
let uuid_bytes: [u8; 16] = *uuid.as_bytes();
openssl::pkcs5::pbkdf2_hmac(
&item.key,
&uuid_bytes,
10,
openssl::hash::MessageDigest::sha256(),
&mut tape_key,
)?;
return self.sg_tape.set_encryption(Some(tape_key));
}
None => bail!("unknown tape encryption key '{}'", key_fingerprint),
}
} else {
return self.sg_tape.set_encryption(None);
}
}
let output = if let Some((fingerprint, uuid)) = key_fingerprint {
if let Some((fingerprint, uuid)) = key_fingerprint {
let fingerprint = fingerprint.signature();
run_sg_tape_cmd(
let output = run_sg_tape_cmd(
"encryption",
&["--fingerprint", &fingerprint, "--uuid", &uuid.to_string()],
self.sg_tape.file_mut().as_raw_fd(),
)?
} else {
run_sg_tape_cmd("encryption", &[], self.sg_tape.file_mut().as_raw_fd())?
};
)?;
self.encryption_key_loaded = true;
let result: Result<(), String> = serde_json::from_str(&output)?;
result.map_err(|err| format_err!("{}", err))
} else {
self.sg_tape.set_encryption(None)
}
}
fn assert_encryption_mode(&mut self, encryption_wanted: bool) -> Result<(), Error> {
let encryption_set = drive_get_encryption(self.sg_tape.file_mut())?;
if encryption_wanted != encryption_set {
bail!("Set encryption mode not what was desired (set: {encryption_set}, wanted: {encryption_wanted})");
}
Ok(())
}
}

47
src/tape/drive/mod.rs
View File

@ -105,11 +105,13 @@ pub trait TapeDriver {
key_config: Option<&KeyConfig>,
) -> Result<(), Error>;
/// Read the media label
/// Read the media label without setting the encryption key
///
/// This tries to read both media labels (label and
/// media_set_label). Also returns the optional encryption key configuration.
fn read_label(&mut self) -> Result<(Option<MediaId>, Option<KeyConfig>), Error> {
/// This is used internally by 'read_label' and when restoring the encryption
/// key from the drive. Should not be used or overwritten otherwise!
fn read_label_without_loading_key(
&mut self,
) -> Result<(Option<MediaId>, Option<KeyConfig>), Error> {
self.rewind()?;
let label = {
@ -187,6 +189,22 @@ pub trait TapeDriver {
Ok((Some(media_id), key_config))
}
/// Read the media label
///
/// This tries to read both media labels (label and
/// media_set_label). Also returns the optional encryption key configuration.
///
/// Automatically sets the encryption key on the drive
fn read_label(&mut self) -> Result<(Option<MediaId>, Option<KeyConfig>), Error> {
let (media_id, key_config) = self.read_label_without_loading_key()?;
let encrypt_fingerprint = media_id.as_ref().and_then(|id| id.get_encryption_fp());
self.set_encryption(encrypt_fingerprint)?;
Ok((media_id, key_config))
}
/// Eject media
fn eject_media(&mut self) -> Result<(), Error>;
@ -203,6 +221,9 @@ pub trait TapeDriver {
/// We use the media_set_uuid to XOR the secret key with the
/// uuid (first 16 bytes), so that each media set uses an unique
/// key for encryption.
///
/// Should be called as part of write_media_set_label or read_label,
/// so this should not be called manually.
fn set_encryption(
&mut self,
key_fingerprint: Option<(Fingerprint, Uuid)>,
@ -212,6 +233,14 @@ pub trait TapeDriver {
}
Ok(())
}
/// Asserts that the encryption mode is set to the given value
fn assert_encryption_mode(&mut self, encryption_wanted: bool) -> Result<(), Error> {
if encryption_wanted {
bail!("drive does not support encryption");
}
Ok(())
}
}
/// A boxed implementor of [`MediaChange`].
@ -280,7 +309,7 @@ pub fn open_drive(config: &SectionConfigData, drive: &str) -> Result<Box<dyn Tap
}
"lto" => {
let tape = LtoTapeDrive::deserialize(config)?;
let handle = open_lto_tape_drive(&tape)?;
let handle = LtoTapeHandle::open_lto_drive(&tape)?;
Ok(Box::new(handle))
}
ty => bail!("unknown drive type '{}' - internal error", ty),
@ -449,7 +478,7 @@ pub fn request_and_load_media(
}
}
let mut handle = match open_lto_tape_drive(&drive_config) {
let mut handle = match LtoTapeHandle::open_lto_drive(&drive_config) {
Ok(handle) => handle,
Err(err) => {
update_and_log_request_error(
@ -572,7 +601,9 @@ fn tape_device_path(config: &SectionConfigData, drive: &str) -> Result<String, E
}
}
pub struct DeviceLockGuard(std::fs::File);
pub struct DeviceLockGuard {
_file: std::fs::File,
}
// Uses systemd escape_unit to compute a file name from `device_path`, the try
// to lock `/var/lock/<name>`.
@ -610,7 +641,7 @@ fn lock_device_path(device_path: &str) -> Result<DeviceLockGuard, TapeLockError>
}
}
Ok(DeviceLockGuard(file))
Ok(DeviceLockGuard { _file: file })
}
// Same logic as lock_device_path, but uses a timeout of 0, making it

10
src/tape/encryption_keys.rs
View File

@ -12,7 +12,7 @@
use std::collections::HashMap;
use anyhow::{bail, Error};
use anyhow::{bail, format_err, Error};
use serde::{Deserialize, Serialize};
use proxmox_sys::fs::file_read_optional_string;
@ -92,6 +92,14 @@ pub fn load_keys() -> Result<(HashMap<Fingerprint, EncryptionKeyInfo>, [u8; 32])
Ok((map, digest))
}
pub fn load_key(fingerprint: &Fingerprint) -> Result<[u8; 32], Error> {
let (key_map, _digest) = crate::tape::encryption_keys::load_keys()?;
key_map
.get(fingerprint)
.map(|data| data.key)
.ok_or_else(|| format_err!("unknown tape encryption key '{fingerprint}'"))
}
/// Load tape encryption key configurations (password protected keys)
pub fn load_key_configs() -> Result<(HashMap<Fingerprint, KeyConfig>, [u8; 32]), Error> {
let content = file_read_optional_string(TAPE_KEY_CONFIG_FILENAME)?;

20
src/tape/inventory.rs
View File

@ -33,7 +33,7 @@ use serde_json::json;
use proxmox_sys::fs::{file_get_json, replace_file, CreateOptions};
use proxmox_uuid::Uuid;
use pbs_api_types::{MediaLocation, MediaSetPolicy, MediaStatus, RetentionPolicy};
use pbs_api_types::{Fingerprint, MediaLocation, MediaSetPolicy, MediaStatus, RetentionPolicy};
use pbs_config::BackupLockGuard;
#[cfg(not(test))]
@ -71,6 +71,10 @@ impl MediaId {
}
self.label.pool.to_owned()
}
pub(crate) fn get_encryption_fp(&self) -> Option<(Fingerprint, Uuid)> {
let label = self.clone().media_set_label?;
label.encryption_key_fingerprint.map(|fp| (fp, label.uuid))
}
}
#[derive(Serialize, Deserialize)]
@ -244,14 +248,24 @@ impl Inventory {
}
/// find media by label_text
pub fn find_media_by_label_text(&self, label_text: &str) -> Option<&MediaId> {
self.map.values().find_map(|entry| {
pub fn find_media_by_label_text(&self, label_text: &str) -> Result<Option<&MediaId>, Error> {
let ids: Vec<_> = self
.map
.values()
.filter_map(|entry| {
if entry.id.label.label_text == label_text {
Some(&entry.id)
} else {
None
}
})
.collect();
match ids.len() {
0 => Ok(None),
1 => Ok(Some(ids[0])),
count => bail!("There are '{count}' tapes with the label '{label_text}'"),
}
}
/// Lookup media pool

7
src/tape/pool_writer/mod.rs
View File

@ -272,12 +272,7 @@ impl PoolWriter {
let media_set = media.media_set_label().unwrap();
let encrypt_fingerprint = media_set
.encryption_key_fingerprint
.clone()
.map(|fp| (fp, media_set.uuid.clone()));
drive.set_encryption(encrypt_fingerprint)?;
drive.assert_encryption_mode(media_set.encryption_key_fingerprint.is_some())?;
self.status = Some(PoolWriterState {
drive,

61
src/tools/disks/mod.rs
View File

@ -214,6 +214,12 @@ struct DiskInfo {
serial: OnceCell<Option<OsString>>,
// for perl: #[serde(skip_serializing)]
partition_table_type: OnceCell<Option<OsString>>,
// for perl: #[serde(skip_serializing)]
partition_entry_scheme: OnceCell<Option<OsString>>,
// for perl: #[serde(skip_serializing)]
partition_entry_uuid: OnceCell<Option<OsString>>,
// for perl: #[serde(skip_serializing)]
partition_entry_type: OnceCell<Option<OsString>>,
gpt: OnceCell<bool>,
// ???
bus: OnceCell<Option<OsString>>,
@ -412,6 +418,50 @@ impl Disk {
})
}
/// Get the partitioning scheme of which this device is a partition.
pub fn partition_entry_scheme(&self) -> Option<&OsStr> {
self.info
.partition_entry_scheme
.get_or_init(|| {
self.device
.property_value("ID_PART_ENTRY_SCHEME")
.map(|v| v.to_owned())
})
.as_ref()
.map(OsString::as_os_str)
}
/// Check if this is a partition.
pub fn is_partition(&self) -> bool {
self.partition_entry_scheme().is_some()
}
/// Get the type of partition entry (ie. type UUID from the entry in the GPT partition table).
pub fn partition_entry_type(&self) -> Option<&OsStr> {
self.info
.partition_entry_type
.get_or_init(|| {
self.device
.property_value("ID_PART_ENTRY_TYPE")
.map(|v| v.to_owned())
})
.as_ref()
.map(OsString::as_os_str)
}
/// Get the partition entry UUID (ie. the UUID from the entry in the GPT partition table).
pub fn partition_entry_uuid(&self) -> Option<&OsStr> {
self.info
.partition_entry_uuid
.get_or_init(|| {
self.device
.property_value("ID_PART_ENTRY_UUID")
.map(|v| v.to_owned())
})
.as_ref()
.map(OsString::as_os_str)
}
/// Get the bus type used for this disk.
pub fn bus(&self) -> Option<&OsStr> {
self.info
@ -1071,17 +1121,8 @@ pub fn wipe_blockdev(disk: &Disk, worker: Arc<WorkerTask>) -> Result<(), Error>
Some(path) => path,
None => bail!("disk {:?} has no node in /dev", disk.syspath()),
};
let disk_path_str = match disk_path.to_str() {
Some(path) => path,
None => bail!("disk {:?} could not transform into a str", disk.syspath()),
};
let mut is_partition = false;
for disk_info in get_lsblk_info()?.iter() {
if disk_info.path == disk_path_str && disk_info.partition_type.is_some() {
is_partition = true;
}
}
let is_partition = disk.is_partition();
let mut to_wipe: Vec<PathBuf> = Vec::new();

1
www/Makefile
View File

@ -21,6 +21,7 @@ TAPE_UI_FILES= \
tape/window/Erase.js \
tape/window/EncryptionEdit.js \
tape/window/LabelMedia.js \
tape/window/MediaRemoveWindow.js \
tape/window/PoolEdit.js \
tape/window/TapeBackup.js \
tape/window/TapeBackupJob.js \

12
www/datastore/DataStoreListSummary.js
View File

@ -52,12 +52,20 @@ Ext.define('PBS.datastore.DataStoreListSummary', {
vm.set('maintenance', '');
}
let usagetext;
let usage;
if (Object.hasOwn(statusData, 'avail') && Object.hasOwn(statusData, 'used')) {
let total = statusData.avail + statusData.used;
let usage = statusData.used / total;
let usagetext = Ext.String.format(gettext('{0} of {1}'),
usage = statusData.used / total;
usagetext = Ext.String.format(gettext('{0} of {1}'),
Proxmox.Utils.format_size(statusData.used, true),
Proxmox.Utils.format_size(total, true),
);
} else {
usagetext = Ext.String.format(gettext('{0} of {1}'), 0, 0);
usage = 0;
}
let usagePanel = me.lookup('usage');
usagePanel.updateValue(usage, usagetext);

239
www/form/GroupFilter.js
View File

@ -35,13 +35,36 @@ Ext.define('PBS.form.GroupFilter', {
// break cyclic reference
me.removeReferences(record);
me.lookup('grid').getStore().remove(record);
me.lookup('grid-include').getStore().remove(record);
me.lookup('grid-exclude').getStore().remove(record);
me.updateRealField();
},
addFilter: function() {
addIncludeFilter: function() {
let me = this;
me.lookup('grid').getStore().add({});
me.lookup('grid-include').getStore().add({ behavior: 'include' });
me.updateRealField();
},
addExcludeFilter: function() {
let me = this;
me.lookup('grid-exclude').getStore().add({ behavior: 'exclude' });
me.updateRealField();
},
onBehaviorChange: function(field, value) {
let me = this;
let record = field.getWidgetRecord();
if (record === undefined) {
return;
}
record.set('behavior', value);
record.commit();
if (record.widgets) {
me.setInputValue(record.widgets, record);
}
me.updateRealField();
},
@ -77,8 +100,12 @@ Ext.define('PBS.form.GroupFilter', {
},
parseGroupFilter: function(filter) {
let [, type, input] = filter.match(/^(type|group|regex):(.*)$/);
let [, behavior, type, input] = filter.match(/^(?:(exclude|include):)?(type|group|regex):(.*)$/);
if (behavior === undefined) {
behavior = "include";
}
return {
behavior,
type,
input,
};
@ -86,13 +113,16 @@ Ext.define('PBS.form.GroupFilter', {
onValueChange: function(field, values) {
let me = this;
let grid = me.lookup('grid');
let grid_include = me.lookup('grid-include');
let grid_exclude = me.lookup('grid-exclude');
if (!values || values.length === 0) {
grid.getStore().removeAll();
grid_include.getStore().removeAll();
grid_exclude.getStore().removeAll();
return;
}
let records = values.map((filter) => me.parseGroupFilter(filter));
grid.getStore().setData(records);
grid_include.getStore().setData(records);
grid_exclude.getStore().setData(records);
},
setInputValue: function(widgets, rec) {
@ -162,11 +192,20 @@ Ext.define('PBS.form.GroupFilter', {
let me = this;
let filter = [];
me.lookup('grid').getStore().each((rec) => {
me.lookup('grid-include').getStore().each((rec) => {
if (rec.data.type && rec.data.input) {
filter.push(`${rec.data.type}:${rec.data.input}`);
}
});
me.lookup('grid-exclude').getStore().each((rec) => {
if (rec.data.type && rec.data.input && rec.data.behavior) {
let behavior_string = '';
if (rec.data.behavior === 'exclude') {
behavior_string = 'exclude:';
}
filter.push(`${behavior_string}${rec.data.type}:${rec.data.input}`);
}
});
let field = me.lookup('realfield');
field.suspendEvent('change');
@ -175,6 +214,9 @@ Ext.define('PBS.form.GroupFilter', {
},
control: {
'grid pbsGroupBehaviorSelector': {
change: 'onBehaviorChange',
},
'grid pbsGroupFilterTypeSelector': {
change: 'onTypeChange',
},
@ -264,18 +306,133 @@ Ext.define('PBS.form.GroupFilter', {
items: [
{
xtype: 'grid',
reference: 'grid',
xtype: 'pbsGroupFilterGrid',
title: 'Include filters',
margin: '0 0 5 0',
scrollable: true,
height: 300,
reference: 'grid-include',
store: {
fields: ['type', 'input'],
filters: [
function(item) {
return item.data.behavior === "include";
},
],
},
emptyText: gettext('Include all groups'),
viewConfig: {
deferEmptyText: false,
},
},
{
xtype: 'container',
layout: {
type: 'hbox',
},
items: [
{
xtype: 'button',
text: gettext('Add include'),
iconCls: 'fa fa-plus-circle',
handler: 'addIncludeFilter',
},
{
xtype: 'box',
flex: 1,
},
{
xtype: 'box',
style: 'margin: 3px 0px;',
html: `<span class="pmx-hint">${gettext('Note')}</span>: `
+ gettext('Filters are additive'),
},
],
},
{
xtype: 'pbsGroupFilterGrid',
title: 'Exclude filters',
margin: '10 0 5 0',
reference: 'grid-exclude',
store: {
filters: [
function(item) {
return item.data.behavior === "exclude";
},
],
},
},
{
xtype: 'hiddenfield',
reference: 'realfield',
setValue: function(value) {
let me = this;
me.value = value;
me.checkChange();
},
getValue: function() {
return this.value;
},
getSubmitValue: function() {
return this.value;
},
cbind: {
name: '{name}',
},
},
{
xtype: 'container',
layout: {
type: 'hbox',
},
items: [
{
xtype: 'button',
text: gettext('Add exclude'),
iconCls: 'fa fa-plus-circle',
handler: 'addExcludeFilter',
},
{
xtype: 'box',
flex: 1,
},
{
xtype: 'box',
style: 'margin: 3px 0px;',
html: `<span class="pmx-hint">${gettext('Note')}</span>: `
+ gettext('Exclude filters will be applied after include filters'),
},
],
},
],
initComponent: function() {
let me = this;
me.callParent();
me.dsStore = Ext.create('Ext.data.Store', {
sorters: 'group',
model: 'pbs-groups',
});
},
});
Ext.define('PBS.form.pbsGroupBehaviorSelector', {
extend: 'Proxmox.form.KVComboBox',
alias: 'widget.pbsGroupBehaviorSelector',
allowBlank: false,
comboItems: [
['include', gettext('Include')],
['exclude', gettext('Exclude')],
],
});
Ext.define('PBS.form.GroupFilterGrid', {
extend: 'Ext.grid.Panel',
alias: 'widget.pbsGroupFilterGrid',
scrollable: true,
height: 200,
store: {
fields: ['type', 'input'],
},
columns: [
{
text: gettext('Filter Type'),
@ -297,7 +454,8 @@ Ext.define('PBS.form.GroupFilter', {
bodyPadding: 0,
xtype: 'fieldcontainer',
layout: 'fit',
defaults: {
defaults:
{
margin: 0,
},
items: [
@ -329,59 +487,6 @@ Ext.define('PBS.form.GroupFilter', {
},
},
],
},
{
xtype: 'hiddenfield',
reference: 'realfield',
setValue: function(value) {
let me = this;
me.value = value;
me.checkChange();
},
getValue: function() {
return this.value;
},
getSubmitValue: function() {
return this.value;
},
cbind: {
name: '{name}',
},
},
{
xtype: 'container',
layout: {
type: 'hbox',
},
items: [
{
xtype: 'button',
text: gettext('Add'),
iconCls: 'fa fa-plus-circle',
handler: 'addFilter',
},
{
xtype: 'box',
flex: 1,
},
{
xtype: 'box',
style: 'margin: 3px 0px;',
html: `<span class="pmx-hint">${gettext('Note')}</span>: `
+ gettext('Filters are additive (OR-like)'),
},
],
},
],
initComponent: function() {
let me = this;
me.callParent();
me.dsStore = Ext.create('Ext.data.Store', {
sorters: 'group',
model: 'pbs-groups',
});
},
});
Ext.define('PBS.form.GroupFilterTypeSelector', {

2
www/panel/NodeInfo.js
View File

@ -151,7 +151,7 @@ Ext.define('PBS.NodeInfoPanel', {
return data.kversion;
}
let kernel = data['current-kernel'];
let buildDate = kernel.version.match(/\((.+)\)\s*$/)[1] ?? 'unknown';
let buildDate = kernel.version.match(/\((.+)\)\s*$/)?.[1] ?? 'unknown';
return `${kernel.sysname} ${kernel.release} (${buildDate})`;
},
value: '',

35
www/tape/TapeInventory.js
View File

@ -17,7 +17,7 @@ Ext.define('pbs-model-tapes', {
'status',
'uuid',
],
idProperty: 'label-text',
idProperty: 'uuid',
proxy: {
type: 'proxmox',
url: '/api2/json/tape/media/list',
@ -60,6 +60,27 @@ Ext.define('PBS.TapeManagement.TapeInventory', {
}).show();
},
remove: function() {
let me = this;
let view = me.getView();
let selection = view.getSelection();
if (!selection || selection.length < 1) {
return;
}
let uuid = selection[0].data.uuid;
let label = selection[0].data['label-text'];
Ext.create('PBS.TapeManagement.MediaRemoveWindow', {
uuid,
label,
autoShow: true,
listeners: {
destroy: function() {
me.reload();
},
},
});
},
moveToVault: function() {
let me = this;
let view = me.getView();
@ -206,6 +227,12 @@ Ext.define('PBS.TapeManagement.TapeInventory', {
disabled: true,
handler: 'format',
},
{
xtype: 'proxmoxButton',
text: gettext('Remove'),
disabled: true,
handler: 'remove',
},
],
features: [
@ -293,5 +320,11 @@ Ext.define('PBS.TapeManagement.TapeInventory', {
},
flex: 1,
},
{
text: gettext('UUID'),
dataIndex: 'uuid',
flex: 1,
hidden: true,
},
],
});

66
www/tape/window/MediaRemoveWindow.js Normal file
View File

@ -0,0 +1,66 @@
Ext.define('PBS.TapeManagement.MediaRemoveWindow', {
extend: 'Proxmox.window.Edit',
mixins: ['Proxmox.Mixin.CBind'],
uuid: undefined,
label: undefined,
cbindData: function(config) {
let me = this;
return {
uuid: me.uuid,
warning: Ext.String.format(gettext("Are you sure you want to remove tape '{0}' ?"), me.label),
};
},
title: gettext('Remove Media'),
url: `/api2/extjs/tape/media/destroy`,
layout: 'hbox',
width: 400,
method: 'GET',
isCreate: true,
submitText: gettext('Ok'),
items: [
{
xtype: 'container',
padding: 0,
layout: {
type: 'hbox',
align: 'stretch',
},
items: [
{
xtype: 'component',
cls: [Ext.baseCSSPrefix + 'message-box-icon',
Ext.baseCSSPrefix + 'message-box-warning',
Ext.baseCSSPrefix + 'dlg-icon'],
},
{
xtype: 'container',
flex: 1,
items: [
{
xtype: 'displayfield',
cbind: {
value: '{warning}',
},
},
{
xtype: 'hidden',
name: 'uuid',
cbind: {
value: '{uuid}',
},
},
{
xtype: 'proxmoxcheckbox',
fieldLabel: gettext('Force'),
name: 'force',
},
],
},
],
},
],
});

1
www/window/UserEdit.js
View File

@ -66,7 +66,6 @@ Ext.define('PBS.window.UserEdit', {
fieldLabel: gettext('User name'),
renderer: Ext.htmlEncode,
allowBlank: false,
minLength: 4,
cbind: {
editable: '{isCreate}',
},