From 8d1a9d2ec6ab2c4dfacdb1c8882fa64f5fbf3ac4 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 07:06:17 +0200 Subject: [PATCH] move RRD code into proxmox-rrd crate --- proxmox-rrd/Cargo.toml | 13 ++++ proxmox-rrd/src/cache.rs | 111 ++++++++++++++++++++++++++++ proxmox-rrd/src/lib.rs | 37 ++++++++++ {src/rrd => proxmox-rrd/src}/rrd.rs | 28 +++---- src/rrd/cache.rs | 81 -------------------- src/rrd/mod.rs | 5 -- 6 files changed, 171 insertions(+), 104 deletions(-) create mode 100644 proxmox-rrd/Cargo.toml create mode 100644 proxmox-rrd/src/cache.rs create mode 100644 proxmox-rrd/src/lib.rs rename {src/rrd => proxmox-rrd/src}/rrd.rs (93%) delete mode 100644 src/rrd/cache.rs delete mode 100644 src/rrd/mod.rs diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml new file mode 100644 index 00000000..c2b2d213 --- /dev/null +++ b/proxmox-rrd/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "proxmox-rrd" +version = "0.1.0" +authors = ["Dietmar Maurer "] +edition = "2018" +description = "Simple RRD database implementation." + +[dependencies] +anyhow = "1.0" +bitflags = "1.2.1" +serde = { version = "1.0", features = [] } + +proxmox = { version = "0.13.5", features = ["api-macro"] } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs new file mode 100644 index 00000000..c87e49fd --- /dev/null +++ b/proxmox-rrd/src/cache.rs @@ -0,0 +1,111 @@ +use std::path::{Path, PathBuf}; +use std::collections::HashMap; +use std::sync::{RwLock}; + +use anyhow::{format_err, Error}; + +use proxmox::tools::fs::{create_path, CreateOptions}; + +use crate::{RRDMode, RRDTimeFrameResolution}; + +use super::*; + +/// RRD cache - keep RRD data in RAM, but write updates to disk +/// +/// This cache is designed to run as single instance (no concurrent +/// access from other processes). +pub struct RRDCache { + basedir: PathBuf, + file_options: CreateOptions, + dir_options: CreateOptions, + cache: RwLock>, +} + +impl RRDCache { + + /// Creates a new instance + pub fn new>( + basedir: P, + file_options: Option, + dir_options: Option, + ) -> Self { + let basedir = basedir.as_ref().to_owned(); + Self { + basedir, + file_options: file_options.unwrap_or_else(|| CreateOptions::new()), + dir_options: dir_options.unwrap_or_else(|| CreateOptions::new()), + cache: RwLock::new(HashMap::new()), + } + } +} + +impl RRDCache { + + /// Create rrdd stat dir with correct permission + pub fn create_rrdb_dir(&self) -> Result<(), Error> { + + create_path(&self.basedir, Some(self.dir_options.clone()), Some(self.file_options.clone())) + .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; + + Ok(()) + } + + /// Update data in RAM and write file back to disk (if `save` is set) + pub fn update_value( + &self, + rel_path: &str, + value: f64, + dst: DST, + save: bool, + ) -> Result<(), Error> { + + let mut path = self.basedir.clone(); + path.push(rel_path); + + std::fs::create_dir_all(path.parent().unwrap())?; // fixme?? + + let mut map = self.cache.write().unwrap(); + let now = proxmox::tools::time::epoch_f64(); + + if let Some(rrd) = map.get_mut(rel_path) { + rrd.update(now, value); + if save { rrd.save(&path, self.file_options.clone())?; } + } else { + let mut rrd = match RRD::load(&path) { + Ok(rrd) => rrd, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err); + } + RRD::new(dst) + }, + }; + rrd.update(now, value); + if save { + rrd.save(&path, self.file_options.clone())?; + } + map.insert(rel_path.into(), rrd); + } + + Ok(()) + } + + /// Extract data from cached RRD + pub fn extract_cached_data( + &self, + base: &str, + name: &str, + now: f64, + timeframe: RRDTimeFrameResolution, + mode: RRDMode, + ) -> Option<(u64, u64, Vec>)> { + + let map = self.cache.read().unwrap(); + + match map.get(&format!("{}/{}", base, name)) { + Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), + None => None, + } + } + +} diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs new file mode 100644 index 00000000..d6ba54c9 --- /dev/null +++ b/proxmox-rrd/src/lib.rs @@ -0,0 +1,37 @@ +mod rrd; +pub use rrd::*; + +mod cache; +pub use cache::*; + +use serde::{Deserialize, Serialize}; +use proxmox::api::api; + +#[api()] +#[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +/// RRD consolidation mode +pub enum RRDMode { + /// Maximum + Max, + /// Average + Average, +} + +#[api()] +#[repr(u64)] +#[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +/// RRD time frame resolution +pub enum RRDTimeFrameResolution { + /// 1 min => last 70 minutes + Hour = 60, + /// 30 min => last 35 hours + Day = 60*30, + /// 3 hours => about 8 days + Week = 60*180, + /// 12 hours => last 35 days + Month = 60*720, + /// 1 week => last 490 days + Year = 60*10080, +} diff --git a/src/rrd/rrd.rs b/proxmox-rrd/src/rrd.rs similarity index 93% rename from src/rrd/rrd.rs rename to proxmox-rrd/src/rrd.rs index b1780307..19a6deba 100644 --- a/src/rrd/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -3,17 +3,21 @@ use std::path::Path; use anyhow::Error; -use pbs_api_types::{RRDMode, RRDTimeFrameResolution}; +use proxmox::tools::{fs::replace_file, fs::CreateOptions}; +use crate::{RRDMode, RRDTimeFrameResolution}; + +/// The number of data entries per RRA pub const RRD_DATA_ENTRIES: usize = 70; +/// Proxmox RRD file magic number // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; use bitflags::bitflags; bitflags!{ - pub struct RRAFlags: u64 { + struct RRAFlags: u64 { // Data Source Types const DST_GAUGE = 1; const DST_DERIVE = 2; @@ -27,6 +31,7 @@ bitflags!{ } } +/// RRD data source tyoe pub enum DST { Gauge, Derive, @@ -227,6 +232,7 @@ impl RRD { timeframe: RRDTimeFrameResolution, mode: RRDMode, ) -> (u64, u64, Vec>) { + let epoch = time as u64; let reso = timeframe as u64; @@ -296,25 +302,11 @@ impl RRD { Self::from_raw(&raw) } - pub fn save(&self, filename: &Path) -> Result<(), Error> { - use proxmox::tools::{fs::replace_file, fs::CreateOptions}; - + pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> { let rrd_slice = unsafe { std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) }; - - let backup_user = pbs_config::backup_user()?; - let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); - // set the correct owner/group/permissions while saving file - // owner(rw) = backup, group(r)= backup - let options = CreateOptions::new() - .perm(mode) - .owner(backup_user.uid) - .group(backup_user.gid); - - replace_file(filename, rrd_slice, options)?; - - Ok(()) + replace_file(filename, rrd_slice, options) } diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs deleted file mode 100644 index d6b79ac0..00000000 --- a/src/rrd/cache.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::path::PathBuf; -use std::collections::HashMap; -use std::sync::{RwLock}; - -use anyhow::{format_err, Error}; -use lazy_static::lazy_static; - -use proxmox::tools::fs::{create_path, CreateOptions}; - -use pbs_api_types::{RRDMode, RRDTimeFrameResolution}; - -use super::*; - -const PBS_RRD_BASEDIR: &str = "/var/lib/proxmox-backup/rrdb"; - -lazy_static!{ - static ref RRD_CACHE: RwLock> = { - RwLock::new(HashMap::new()) - }; -} - -/// Create rrdd stat dir with correct permission -pub fn create_rrdb_dir() -> Result<(), Error> { - - let backup_user = pbs_config::backup_user()?; - let opts = CreateOptions::new() - .owner(backup_user.uid) - .group(backup_user.gid); - - create_path(PBS_RRD_BASEDIR, None, Some(opts)) - .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; - - Ok(()) -} - -pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result<(), Error> { - - let mut path = PathBuf::from(PBS_RRD_BASEDIR); - path.push(rel_path); - - std::fs::create_dir_all(path.parent().unwrap())?; - - let mut map = RRD_CACHE.write().unwrap(); - let now = proxmox::tools::time::epoch_f64(); - - if let Some(rrd) = map.get_mut(rel_path) { - rrd.update(now, value); - if save { rrd.save(&path)?; } - } else { - let mut rrd = match RRD::load(&path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - RRD::new(dst) - }, - }; - rrd.update(now, value); - if save { rrd.save(&path)?; } - map.insert(rel_path.into(), rrd); - } - - Ok(()) -} - -pub fn extract_cached_data( - base: &str, - name: &str, - now: f64, - timeframe: RRDTimeFrameResolution, - mode: RRDMode, -) -> Option<(u64, u64, Vec>)> { - - let map = RRD_CACHE.read().unwrap(); - - match map.get(&format!("{}/{}", base, name)) { - Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), - None => None, - } -} diff --git a/src/rrd/mod.rs b/src/rrd/mod.rs deleted file mode 100644 index 03e4c9de..00000000 --- a/src/rrd/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[allow(clippy::module_inception)] -mod rrd; -pub use rrd::*; -mod cache; -pub use cache::*;