From 5bc85d2d04ed7a388b56f5efa91667086753a265 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 23 May 2020 09:29:33 +0200 Subject: [PATCH 001/111] add simple rrd implementation --- src/rrd/cache.rs | 82 +++++++++++++++++ src/rrd/mod.rs | 4 + src/rrd/rrd.rs | 224 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 src/rrd/cache.rs create mode 100644 src/rrd/mod.rs create mode 100644 src/rrd/rrd.rs diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs new file mode 100644 index 00000000..9f8ef9f4 --- /dev/null +++ b/src/rrd/cache.rs @@ -0,0 +1,82 @@ +use std::time::{SystemTime, UNIX_EPOCH}; +use std::path::PathBuf; +use std::collections::HashMap; +use std::sync::{RwLock}; + +use anyhow::{format_err, Error}; +use lazy_static::lazy_static; +use serde_json::Value; + +use proxmox::tools::fs::{create_path, CreateOptions}; + +use super::*; + +const PBS_RRD_BASEDIR: &str = "/var/lib/proxmox-backup/rrdb"; + +lazy_static!{ + static ref RRD_CACHE: RwLock> = { + RwLock::new(HashMap::new()) + }; +} + +/// Create rrdd stat dir with correct permission +pub fn create_rrdb_dir() -> Result<(), Error> { + + let backup_user = crate::backup::backup_user()?; + let opts = CreateOptions::new() + .owner(backup_user.uid) + .group(backup_user.gid); + + create_path(PBS_RRD_BASEDIR, None, Some(opts)) + .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; + + Ok(()) +} + +fn now() -> Result { + let epoch = SystemTime::now().duration_since(UNIX_EPOCH)?; + Ok(epoch.as_secs()) +} + +pub fn update_value(rel_path: &str, value: f64) -> Result<(), Error> { + + let mut path = PathBuf::from(PBS_RRD_BASEDIR); + path.push(rel_path); + + std::fs::create_dir_all(path.parent().unwrap())?; + + let mut map = RRD_CACHE.write().unwrap(); + let now = now()?; + + if let Some(rrd) = map.get_mut(rel_path) { + rrd.update(now, value); + rrd.save(&path)?; + } else { + let mut rrd = match RRD::load(&path) { + Ok(rrd) => rrd, + Err(_) => RRD::new(), + }; + rrd.update(now, value); + rrd.save(&path)?; + map.insert(rel_path.into(), rrd); + } + + Ok(()) +} + +pub fn extract_data( + rel_path: &str, + timeframe: RRDTimeFrameResolution, + mode: RRDMode, +) -> Result { + + let now = now()?; + + let map = RRD_CACHE.read().unwrap(); + + if let Some(rrd) = map.get(rel_path) { + Ok(rrd.extract_data(now, timeframe, mode)) + } else { + Ok(RRD::new().extract_data(now, timeframe, mode)) + } +} diff --git a/src/rrd/mod.rs b/src/rrd/mod.rs new file mode 100644 index 00000000..c09efebf --- /dev/null +++ b/src/rrd/mod.rs @@ -0,0 +1,4 @@ +mod rrd; +pub use rrd::*; +mod cache; +pub use cache::*; diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs new file mode 100644 index 00000000..4fcf64f8 --- /dev/null +++ b/src/rrd/rrd.rs @@ -0,0 +1,224 @@ +use std::io::Read; +use std::path::Path; + +use anyhow::{bail, Error}; +use serde_json::{json, Value}; + +const RRD_DATA_ENTRIES: usize = 70; + +#[derive(Copy, Clone)] +pub enum RRDMode { + Max, + Average, +} + +#[repr(u64)] +#[derive(Copy, Clone)] +pub enum RRDTimeFrameResolution { + Hour = 60, // 1 min => last 70 minutes + Day = 60*30, // 30 min => last 35 hours + Week = 60*180, // 3 hours => about 8 days + Month = 60*720, // 12 hours => last 35 days + Year = 60*10080, // 1 week => last 490 days +} + +#[repr(C)] +#[derive(Default, Copy, Clone)] +struct RRDEntry { + max: f64, + average: f64, + count: u64, +} + +#[repr(C)] +// Note: Avoid alignment problems by using 8byte types only +pub struct RRD { + last_update: u64, + hour: [RRDEntry; RRD_DATA_ENTRIES], + day: [RRDEntry; RRD_DATA_ENTRIES], + week: [RRDEntry; RRD_DATA_ENTRIES], + month: [RRDEntry; RRD_DATA_ENTRIES], + year: [RRDEntry; RRD_DATA_ENTRIES], +} + +impl RRD { + + pub fn new() -> Self { + Self { + last_update: 0, + hour: [RRDEntry::default(); RRD_DATA_ENTRIES], + day: [RRDEntry::default(); RRD_DATA_ENTRIES], + week: [RRDEntry::default(); RRD_DATA_ENTRIES], + month: [RRDEntry::default(); RRD_DATA_ENTRIES], + year: [RRDEntry::default(); RRD_DATA_ENTRIES], + } + } + + pub fn extract_data( + &self, + epoch: u64, + timeframe: RRDTimeFrameResolution, + mode: RRDMode, + ) -> Value { + + let reso = timeframe as u64; + + let end = reso*(epoch/reso); + let start = end - reso*(RRD_DATA_ENTRIES as u64); + + let rrd_end = reso*(self.last_update/reso); + let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); + + let mut list = Vec::new(); + + let data = match timeframe { + RRDTimeFrameResolution::Hour => &self.hour, + RRDTimeFrameResolution::Day => &self.day, + RRDTimeFrameResolution::Week => &self.week, + RRDTimeFrameResolution::Month => &self.month, + RRDTimeFrameResolution::Year => &self.year, + }; + + let mut t = start; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + if t < rrd_start || t > rrd_end { + list.push(json!({ "time": t })); + } else { + let entry = data[index]; + if entry.count == 0 { + list.push(json!({ "time": t })); + } else { + let value = match mode { + RRDMode::Max => entry.max, + RRDMode::Average => entry.average, + }; + list.push(json!({ "time": t, "value": value })); + } + } + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + + list.into() + } + + pub fn from_raw(mut raw: &[u8]) -> Result { + let expected_len = std::mem::size_of::(); + if raw.len() != expected_len { + bail!("RRD::from_raw failed - wrong data size ({} != {})", raw.len(), expected_len); + } + + let mut rrd: RRD = unsafe { std::mem::zeroed() }; + unsafe { + let rrd_slice = std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len); + raw.read_exact(rrd_slice)?; + } + + Ok(rrd) + } + + pub fn load(filename: &Path) -> Result { + let raw = proxmox::tools::fs::file_get_contents(filename)?; + Self::from_raw(&raw) + } + + pub fn save(&self, filename: &Path) -> Result<(), Error> { + use proxmox::tools::{fs::replace_file, fs::CreateOptions}; + + let rrd_slice = unsafe { + std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) + }; + + let backup_user = crate::backup::backup_user()?; + let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); + // set the correct owner/group/permissions while saving file + // owner(rw) = backup, group(r)= backup + let options = CreateOptions::new() + .perm(mode) + .owner(backup_user.uid) + .group(backup_user.gid); + + replace_file(filename, rrd_slice, options)?; + + Ok(()) + } + + fn compute_new_value( + data: &[RRDEntry; RRD_DATA_ENTRIES], + index: usize, + value: f64, + ) -> RRDEntry { + let RRDEntry { max, average, count } = data[index]; + let new_count = count + 1; // fixme: check overflow? + if count == 0 { + RRDEntry { max: value, average: value, count: 1 } + } else { + let new_max = if max > value { max } else { value }; + let new_average = (average*(count as f64) + value)/(new_count as f64); + RRDEntry { max: new_max, average: new_average, count: new_count } + } + } + + pub fn update(&mut self, epoch: u64, value: f64) { + // fixme: check time progress (epoch last) + let last = self.last_update; + + let reso = RRDTimeFrameResolution::Hour as u64; + + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + if t < min_time { self.hour[index] = RRDEntry::default(); } + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + self.hour[index] = Self::compute_new_value(&self.hour, index, value); + + let reso = RRDTimeFrameResolution::Day as u64; + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + if t < min_time { self.day[index] = RRDEntry::default(); } + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + self.day[index] = Self::compute_new_value(&self.day, index, value); + + let reso = RRDTimeFrameResolution::Week as u64; + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + if t < min_time { self.week[index] = RRDEntry::default(); } + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + self.week[index] = Self::compute_new_value(&self.week, index, value); + + let reso = RRDTimeFrameResolution::Month as u64; + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + if t < min_time { self.month[index] = RRDEntry::default(); } + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + self.month[index] = Self::compute_new_value(&self.month, index, value); + + let reso = RRDTimeFrameResolution::Year as u64; + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + if t < min_time { self.year[index] = RRDEntry::default(); } + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + self.year[index] = Self::compute_new_value(&self.year, index, value); + + self.last_update = epoch; + } +} From 1c0117790fb600ed7a6231fdc0e6abcda53d575c Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 23 May 2020 11:10:02 +0200 Subject: [PATCH 002/111] add experimental rrd api to get cpu stats --- src/rrd/cache.rs | 2 ++ src/rrd/rrd.rs | 17 ++--------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 9f8ef9f4..6baa2e33 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -9,6 +9,8 @@ use serde_json::Value; use proxmox::tools::fs::{create_path, CreateOptions}; +use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; + use super::*; const PBS_RRD_BASEDIR: &str = "/var/lib/proxmox-backup/rrdb"; diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 4fcf64f8..681d5be1 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -4,23 +4,10 @@ use std::path::Path; use anyhow::{bail, Error}; use serde_json::{json, Value}; +use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; + const RRD_DATA_ENTRIES: usize = 70; -#[derive(Copy, Clone)] -pub enum RRDMode { - Max, - Average, -} - -#[repr(u64)] -#[derive(Copy, Clone)] -pub enum RRDTimeFrameResolution { - Hour = 60, // 1 min => last 70 minutes - Day = 60*30, // 30 min => last 35 hours - Week = 60*180, // 3 hours => about 8 days - Month = 60*720, // 12 hours => last 35 days - Year = 60*10080, // 1 week => last 490 days -} #[repr(C)] #[derive(Default, Copy, Clone)] From e073e5c107af920f0b606f851121d94e14ff06cb Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 23 May 2020 14:03:44 +0200 Subject: [PATCH 003/111] rrd: pack multiple rrd values into th estat list --- src/rrd/cache.rs | 33 ++++++++++++++++++-- src/rrd/rrd.rs | 80 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 6baa2e33..c80b9f76 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -49,7 +49,7 @@ pub fn update_value(rel_path: &str, value: f64) -> Result<(), Error> { let mut map = RRD_CACHE.write().unwrap(); let now = now()?; - + if let Some(rrd) = map.get_mut(rel_path) { rrd.update(now, value); rrd.save(&path)?; @@ -62,7 +62,7 @@ pub fn update_value(rel_path: &str, value: f64) -> Result<(), Error> { rrd.save(&path)?; map.insert(rel_path.into(), rrd); } - + Ok(()) } @@ -75,10 +75,37 @@ pub fn extract_data( let now = now()?; let map = RRD_CACHE.read().unwrap(); - + if let Some(rrd) = map.get(rel_path) { Ok(rrd.extract_data(now, timeframe, mode)) } else { Ok(RRD::new().extract_data(now, timeframe, mode)) } } + + +pub fn extract_data_list( + base: &str, + items: &[&str], + timeframe: RRDTimeFrameResolution, + mode: RRDMode, +) -> Result { + + let now = now()?; + + let map = RRD_CACHE.read().unwrap(); + + let mut list: Vec<(&str, &RRD)> = Vec::new(); + + let empty_rrd = RRD::new(); + + for name in items.iter() { + if let Some(rrd) = map.get(&format!("{}/{}", base, name)) { + list.push((name, rrd)); + } else { + list.push((name, &empty_rrd)); + } + } + + Ok(extract_rrd_data(&list, now, timeframe, mode)) +} diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 681d5be1..398d48cb 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -6,8 +6,7 @@ use serde_json::{json, Value}; use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; -const RRD_DATA_ENTRIES: usize = 70; - +pub const RRD_DATA_ENTRIES: usize = 70; #[repr(C)] #[derive(Default, Copy, Clone)] @@ -65,7 +64,7 @@ impl RRD { RRDTimeFrameResolution::Month => &self.month, RRDTimeFrameResolution::Year => &self.year, }; - + let mut t = start; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; for _ in 0..RRD_DATA_ENTRIES { @@ -94,7 +93,7 @@ impl RRD { if raw.len() != expected_len { bail!("RRD::from_raw failed - wrong data size ({} != {})", raw.len(), expected_len); } - + let mut rrd: RRD = unsafe { std::mem::zeroed() }; unsafe { let rrd_slice = std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len); @@ -108,7 +107,7 @@ impl RRD { let raw = proxmox::tools::fs::file_get_contents(filename)?; Self::from_raw(&raw) } - + pub fn save(&self, filename: &Path) -> Result<(), Error> { use proxmox::tools::{fs::replace_file, fs::CreateOptions}; @@ -126,15 +125,15 @@ impl RRD { .group(backup_user.gid); replace_file(filename, rrd_slice, options)?; - + Ok(()) } - + fn compute_new_value( data: &[RRDEntry; RRD_DATA_ENTRIES], index: usize, value: f64, - ) -> RRDEntry { + ) -> RRDEntry { let RRDEntry { max, average, count } = data[index]; let new_count = count + 1; // fixme: check overflow? if count == 0 { @@ -145,7 +144,7 @@ impl RRD { RRDEntry { max: new_max, average: new_average, count: new_count } } } - + pub fn update(&mut self, epoch: u64, value: f64) { // fixme: check time progress (epoch last) let last = self.last_update; @@ -162,7 +161,7 @@ impl RRD { let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; self.hour[index] = Self::compute_new_value(&self.hour, index, value); - let reso = RRDTimeFrameResolution::Day as u64; + let reso = RRDTimeFrameResolution::Day as u64; let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; let mut t = last; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; @@ -172,8 +171,8 @@ impl RRD { } let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; self.day[index] = Self::compute_new_value(&self.day, index, value); - - let reso = RRDTimeFrameResolution::Week as u64; + + let reso = RRDTimeFrameResolution::Week as u64; let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; let mut t = last; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; @@ -184,7 +183,7 @@ impl RRD { let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; self.week[index] = Self::compute_new_value(&self.week, index, value); - let reso = RRDTimeFrameResolution::Month as u64; + let reso = RRDTimeFrameResolution::Month as u64; let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; let mut t = last; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; @@ -194,8 +193,8 @@ impl RRD { } let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; self.month[index] = Self::compute_new_value(&self.month, index, value); - - let reso = RRDTimeFrameResolution::Year as u64; + + let reso = RRDTimeFrameResolution::Year as u64; let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; let mut t = last; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; @@ -209,3 +208,54 @@ impl RRD { self.last_update = epoch; } } + +pub fn extract_rrd_data( + rrd_list: &[(&str, &RRD)], + epoch: u64, + timeframe: RRDTimeFrameResolution, + mode: RRDMode, +) -> Value { + + let reso = timeframe as u64; + + let end = reso*(epoch/reso); + let start = end - reso*(RRD_DATA_ENTRIES as u64); + + let mut list = Vec::new(); + + let mut t = start; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + let mut item = json!({ "time": t }); + for (name, rrd) in rrd_list.iter() { + let rrd_end = reso*(rrd.last_update/reso); + let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); + + if t < rrd_start || t > rrd_end { + continue; + } else { + let data = match timeframe { + RRDTimeFrameResolution::Hour => &rrd.hour, + RRDTimeFrameResolution::Day => &rrd.day, + RRDTimeFrameResolution::Week => &rrd.week, + RRDTimeFrameResolution::Month => &rrd.month, + RRDTimeFrameResolution::Year => &rrd.year, + }; + let entry = data[index]; + if entry.count == 0 { + continue; + } else { + let value = match mode { + RRDMode::Max => entry.max, + RRDMode::Average => entry.average, + }; + item[name] = value.into(); + } + } + } + list.push(item); + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + + list.into() +} From 69da9baf59807b3ca1decc6ee363ca74e46a78ec Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 23 May 2020 15:37:17 +0200 Subject: [PATCH 004/111] rrd: simplify code --- src/rrd/cache.rs | 46 +++++++++++++++-------------------- src/rrd/rrd.rs | 63 +++++------------------------------------------- 2 files changed, 26 insertions(+), 83 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index c80b9f76..e1209aff 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -5,7 +5,7 @@ use std::sync::{RwLock}; use anyhow::{format_err, Error}; use lazy_static::lazy_static; -use serde_json::Value; +use serde_json::{json, Value}; use proxmox::tools::fs::{create_path, CreateOptions}; @@ -67,24 +67,6 @@ pub fn update_value(rel_path: &str, value: f64) -> Result<(), Error> { } pub fn extract_data( - rel_path: &str, - timeframe: RRDTimeFrameResolution, - mode: RRDMode, -) -> Result { - - let now = now()?; - - let map = RRD_CACHE.read().unwrap(); - - if let Some(rrd) = map.get(rel_path) { - Ok(rrd.extract_data(now, timeframe, mode)) - } else { - Ok(RRD::new().extract_data(now, timeframe, mode)) - } -} - - -pub fn extract_data_list( base: &str, items: &[&str], timeframe: RRDTimeFrameResolution, @@ -95,17 +77,29 @@ pub fn extract_data_list( let map = RRD_CACHE.read().unwrap(); - let mut list: Vec<(&str, &RRD)> = Vec::new(); - let empty_rrd = RRD::new(); + let mut result = Vec::new(); + for name in items.iter() { - if let Some(rrd) = map.get(&format!("{}/{}", base, name)) { - list.push((name, rrd)); - } else { - list.push((name, &empty_rrd)); + let rrd = map.get(&format!("{}/{}", base, name)).unwrap_or(&empty_rrd); + let (start, reso, list) = rrd.extract_data(now, timeframe, mode); + let mut t = start; + for index in 0..RRD_DATA_ENTRIES { + if result.len() <= index { + if let Some(value) = list[index] { + result.push(json!({ "time": t, *name: value })); + } else { + result.push(json!({ "time": t })); + } + } else { + if let Some(value) = list[index] { + result[index][name] = value.into(); + } + } + t += reso; } } - Ok(extract_rrd_data(&list, now, timeframe, mode)) + Ok(result.into()) } diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 398d48cb..0088e3bb 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -45,11 +45,11 @@ impl RRD { epoch: u64, timeframe: RRDTimeFrameResolution, mode: RRDMode, - ) -> Value { + ) -> (u64, u64, Vec>) { let reso = timeframe as u64; - let end = reso*(epoch/reso); + let end = reso*((epoch + reso -1)/reso); let start = end - reso*(RRD_DATA_ENTRIES as u64); let rrd_end = reso*(self.last_update/reso); @@ -69,23 +69,23 @@ impl RRD { let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; for _ in 0..RRD_DATA_ENTRIES { if t < rrd_start || t > rrd_end { - list.push(json!({ "time": t })); + list.push(None); } else { let entry = data[index]; if entry.count == 0 { - list.push(json!({ "time": t })); + list.push(None); } else { let value = match mode { RRDMode::Max => entry.max, RRDMode::Average => entry.average, }; - list.push(json!({ "time": t, "value": value })); + list.push(Some(value)); } } t += reso; index = (index + 1) % RRD_DATA_ENTRIES; } - list.into() + (start, reso, list.into()) } pub fn from_raw(mut raw: &[u8]) -> Result { @@ -208,54 +208,3 @@ impl RRD { self.last_update = epoch; } } - -pub fn extract_rrd_data( - rrd_list: &[(&str, &RRD)], - epoch: u64, - timeframe: RRDTimeFrameResolution, - mode: RRDMode, -) -> Value { - - let reso = timeframe as u64; - - let end = reso*(epoch/reso); - let start = end - reso*(RRD_DATA_ENTRIES as u64); - - let mut list = Vec::new(); - - let mut t = start; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - let mut item = json!({ "time": t }); - for (name, rrd) in rrd_list.iter() { - let rrd_end = reso*(rrd.last_update/reso); - let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); - - if t < rrd_start || t > rrd_end { - continue; - } else { - let data = match timeframe { - RRDTimeFrameResolution::Hour => &rrd.hour, - RRDTimeFrameResolution::Day => &rrd.day, - RRDTimeFrameResolution::Week => &rrd.week, - RRDTimeFrameResolution::Month => &rrd.month, - RRDTimeFrameResolution::Year => &rrd.year, - }; - let entry = data[index]; - if entry.count == 0 { - continue; - } else { - let value = match mode { - RRDMode::Max => entry.max, - RRDMode::Average => entry.average, - }; - item[name] = value.into(); - } - } - } - list.push(item); - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - } - - list.into() -} From 6357746f2ec95bcf71841c63933fb8a93294729e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 23 May 2020 16:03:43 +0200 Subject: [PATCH 005/111] rrd: fix display interval, try to avoid numeric errors --- src/rrd/rrd.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 0088e3bb..1553e72c 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -2,7 +2,6 @@ use std::io::Read; use std::path::Path; use anyhow::{bail, Error}; -use serde_json::{json, Value}; use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; @@ -49,7 +48,7 @@ impl RRD { let reso = timeframe as u64; - let end = reso*((epoch + reso -1)/reso); + let end = reso*(epoch/reso + 1); let start = end - reso*(RRD_DATA_ENTRIES as u64); let rrd_end = reso*(self.last_update/reso); @@ -140,7 +139,10 @@ impl RRD { RRDEntry { max: value, average: value, count: 1 } } else { let new_max = if max > value { max } else { value }; - let new_average = (average*(count as f64) + value)/(new_count as f64); + //let new_average = (average*(count as f64) + value)/(new_count as f64); + // Note: Try to avoid numeric errors + let new_average = average*((count as f64)/(new_count as f64)) + + value/(new_count as f64); RRDEntry { max: new_max, average: new_average, count: new_count } } } From fd2c16f704b1e895fb929aeb614b556f10cfbc70 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 24 May 2020 06:44:06 +0200 Subject: [PATCH 006/111] src/rrd/rrd.rs: simplify an fix old value deletion --- src/rrd/rrd.rs | 93 +++++++++++++++++++++----------------------------- 1 file changed, 38 insertions(+), 55 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 1553e72c..6d610a0b 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -129,83 +129,66 @@ impl RRD { } fn compute_new_value( - data: &[RRDEntry; RRD_DATA_ENTRIES], - index: usize, + data: &mut [RRDEntry; RRD_DATA_ENTRIES], + epoch: u64, + reso: u64, value: f64, - ) -> RRDEntry { + ) { + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; let RRDEntry { max, average, count } = data[index]; let new_count = count + 1; // fixme: check overflow? if count == 0 { - RRDEntry { max: value, average: value, count: 1 } - } else { + data[index] = RRDEntry { max: value, average: value, count: 1 }; + } else { let new_max = if max > value { max } else { value }; - //let new_average = (average*(count as f64) + value)/(new_count as f64); + // let new_average = (average*(count as f64) + value)/(new_count as f64); // Note: Try to avoid numeric errors - let new_average = average*((count as f64)/(new_count as f64)) + let new_average = (average*(count as f64))/(new_count as f64) + value/(new_count as f64); - RRDEntry { max: new_max, average: new_average, count: new_count } + data[index] = RRDEntry { max: new_max, average: new_average, count: new_count }; + } + } + + fn delete_old(data: &mut [RRDEntry], epoch: u64, last: u64, reso: u64) { + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let min_time = (min_time/reso + 1)*reso; + let mut t = last - (RRD_DATA_ENTRIES as u64)*reso; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + if t < min_time { + data[index] = RRDEntry::default(); + } else { + break; + } } } pub fn update(&mut self, epoch: u64, value: f64) { - // fixme: check time progress (epoch last) let last = self.last_update; + if epoch < last { + eprintln!("rrdb update failed - time in past ({} < {})", epoch, last); + } let reso = RRDTimeFrameResolution::Hour as u64; - - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; - let mut t = last; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - if t < min_time { self.hour[index] = RRDEntry::default(); } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - } - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - self.hour[index] = Self::compute_new_value(&self.hour, index, value); + Self::delete_old(&mut self.hour, epoch, last, reso); + Self::compute_new_value(&mut self.hour, epoch, reso, value); let reso = RRDTimeFrameResolution::Day as u64; - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; - let mut t = last; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - if t < min_time { self.day[index] = RRDEntry::default(); } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - } - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - self.day[index] = Self::compute_new_value(&self.day, index, value); + Self::delete_old(&mut self.day, epoch, last, reso); + Self::compute_new_value(&mut self.day, epoch, reso, value); let reso = RRDTimeFrameResolution::Week as u64; - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; - let mut t = last; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - if t < min_time { self.week[index] = RRDEntry::default(); } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - } - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - self.week[index] = Self::compute_new_value(&self.week, index, value); + Self::delete_old(&mut self.week, epoch, last, reso); + Self::compute_new_value(&mut self.week, epoch, reso, value); let reso = RRDTimeFrameResolution::Month as u64; - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; - let mut t = last; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - if t < min_time { self.month[index] = RRDEntry::default(); } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - } - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - self.month[index] = Self::compute_new_value(&self.month, index, value); + Self::delete_old(&mut self.month, epoch, last, reso); + Self::compute_new_value(&mut self.month, epoch, reso, value); let reso = RRDTimeFrameResolution::Year as u64; - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; - let mut t = last; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - if t < min_time { self.year[index] = RRDEntry::default(); } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - } - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - self.year[index] = Self::compute_new_value(&self.year, index, value); + Self::delete_old(&mut self.year, epoch, last, reso); + Self::compute_new_value(&mut self.year, epoch, reso, value); self.last_update = epoch; } From 777f52760352279de285890a138b5ad894a6b668 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 24 May 2020 09:09:09 +0200 Subject: [PATCH 007/111] src/rrd/rrd.rs: reduce size by using f64:NAN as UNKNOWN --- src/rrd/rrd.rs | 70 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 6d610a0b..0edfe1cb 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -8,21 +8,31 @@ use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; pub const RRD_DATA_ENTRIES: usize = 70; #[repr(C)] -#[derive(Default, Copy, Clone)] +#[derive(Copy, Clone)] struct RRDEntry { max: f64, average: f64, - count: u64, +} + +impl Default for RRDEntry { + fn default() -> Self { + Self { max: f64::NAN, average: f64::NAN } + } } #[repr(C)] // Note: Avoid alignment problems by using 8byte types only pub struct RRD { last_update: u64, + last_hour_count: u64, hour: [RRDEntry; RRD_DATA_ENTRIES], + last_day_count: u64, day: [RRDEntry; RRD_DATA_ENTRIES], + last_week_count: u64, week: [RRDEntry; RRD_DATA_ENTRIES], + last_month_count: u64, month: [RRDEntry; RRD_DATA_ENTRIES], + last_year_count: u64, year: [RRDEntry; RRD_DATA_ENTRIES], } @@ -31,10 +41,15 @@ impl RRD { pub fn new() -> Self { Self { last_update: 0, + last_hour_count: 0, hour: [RRDEntry::default(); RRD_DATA_ENTRIES], + last_day_count: 0, day: [RRDEntry::default(); RRD_DATA_ENTRIES], + last_week_count: 0, week: [RRDEntry::default(); RRD_DATA_ENTRIES], + last_month_count: 0, month: [RRDEntry::default(); RRD_DATA_ENTRIES], + last_year_count: 0, year: [RRDEntry::default(); RRD_DATA_ENTRIES], } } @@ -71,13 +86,13 @@ impl RRD { list.push(None); } else { let entry = data[index]; - if entry.count == 0 { + let value = match mode { + RRDMode::Max => entry.max, + RRDMode::Average => entry.average, + }; + if value.is_nan() { list.push(None); } else { - let value = match mode { - RRDMode::Max => entry.max, - RRDMode::Average => entry.average, - }; list.push(Some(value)); } } @@ -130,22 +145,41 @@ impl RRD { fn compute_new_value( data: &mut [RRDEntry; RRD_DATA_ENTRIES], + count: &mut u64, epoch: u64, + last: u64, reso: u64, value: f64, ) { + if value.is_nan() { + eprintln!("rrdb update failed - new value is NAN"); + return; + } + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - let RRDEntry { max, average, count } = data[index]; - let new_count = count + 1; // fixme: check overflow? - if count == 0 { - data[index] = RRDEntry { max: value, average: value, count: 1 }; + let last_index = ((last/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + + if (epoch - last) > reso || index != last_index { + *count = 0; + } + + let RRDEntry { max, average } = data[index]; + if max.is_nan() || average.is_nan() { + *count = 0; + } + + let new_count = *count + 1; // fixme: check overflow? + if *count == 0 { + data[index] = RRDEntry { max: value, average: value }; + *count = 1; } else { let new_max = if max > value { max } else { value }; // let new_average = (average*(count as f64) + value)/(new_count as f64); // Note: Try to avoid numeric errors - let new_average = (average*(count as f64))/(new_count as f64) + let new_average = (average*(*count as f64))/(new_count as f64) + value/(new_count as f64); - data[index] = RRDEntry { max: new_max, average: new_average, count: new_count }; + data[index] = RRDEntry { max: new_max, average: new_average }; + *count = new_count; } } @@ -172,23 +206,23 @@ impl RRD { let reso = RRDTimeFrameResolution::Hour as u64; Self::delete_old(&mut self.hour, epoch, last, reso); - Self::compute_new_value(&mut self.hour, epoch, reso, value); + Self::compute_new_value(&mut self.hour, &mut self.last_hour_count, epoch, last, reso, value); let reso = RRDTimeFrameResolution::Day as u64; Self::delete_old(&mut self.day, epoch, last, reso); - Self::compute_new_value(&mut self.day, epoch, reso, value); + Self::compute_new_value(&mut self.day, &mut self.last_day_count, epoch, last, reso, value); let reso = RRDTimeFrameResolution::Week as u64; Self::delete_old(&mut self.week, epoch, last, reso); - Self::compute_new_value(&mut self.week, epoch, reso, value); + Self::compute_new_value(&mut self.week, &mut self.last_week_count, epoch, last, reso, value); let reso = RRDTimeFrameResolution::Month as u64; Self::delete_old(&mut self.month, epoch, last, reso); - Self::compute_new_value(&mut self.month, epoch, reso, value); + Self::compute_new_value(&mut self.month, &mut self.last_month_count, epoch, last, reso, value); let reso = RRDTimeFrameResolution::Year as u64; Self::delete_old(&mut self.year, epoch, last, reso); - Self::compute_new_value(&mut self.year, epoch, reso, value); + Self::compute_new_value(&mut self.year, &mut self.last_year_count, epoch, last, reso, value); self.last_update = epoch; } From 7acdec12e84824e444034e475a113e69a2c266fc Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 24 May 2020 16:51:28 +0200 Subject: [PATCH 008/111] src/rrd/rrd.rs: restructure whole code --- src/rrd/cache.rs | 11 +- src/rrd/rrd.rs | 301 ++++++++++++++++++++++++++++------------------- 2 files changed, 186 insertions(+), 126 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index e1209aff..4e61c949 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -40,7 +40,7 @@ fn now() -> Result { Ok(epoch.as_secs()) } -pub fn update_value(rel_path: &str, value: f64) -> Result<(), Error> { +pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { let mut path = PathBuf::from(PBS_RRD_BASEDIR); path.push(rel_path); @@ -56,7 +56,7 @@ pub fn update_value(rel_path: &str, value: f64) -> Result<(), Error> { } else { let mut rrd = match RRD::load(&path) { Ok(rrd) => rrd, - Err(_) => RRD::new(), + Err(_) => RRD::new(dst), }; rrd.update(now, value); rrd.save(&path)?; @@ -77,12 +77,13 @@ pub fn extract_data( let map = RRD_CACHE.read().unwrap(); - let empty_rrd = RRD::new(); - let mut result = Vec::new(); for name in items.iter() { - let rrd = map.get(&format!("{}/{}", base, name)).unwrap_or(&empty_rrd); + let rrd = match map.get(&format!("{}/{}", base, name)) { + Some(rrd) => rrd, + None => continue, + }; let (start, reso, list) = rrd.extract_data(now, timeframe, mode); let mut t = start; for index in 0..RRD_DATA_ENTRIES { diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 0edfe1cb..a48b7b04 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -7,50 +7,174 @@ use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; pub const RRD_DATA_ENTRIES: usize = 70; -#[repr(C)] -#[derive(Copy, Clone)] -struct RRDEntry { - max: f64, - average: f64, +use bitflags::bitflags; + +bitflags!{ + pub struct RRAFlags: u64 { + // Data Source Types + const DST_GAUGE = 1; + const DST_DERIVE = 2; + const DST_MASK = 255; // first 8 bits + + // Consolidation Functions + const CF_AVERAGE = 1 << 8; + const CF_MAX = 2 << 8; + const CF_MASK = 255 << 8; + } } -impl Default for RRDEntry { - fn default() -> Self { - Self { max: f64::NAN, average: f64::NAN } +pub enum DST { + Gauge, + Derive, +} + +#[repr(C)] +struct RRA { + flags: RRAFlags, + resolution: u64, + last_update: u64, + last_count: u64, + data: [f64; RRD_DATA_ENTRIES], +} + +impl RRA { + fn new(flags: RRAFlags, resolution: u64) -> Self { + Self { + flags, resolution, + last_update: 0, + last_count: 0, + data: [f64::NAN; RRD_DATA_ENTRIES], + } + } + + fn delete_old(&mut self, epoch: u64) { + let reso = self.resolution; + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + let min_time = (min_time/reso + 1)*reso; + let mut t = self.last_update - (RRD_DATA_ENTRIES as u64)*reso; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + if t < min_time { + self.data[index] = f64::NAN; + } else { + break; + } + } + } + + fn compute_new_value(&mut self, epoch: u64, value: f64) { + let reso = self.resolution; + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + let last_index = ((self.last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + + if (epoch - self.last_update) > reso || index != last_index { + self.last_count = 0; + } + + let last_value = self.data[index]; + if last_value.is_nan() { + self.last_count = 0; + } + + let new_count = self.last_count + 1; // fixme: check overflow? + if self.last_count == 0 { + self.data[index] = value; + self.last_count = 1; + } else { + let new_value = if self.flags.contains(RRAFlags::CF_MAX) { + if last_value > value { last_value } else { value } + } else if self.flags.contains(RRAFlags::CF_AVERAGE) { + (last_value*(self.last_count as f64))/(new_count as f64) + + value/(new_count as f64) + } else { + eprintln!("rrdb update failed - unknown CF"); + return; + }; + self.data[index] = new_value; + self.last_count = new_count; + } + self.last_update = epoch; + } + + fn update(&mut self, epoch: u64, value: f64) { + if epoch < self.last_update { + eprintln!("rrdb update failed - time in past ({} < {})", epoch, self.last_update); + } + if value.is_nan() { + eprintln!("rrdb update failed - new value is NAN"); + return; + } + + self.delete_old(epoch); + self.compute_new_value(epoch, value); } } #[repr(C)] // Note: Avoid alignment problems by using 8byte types only pub struct RRD { - last_update: u64, - last_hour_count: u64, - hour: [RRDEntry; RRD_DATA_ENTRIES], - last_day_count: u64, - day: [RRDEntry; RRD_DATA_ENTRIES], - last_week_count: u64, - week: [RRDEntry; RRD_DATA_ENTRIES], - last_month_count: u64, - month: [RRDEntry; RRD_DATA_ENTRIES], - last_year_count: u64, - year: [RRDEntry; RRD_DATA_ENTRIES], + hour_avg: RRA, + hour_max: RRA, + day_avg: RRA, + day_max: RRA, + week_avg: RRA, + week_max: RRA, + month_avg: RRA, + month_max: RRA, + year_avg: RRA, + year_max: RRA, } impl RRD { - pub fn new() -> Self { + pub fn new(dst: DST) -> Self { + let flags = match dst { + DST::Gauge => RRAFlags::DST_GAUGE, + DST::Derive => RRAFlags::DST_DERIVE, + }; + Self { - last_update: 0, - last_hour_count: 0, - hour: [RRDEntry::default(); RRD_DATA_ENTRIES], - last_day_count: 0, - day: [RRDEntry::default(); RRD_DATA_ENTRIES], - last_week_count: 0, - week: [RRDEntry::default(); RRD_DATA_ENTRIES], - last_month_count: 0, - month: [RRDEntry::default(); RRD_DATA_ENTRIES], - last_year_count: 0, - year: [RRDEntry::default(); RRD_DATA_ENTRIES], + hour_avg: RRA::new( + flags | RRAFlags::CF_AVERAGE, + RRDTimeFrameResolution::Hour as u64, + ), + hour_max: RRA::new( + flags | RRAFlags::CF_MAX, + RRDTimeFrameResolution::Hour as u64, + ), + day_avg: RRA::new( + flags | RRAFlags::CF_AVERAGE, + RRDTimeFrameResolution::Day as u64, + ), + day_max: RRA::new( + flags | RRAFlags::CF_MAX, + RRDTimeFrameResolution::Day as u64, + ), + week_avg: RRA::new( + flags | RRAFlags::CF_AVERAGE, + RRDTimeFrameResolution::Week as u64, + ), + week_max: RRA::new( + flags | RRAFlags::CF_MAX, + RRDTimeFrameResolution::Week as u64, + ), + month_avg: RRA::new( + flags | RRAFlags::CF_AVERAGE, + RRDTimeFrameResolution::Month as u64, + ), + month_max: RRA::new( + flags | RRAFlags::CF_MAX, + RRDTimeFrameResolution::Month as u64, + ), + year_avg: RRA::new( + flags | RRAFlags::CF_AVERAGE, + RRDTimeFrameResolution::Year as u64, + ), + year_max: RRA::new( + flags | RRAFlags::CF_MAX, + RRDTimeFrameResolution::Year as u64, + ), } } @@ -66,30 +190,31 @@ impl RRD { let end = reso*(epoch/reso + 1); let start = end - reso*(RRD_DATA_ENTRIES as u64); - let rrd_end = reso*(self.last_update/reso); - let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); - let mut list = Vec::new(); - let data = match timeframe { - RRDTimeFrameResolution::Hour => &self.hour, - RRDTimeFrameResolution::Day => &self.day, - RRDTimeFrameResolution::Week => &self.week, - RRDTimeFrameResolution::Month => &self.month, - RRDTimeFrameResolution::Year => &self.year, + let raa = match (mode, timeframe) { + (RRDMode::Average, RRDTimeFrameResolution::Hour) => &self.hour_avg, + (RRDMode::Max, RRDTimeFrameResolution::Hour) => &self.hour_max, + (RRDMode::Average, RRDTimeFrameResolution::Day) => &self.day_avg, + (RRDMode::Max, RRDTimeFrameResolution::Day) => &self.day_max, + (RRDMode::Average, RRDTimeFrameResolution::Week) => &self.week_avg, + (RRDMode::Max, RRDTimeFrameResolution::Week) => &self.week_max, + (RRDMode::Average, RRDTimeFrameResolution::Month) => &self.month_avg, + (RRDMode::Max, RRDTimeFrameResolution::Month) => &self.month_max, + (RRDMode::Average, RRDTimeFrameResolution::Year) => &self.year_avg, + (RRDMode::Max, RRDTimeFrameResolution::Year) => &self.year_max, }; + let rrd_end = reso*(raa.last_update/reso); + let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); + let mut t = start; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; for _ in 0..RRD_DATA_ENTRIES { if t < rrd_start || t > rrd_end { list.push(None); } else { - let entry = data[index]; - let value = match mode { - RRDMode::Max => entry.max, - RRDMode::Average => entry.average, - }; + let value = raa.data[index]; if value.is_nan() { list.push(None); } else { @@ -143,87 +268,21 @@ impl RRD { Ok(()) } - fn compute_new_value( - data: &mut [RRDEntry; RRD_DATA_ENTRIES], - count: &mut u64, - epoch: u64, - last: u64, - reso: u64, - value: f64, - ) { - if value.is_nan() { - eprintln!("rrdb update failed - new value is NAN"); - return; - } - - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - let last_index = ((last/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - - if (epoch - last) > reso || index != last_index { - *count = 0; - } - - let RRDEntry { max, average } = data[index]; - if max.is_nan() || average.is_nan() { - *count = 0; - } - - let new_count = *count + 1; // fixme: check overflow? - if *count == 0 { - data[index] = RRDEntry { max: value, average: value }; - *count = 1; - } else { - let new_max = if max > value { max } else { value }; - // let new_average = (average*(count as f64) + value)/(new_count as f64); - // Note: Try to avoid numeric errors - let new_average = (average*(*count as f64))/(new_count as f64) - + value/(new_count as f64); - data[index] = RRDEntry { max: new_max, average: new_average }; - *count = new_count; - } - } - - fn delete_old(data: &mut [RRDEntry], epoch: u64, last: u64, reso: u64) { - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; - let min_time = (min_time/reso + 1)*reso; - let mut t = last - (RRD_DATA_ENTRIES as u64)*reso; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; - if t < min_time { - data[index] = RRDEntry::default(); - } else { - break; - } - } - } pub fn update(&mut self, epoch: u64, value: f64) { - let last = self.last_update; - if epoch < last { - eprintln!("rrdb update failed - time in past ({} < {})", epoch, last); - } + self.hour_avg.update(epoch, value); + self.hour_max.update(epoch, value); - let reso = RRDTimeFrameResolution::Hour as u64; - Self::delete_old(&mut self.hour, epoch, last, reso); - Self::compute_new_value(&mut self.hour, &mut self.last_hour_count, epoch, last, reso, value); + self.day_avg.update(epoch, value); + self.day_max.update(epoch, value); - let reso = RRDTimeFrameResolution::Day as u64; - Self::delete_old(&mut self.day, epoch, last, reso); - Self::compute_new_value(&mut self.day, &mut self.last_day_count, epoch, last, reso, value); + self.week_avg.update(epoch, value); + self.week_max.update(epoch, value); - let reso = RRDTimeFrameResolution::Week as u64; - Self::delete_old(&mut self.week, epoch, last, reso); - Self::compute_new_value(&mut self.week, &mut self.last_week_count, epoch, last, reso, value); + self.month_avg.update(epoch, value); + self.month_max.update(epoch, value); - let reso = RRDTimeFrameResolution::Month as u64; - Self::delete_old(&mut self.month, epoch, last, reso); - Self::compute_new_value(&mut self.month, &mut self.last_month_count, epoch, last, reso, value); - - let reso = RRDTimeFrameResolution::Year as u64; - Self::delete_old(&mut self.year, epoch, last, reso); - Self::compute_new_value(&mut self.year, &mut self.last_year_count, epoch, last, reso, value); - - self.last_update = epoch; + self.year_avg.update(epoch, value); + self.year_max.update(epoch, value); } } From f4561029bac310d0c8ac7fc910c24f36db71053b Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 24 May 2020 17:03:02 +0200 Subject: [PATCH 009/111] src/rrd/rrd.rs: implement DST_DERIVE --- src/rrd/rrd.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index a48b7b04..002b442a 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -34,6 +34,7 @@ struct RRA { resolution: u64, last_update: u64, last_count: u64, + counter_value: f64, // used for derive/counters data: [f64; RRD_DATA_ENTRIES], } @@ -43,6 +44,7 @@ impl RRA { flags, resolution, last_update: 0, last_count: 0, + counter_value: f64::NAN, data: [f64::NAN; RRD_DATA_ENTRIES], } } @@ -63,7 +65,7 @@ impl RRA { } } - fn compute_new_value(&mut self, epoch: u64, value: f64) { + fn compute_new_value(&mut self, epoch: u64, mut value: f64) { let reso = self.resolution; let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; let last_index = ((self.last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize; @@ -77,7 +79,20 @@ impl RRA { self.last_count = 0; } - let new_count = self.last_count + 1; // fixme: check overflow? + let new_count = if self.last_count < u64::MAX { + self.last_count + 1 + } else { + u64::MAX // should never happen + }; + + if self.flags.contains(RRAFlags::DST_DERIVE) { + let diff = if self.counter_value.is_nan() { 0.0 } else { value - self.counter_value }; + self.counter_value = value; + + value = diff/(reso as f64); + if !last_value.is_nan() { value += last_value }; + } + if self.last_count == 0 { self.data[index] = value; self.last_count = 1; From d261516f2da1d8d55c021dce6bb48636050f5544 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 25 May 2020 07:02:04 +0200 Subject: [PATCH 010/111] src/rrd/rrd.rs: correctly compute derived values use f64 for time. --- src/rrd/cache.rs | 6 ++-- src/rrd/rrd.rs | 72 ++++++++++++++++++++++++++---------------------- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 4e61c949..0a9350a2 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -35,9 +35,9 @@ pub fn create_rrdb_dir() -> Result<(), Error> { Ok(()) } -fn now() -> Result { - let epoch = SystemTime::now().duration_since(UNIX_EPOCH)?; - Ok(epoch.as_secs()) +fn now() -> Result { + let time = SystemTime::now().duration_since(UNIX_EPOCH)?; + Ok(time.as_secs_f64()) } pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 002b442a..cf6c64c7 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -32,7 +32,7 @@ pub enum DST { struct RRA { flags: RRAFlags, resolution: u64, - last_update: u64, + last_update: f64, last_count: u64, counter_value: f64, // used for derive/counters data: [f64; RRD_DATA_ENTRIES], @@ -42,18 +42,21 @@ impl RRA { fn new(flags: RRAFlags, resolution: u64) -> Self { Self { flags, resolution, - last_update: 0, + last_update: 0.0, last_count: 0, counter_value: f64::NAN, data: [f64::NAN; RRD_DATA_ENTRIES], } } - fn delete_old(&mut self, epoch: u64) { + fn delete_old(&mut self, time: f64) { + let epoch = time as u64; + let last_update = self.last_update as u64; let reso = self.resolution; + let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; let min_time = (min_time/reso + 1)*reso; - let mut t = self.last_update - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last_update - (RRD_DATA_ENTRIES as u64)*reso; let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; for _ in 0..RRD_DATA_ENTRIES { t += reso; index = (index + 1) % RRD_DATA_ENTRIES; @@ -65,12 +68,23 @@ impl RRA { } } - fn compute_new_value(&mut self, epoch: u64, mut value: f64) { + fn compute_new_value(&mut self, time: f64, mut value: f64) { + let epoch = time as u64; + let time_diff = time - self.last_update; + let last_update = self.last_update as u64; let reso = self.resolution; - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - let last_index = ((self.last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - if (epoch - self.last_update) > reso || index != last_index { + // derive counter value + if self.flags.contains(RRAFlags::DST_DERIVE) { + let diff = if self.counter_value.is_nan() { 0.0 } else { value - self.counter_value }; + self.counter_value = value; + value = diff/time_diff; + } + + let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + let last_index = ((last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + + if (epoch - (last_update as u64)) > reso || index != last_index { self.last_count = 0; } @@ -85,14 +99,6 @@ impl RRA { u64::MAX // should never happen }; - if self.flags.contains(RRAFlags::DST_DERIVE) { - let diff = if self.counter_value.is_nan() { 0.0 } else { value - self.counter_value }; - self.counter_value = value; - - value = diff/(reso as f64); - if !last_value.is_nan() { value += last_value }; - } - if self.last_count == 0 { self.data[index] = value; self.last_count = 1; @@ -109,11 +115,11 @@ impl RRA { self.data[index] = new_value; self.last_count = new_count; } - self.last_update = epoch; + self.last_update = time; } - fn update(&mut self, epoch: u64, value: f64) { - if epoch < self.last_update { + fn update(&mut self, epoch: f64, value: f64) { + if epoch <= self.last_update { eprintln!("rrdb update failed - time in past ({} < {})", epoch, self.last_update); } if value.is_nan() { @@ -195,11 +201,11 @@ impl RRD { pub fn extract_data( &self, - epoch: u64, + time: f64, timeframe: RRDTimeFrameResolution, mode: RRDMode, ) -> (u64, u64, Vec>) { - + let epoch = time as u64; let reso = timeframe as u64; let end = reso*(epoch/reso + 1); @@ -220,7 +226,7 @@ impl RRD { (RRDMode::Max, RRDTimeFrameResolution::Year) => &self.year_max, }; - let rrd_end = reso*(raa.last_update/reso); + let rrd_end = reso*((raa.last_update as u64)/reso); let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); let mut t = start; @@ -284,20 +290,20 @@ impl RRD { } - pub fn update(&mut self, epoch: u64, value: f64) { - self.hour_avg.update(epoch, value); - self.hour_max.update(epoch, value); + pub fn update(&mut self, time: f64, value: f64) { + self.hour_avg.update(time, value); + self.hour_max.update(time, value); - self.day_avg.update(epoch, value); - self.day_max.update(epoch, value); + self.day_avg.update(time, value); + self.day_max.update(time, value); - self.week_avg.update(epoch, value); - self.week_max.update(epoch, value); + self.week_avg.update(time, value); + self.week_max.update(time, value); - self.month_avg.update(epoch, value); - self.month_max.update(epoch, value); + self.month_avg.update(time, value); + self.month_max.update(time, value); - self.year_avg.update(epoch, value); - self.year_max.update(epoch, value); + self.year_avg.update(time, value); + self.year_max.update(time, value); } } From bfc2f8b7a2729d95a30d5e432c473d6b97ea724f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 25 May 2020 08:14:30 +0200 Subject: [PATCH 011/111] src/rrd/rrd.rs: implement DST_COUNTER --- src/rrd/rrd.rs | 50 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index cf6c64c7..547a197b 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -14,6 +14,7 @@ bitflags!{ // Data Source Types const DST_GAUGE = 1; const DST_DERIVE = 2; + const DST_COUNTER = 4; const DST_MASK = 255; // first 8 bits // Consolidation Functions @@ -68,19 +69,11 @@ impl RRA { } } - fn compute_new_value(&mut self, time: f64, mut value: f64) { + fn compute_new_value(&mut self, time: f64, value: f64) { let epoch = time as u64; - let time_diff = time - self.last_update; let last_update = self.last_update as u64; let reso = self.resolution; - // derive counter value - if self.flags.contains(RRAFlags::DST_DERIVE) { - let diff = if self.counter_value.is_nan() { 0.0 } else { value - self.counter_value }; - self.counter_value = value; - value = diff/time_diff; - } - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; let last_index = ((last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize; @@ -118,17 +111,46 @@ impl RRA { self.last_update = time; } - fn update(&mut self, epoch: f64, value: f64) { - if epoch <= self.last_update { - eprintln!("rrdb update failed - time in past ({} < {})", epoch, self.last_update); + fn update(&mut self, time: f64, mut value: f64) { + + if time <= self.last_update { + eprintln!("rrdb update failed - time in past ({} < {})", time, self.last_update); } + if value.is_nan() { eprintln!("rrdb update failed - new value is NAN"); return; } - self.delete_old(epoch); - self.compute_new_value(epoch, value); + // derive counter value + if self.flags.intersects(RRAFlags::DST_DERIVE | RRAFlags::DST_COUNTER) { + let time_diff = time - self.last_update; + let diff = if self.counter_value.is_nan() { + 0.0 + } else { + if self.flags.contains(RRAFlags::DST_COUNTER) { // check for overflow + if value < 0.0 { + eprintln!("rrdb update failed - got negative value for counter"); + return; + } + // Note: We do not try automatic overflow corrections + if value < self.counter_value { // overflow or counter reset + self.counter_value = value; + eprintln!("rrdb update failed - conter overflow/reset detected"); + return; + } else { + value - self.counter_value + } + } else { + value - self.counter_value + } + }; + self.counter_value = value; + value = diff/time_diff; + } + + self.delete_old(time); + self.compute_new_value(time, value); } } From 207d0c77147cc8b21ad977be1c73aa63bbfbc1db Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 25 May 2020 09:21:54 +0200 Subject: [PATCH 012/111] src/rrd/rrd.rs: store/verify magic number --- src/rrd/rrd.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 547a197b..87c435c9 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -7,6 +7,9 @@ use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; pub const RRD_DATA_ENTRIES: usize = 70; +// openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; +pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; + use bitflags::bitflags; bitflags!{ @@ -157,6 +160,7 @@ impl RRA { #[repr(C)] // Note: Avoid alignment problems by using 8byte types only pub struct RRD { + magic: [u8; 8], hour_avg: RRA, hour_max: RRA, day_avg: RRA, @@ -178,6 +182,7 @@ impl RRD { }; Self { + magic: PROXMOX_RRD_MAGIC_1_0, hour_avg: RRA::new( flags | RRAFlags::CF_AVERAGE, RRDTimeFrameResolution::Hour as u64, @@ -282,6 +287,10 @@ impl RRD { raw.read_exact(rrd_slice)?; } + if rrd.magic != PROXMOX_RRD_MAGIC_1_0 { + bail!("RRD::from_raw failed - wrong magic number"); + } + Ok(rrd) } From 9ce21e005dd5a020814746ecc5a618b7266d7aa5 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 25 May 2020 10:18:53 +0200 Subject: [PATCH 013/111] src/rrd/cache.rs: display/log error when RRD load fails --- src/rrd/cache.rs | 7 ++++++- src/rrd/rrd.rs | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 0a9350a2..0a39878b 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -56,7 +56,12 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { } else { let mut rrd = match RRD::load(&path) { Ok(rrd) => rrd, - Err(_) => RRD::new(dst), + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + eprintln!("overwriting old RRD file, because of load error: {}", err); + } + RRD::new(dst) + }, }; rrd.update(now, value); rrd.save(&path)?; diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 87c435c9..dbfb98aa 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -1,7 +1,7 @@ use std::io::Read; use std::path::Path; -use anyhow::{bail, Error}; +use anyhow::Error; use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; @@ -275,10 +275,11 @@ impl RRD { (start, reso, list.into()) } - pub fn from_raw(mut raw: &[u8]) -> Result { + pub fn from_raw(mut raw: &[u8]) -> Result { let expected_len = std::mem::size_of::(); if raw.len() != expected_len { - bail!("RRD::from_raw failed - wrong data size ({} != {})", raw.len(), expected_len); + let msg = format!("wrong data size ({} != {})", raw.len(), expected_len); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); } let mut rrd: RRD = unsafe { std::mem::zeroed() }; @@ -288,15 +289,21 @@ impl RRD { } if rrd.magic != PROXMOX_RRD_MAGIC_1_0 { - bail!("RRD::from_raw failed - wrong magic number"); + let msg = format!("wrong magic number"); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); } Ok(rrd) } - pub fn load(filename: &Path) -> Result { - let raw = proxmox::tools::fs::file_get_contents(filename)?; - Self::from_raw(&raw) + pub fn load(path: &Path) -> Result { + proxmox::try_block!({ + let raw = std::fs::read(path)?; + Self::from_raw(&raw) + }).map_err(|err| { + let msg = format!("RRD load {:?} failed - {}", path, err); + std::io::Error::new(std::io::ErrorKind::Other, msg) + }) } pub fn save(&self, filename: &Path) -> Result<(), Error> { From a76b0e3c09b3bad965b8b06bc95d1464b82824b8 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 25 May 2020 10:30:04 +0200 Subject: [PATCH 014/111] src/rrd/rrd.rs: do not wrap error and return ErrorKind::NotFound --- src/rrd/cache.rs | 2 +- src/rrd/rrd.rs | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 0a39878b..a89d0615 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -58,7 +58,7 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { Ok(rrd) => rrd, Err(err) => { if err.kind() != std::io::ErrorKind::NotFound { - eprintln!("overwriting old RRD file, because of load error: {}", err); + eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err); } RRD::new(dst) }, diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index dbfb98aa..8f542b52 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -297,13 +297,8 @@ impl RRD { } pub fn load(path: &Path) -> Result { - proxmox::try_block!({ - let raw = std::fs::read(path)?; - Self::from_raw(&raw) - }).map_err(|err| { - let msg = format!("RRD load {:?} failed - {}", path, err); - std::io::Error::new(std::io::ErrorKind::Other, msg) - }) + let raw = std::fs::read(path)?; + Self::from_raw(&raw) } pub fn save(&self, filename: &Path) -> Result<(), Error> { From 1fa3341d68728fe38ffc12aeda579ab923b409ae Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 29 May 2020 09:16:13 +0200 Subject: [PATCH 015/111] rrd: reduce io by saving data only once a minute --- src/rrd/cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index a89d0615..33e99935 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -40,7 +40,7 @@ fn now() -> Result { Ok(time.as_secs_f64()) } -pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { +pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result<(), Error> { let mut path = PathBuf::from(PBS_RRD_BASEDIR); path.push(rel_path); @@ -52,7 +52,7 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { if let Some(rrd) = map.get_mut(rel_path) { rrd.update(now, value); - rrd.save(&path)?; + if save { rrd.save(&path)?; } } else { let mut rrd = match RRD::load(&path) { Ok(rrd) => rrd, @@ -64,7 +64,7 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST) -> Result<(), Error> { }, }; rrd.update(now, value); - rrd.save(&path)?; + if save { rrd.save(&path)?; } map.insert(rel_path.into(), rrd); } From 20ada7a08cd640aa6598966a56e250df3e7808bf Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Tue, 9 Jun 2020 10:01:11 +0200 Subject: [PATCH 016/111] rrd: add 'extract_lists' this is an interface to simply get the Vec> out of rrd without going through serde values we return a list of timestamps and a HashMap with the lists we could find (otherwise it is not in the map) if no lists could be extracted, the time list is also empty Signed-off-by: Dominik Csapak --- src/rrd/cache.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 33e99935..b2ad09cb 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -71,6 +71,43 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result< Ok(()) } +/// extracts the lists of the given items and a list of timestamps +pub fn extract_lists( + base: &str, + items: &[&str], + timeframe: RRDTimeFrameResolution, + mode: RRDMode, +) -> Result<(Vec, HashMap>>), Error> { + + let now = now()?; + + let map = RRD_CACHE.read().unwrap(); + + let mut result = HashMap::new(); + + let mut times = Vec::new(); + + for name in items.iter() { + let rrd = match map.get(&format!("{}/{}", base, name)) { + Some(rrd) => rrd, + None => continue, + }; + let (start, reso, list) = rrd.extract_data(now, timeframe, mode); + + result.insert(name.to_string(), list); + + if times.len() == 0 { + let mut t = start; + for _ in 0..RRD_DATA_ENTRIES { + times.push(t); + t += reso; + } + } + } + + Ok((times, result)) +} + pub fn extract_data( base: &str, items: &[&str], From 9a860821e4964037b528c2f09eba050a24afff39 Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Wed, 10 Jun 2020 12:02:56 +0200 Subject: [PATCH 017/111] refactor time functions to tools Signed-off-by: Dominik Csapak Signed-off-by: Wolfgang Bumiller --- src/rrd/cache.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index b2ad09cb..90d2c9d1 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -1,4 +1,3 @@ -use std::time::{SystemTime, UNIX_EPOCH}; use std::path::PathBuf; use std::collections::HashMap; use std::sync::{RwLock}; @@ -10,6 +9,7 @@ use serde_json::{json, Value}; use proxmox::tools::fs::{create_path, CreateOptions}; use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; +use crate::tools::epoch_now_f64; use super::*; @@ -35,11 +35,6 @@ pub fn create_rrdb_dir() -> Result<(), Error> { Ok(()) } -fn now() -> Result { - let time = SystemTime::now().duration_since(UNIX_EPOCH)?; - Ok(time.as_secs_f64()) -} - pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result<(), Error> { let mut path = PathBuf::from(PBS_RRD_BASEDIR); @@ -48,7 +43,7 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result< std::fs::create_dir_all(path.parent().unwrap())?; let mut map = RRD_CACHE.write().unwrap(); - let now = now()?; + let now = epoch_now_f64()?; if let Some(rrd) = map.get_mut(rel_path) { rrd.update(now, value); @@ -115,7 +110,7 @@ pub fn extract_data( mode: RRDMode, ) -> Result { - let now = now()?; + let now = epoch_now_f64()?; let map = RRD_CACHE.read().unwrap(); From 11cb8cd0081e3ed8cba0297689066a890ff46f40 Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Wed, 10 Jun 2020 12:02:57 +0200 Subject: [PATCH 018/111] rrd: move creation of serde value into api there is now a 'extract_cached_data' which just returns the data of the specified field, and an api function that converts a list of fields to the correct serde value this way we do not have to create a serde value in rrd/cache.rs (makes for a better interface) Signed-off-by: Dominik Csapak Signed-off-by: Wolfgang Bumiller --- src/rrd/cache.rs | 38 +++++++------------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index 90d2c9d1..ce3c551c 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -4,7 +4,6 @@ use std::sync::{RwLock}; use anyhow::{format_err, Error}; use lazy_static::lazy_static; -use serde_json::{json, Value}; use proxmox::tools::fs::{create_path, CreateOptions}; @@ -103,41 +102,18 @@ pub fn extract_lists( Ok((times, result)) } -pub fn extract_data( +pub fn extract_cached_data( base: &str, - items: &[&str], + name: &str, + now: f64, timeframe: RRDTimeFrameResolution, mode: RRDMode, -) -> Result { - - let now = epoch_now_f64()?; +) -> Option<(u64, u64, Vec>)> { let map = RRD_CACHE.read().unwrap(); - let mut result = Vec::new(); - - for name in items.iter() { - let rrd = match map.get(&format!("{}/{}", base, name)) { - Some(rrd) => rrd, - None => continue, - }; - let (start, reso, list) = rrd.extract_data(now, timeframe, mode); - let mut t = start; - for index in 0..RRD_DATA_ENTRIES { - if result.len() <= index { - if let Some(value) = list[index] { - result.push(json!({ "time": t, *name: value })); - } else { - result.push(json!({ "time": t })); - } - } else { - if let Some(value) = list[index] { - result[index][name] = value.into(); - } - } - t += reso; - } + match map.get(&format!("{}/{}", base, name)) { + Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), + None => None, } - - Ok(result.into()) } From 50fa6045b3f62bf608e91df8f8220ffb52851248 Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Wed, 10 Jun 2020 12:02:58 +0200 Subject: [PATCH 019/111] api2/status: use new rrd::extract_cached_data and drop the now unused extract_lists function this also fixes a bug, where we did not add the datastore to the list at all when there was no rrd data Signed-off-by: Dominik Csapak Signed-off-by: Wolfgang Bumiller --- src/rrd/cache.rs | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index ce3c551c..f08d6c9e 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -65,43 +65,6 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result< Ok(()) } -/// extracts the lists of the given items and a list of timestamps -pub fn extract_lists( - base: &str, - items: &[&str], - timeframe: RRDTimeFrameResolution, - mode: RRDMode, -) -> Result<(Vec, HashMap>>), Error> { - - let now = now()?; - - let map = RRD_CACHE.read().unwrap(); - - let mut result = HashMap::new(); - - let mut times = Vec::new(); - - for name in items.iter() { - let rrd = match map.get(&format!("{}/{}", base, name)) { - Some(rrd) => rrd, - None => continue, - }; - let (start, reso, list) = rrd.extract_data(now, timeframe, mode); - - result.insert(name.to_string(), list); - - if times.len() == 0 { - let mut t = start; - for _ in 0..RRD_DATA_ENTRIES { - times.push(t); - t += reso; - } - } - } - - Ok((times, result)) -} - pub fn extract_cached_data( base: &str, name: &str, From 110ceff08cf9f2c615b829c98f853f19b48a4a6e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 12 Sep 2020 15:10:47 +0200 Subject: [PATCH 020/111] avoid chrono dependency, depend on proxmox 0.3.8 - remove chrono dependency - depend on proxmox 0.3.8 - remove epoch_now, epoch_now_u64 and epoch_now_f64 - remove tm_editor (moved to proxmox crate) - use new helpers from proxmox 0.3.8 * epoch_i64 and epoch_f64 * parse_rfc3339 * epoch_to_rfc3339_utc * strftime_local - BackupDir changes: * store epoch and rfc3339 string instead of DateTime * backup_time_to_string now return a Result * remove unnecessary TryFrom<(BackupGroup, i64)> for BackupDir - DynamicIndexHeader: change ctime to i64 - FixedIndexHeader: change ctime to i64 --- src/rrd/cache.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index f08d6c9e..e5e3fe09 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -8,7 +8,6 @@ use lazy_static::lazy_static; use proxmox::tools::fs::{create_path, CreateOptions}; use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; -use crate::tools::epoch_now_f64; use super::*; @@ -42,7 +41,7 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result< std::fs::create_dir_all(path.parent().unwrap())?; let mut map = RRD_CACHE.write().unwrap(); - let now = epoch_now_f64()?; + let now = proxmox::tools::time::epoch_f64(); if let Some(rrd) = map.get_mut(rel_path) { rrd.update(now, value); From 6eff0b289eff56df6b1a66d7324ca0d0e09d4cfa Mon Sep 17 00:00:00 2001 From: Stefan Reiter Date: Thu, 1 Oct 2020 11:40:44 +0200 Subject: [PATCH 021/111] rrd: fix integer underflow Causes a panic if last_update is smaller than RRD_DATA_ENTRIES*reso, which (I believe) can happen when inserting the first value for a DB. Clamp the value to 0 in that case. Signed-off-by: Stefan Reiter --- src/rrd/rrd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 8f542b52..86d6b50b 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -60,7 +60,7 @@ impl RRA { let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; let min_time = (min_time/reso + 1)*reso; - let mut t = last_update - (RRD_DATA_ENTRIES as u64)*reso; + let mut t = last_update.saturating_sub((RRD_DATA_ENTRIES as u64)*reso); let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; for _ in 0..RRD_DATA_ENTRIES { t += reso; index = (index + 1) % RRD_DATA_ENTRIES; From 4d8bd987a4c54d2ef2266b6c67dc5d1d836fc0e1 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 14 Oct 2020 11:18:26 +0200 Subject: [PATCH 022/111] clippy fixups Signed-off-by: Wolfgang Bumiller --- src/rrd/rrd.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 86d6b50b..c8b16471 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -272,7 +272,7 @@ impl RRD { t += reso; index = (index + 1) % RRD_DATA_ENTRIES; } - (start, reso, list.into()) + (start, reso, list) } pub fn from_raw(mut raw: &[u8]) -> Result { @@ -289,7 +289,7 @@ impl RRD { } if rrd.magic != PROXMOX_RRD_MAGIC_1_0 { - let msg = format!("wrong magic number"); + let msg = "wrong magic number".to_string(); return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); } From 655ceac1c6249ec79844af11ce5fbd0ec1a6475c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Fri, 15 Jan 2021 14:10:24 +0100 Subject: [PATCH 023/111] clippy: collapse/rework nested ifs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no semantic changes (intended). Signed-off-by: Fabian Grünbichler --- src/rrd/rrd.rs | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index c8b16471..37bdf3b9 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -128,25 +128,20 @@ impl RRA { // derive counter value if self.flags.intersects(RRAFlags::DST_DERIVE | RRAFlags::DST_COUNTER) { let time_diff = time - self.last_update; + let is_counter = self.flags.contains(RRAFlags::DST_COUNTER); + let diff = if self.counter_value.is_nan() { 0.0 + } else if is_counter && value < 0.0 { + eprintln!("rrdb update failed - got negative value for counter"); + return; + } else if is_counter && value < self.counter_value { + // Note: We do not try automatic overflow corrections + self.counter_value = value; + eprintln!("rrdb update failed - conter overflow/reset detected"); + return; } else { - if self.flags.contains(RRAFlags::DST_COUNTER) { // check for overflow - if value < 0.0 { - eprintln!("rrdb update failed - got negative value for counter"); - return; - } - // Note: We do not try automatic overflow corrections - if value < self.counter_value { // overflow or counter reset - self.counter_value = value; - eprintln!("rrdb update failed - conter overflow/reset detected"); - return; - } else { - value - self.counter_value - } - } else { - value - self.counter_value - } + value - self.counter_value }; self.counter_value = value; value = diff/time_diff; From c94f367decf1c20acf4782d8efc0f0f5487da616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Mon, 25 Jan 2021 14:43:00 +0100 Subject: [PATCH 024/111] clippy: more misc fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- src/rrd/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rrd/mod.rs b/src/rrd/mod.rs index c09efebf..03e4c9de 100644 --- a/src/rrd/mod.rs +++ b/src/rrd/mod.rs @@ -1,3 +1,4 @@ +#[allow(clippy::module_inception)] mod rrd; pub use rrd::*; mod cache; From 80f7bf38225ac2d1d6da33d2a40dc0c7a6bfb757 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 2 Sep 2021 12:47:11 +0200 Subject: [PATCH 025/111] start new pbs-config workspace moved src/config/domains.rs --- src/rrd/cache.rs | 2 +- src/rrd/rrd.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index e5e3fe09..d593ffb5 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -22,7 +22,7 @@ lazy_static!{ /// Create rrdd stat dir with correct permission pub fn create_rrdb_dir() -> Result<(), Error> { - let backup_user = crate::backup::backup_user()?; + let backup_user = pbs_config::backup_user()?; let opts = CreateOptions::new() .owner(backup_user.uid) .group(backup_user.gid); diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index 37bdf3b9..b298f0ad 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -303,7 +303,7 @@ impl RRD { std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) }; - let backup_user = crate::backup::backup_user()?; + let backup_user = pbs_config::backup_user()?; let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); // set the correct owner/group/permissions while saving file // owner(rw) = backup, group(r)= backup From a4e56bff60531e3593e489ce019ad2aa3a5a3e5c Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 10 Sep 2021 12:25:32 +0200 Subject: [PATCH 026/111] more api type cleanups: avoid re-exports --- src/rrd/cache.rs | 2 +- src/rrd/rrd.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs index d593ffb5..d6b79ac0 100644 --- a/src/rrd/cache.rs +++ b/src/rrd/cache.rs @@ -7,7 +7,7 @@ use lazy_static::lazy_static; use proxmox::tools::fs::{create_path, CreateOptions}; -use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; +use pbs_api_types::{RRDMode, RRDTimeFrameResolution}; use super::*; diff --git a/src/rrd/rrd.rs b/src/rrd/rrd.rs index b298f0ad..b1780307 100644 --- a/src/rrd/rrd.rs +++ b/src/rrd/rrd.rs @@ -3,7 +3,7 @@ use std::path::Path; use anyhow::Error; -use crate::api2::types::{RRDMode, RRDTimeFrameResolution}; +use pbs_api_types::{RRDMode, RRDTimeFrameResolution}; pub const RRD_DATA_ENTRIES: usize = 70; From 8d1a9d2ec6ab2c4dfacdb1c8882fa64f5fbf3ac4 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 07:06:17 +0200 Subject: [PATCH 027/111] move RRD code into proxmox-rrd crate --- proxmox-rrd/Cargo.toml | 13 ++++ proxmox-rrd/src/cache.rs | 111 ++++++++++++++++++++++++++++ proxmox-rrd/src/lib.rs | 37 ++++++++++ {src/rrd => proxmox-rrd/src}/rrd.rs | 28 +++---- src/rrd/cache.rs | 81 -------------------- src/rrd/mod.rs | 5 -- 6 files changed, 171 insertions(+), 104 deletions(-) create mode 100644 proxmox-rrd/Cargo.toml create mode 100644 proxmox-rrd/src/cache.rs create mode 100644 proxmox-rrd/src/lib.rs rename {src/rrd => proxmox-rrd/src}/rrd.rs (93%) delete mode 100644 src/rrd/cache.rs delete mode 100644 src/rrd/mod.rs diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml new file mode 100644 index 00000000..c2b2d213 --- /dev/null +++ b/proxmox-rrd/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "proxmox-rrd" +version = "0.1.0" +authors = ["Dietmar Maurer "] +edition = "2018" +description = "Simple RRD database implementation." + +[dependencies] +anyhow = "1.0" +bitflags = "1.2.1" +serde = { version = "1.0", features = [] } + +proxmox = { version = "0.13.5", features = ["api-macro"] } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs new file mode 100644 index 00000000..c87e49fd --- /dev/null +++ b/proxmox-rrd/src/cache.rs @@ -0,0 +1,111 @@ +use std::path::{Path, PathBuf}; +use std::collections::HashMap; +use std::sync::{RwLock}; + +use anyhow::{format_err, Error}; + +use proxmox::tools::fs::{create_path, CreateOptions}; + +use crate::{RRDMode, RRDTimeFrameResolution}; + +use super::*; + +/// RRD cache - keep RRD data in RAM, but write updates to disk +/// +/// This cache is designed to run as single instance (no concurrent +/// access from other processes). +pub struct RRDCache { + basedir: PathBuf, + file_options: CreateOptions, + dir_options: CreateOptions, + cache: RwLock>, +} + +impl RRDCache { + + /// Creates a new instance + pub fn new>( + basedir: P, + file_options: Option, + dir_options: Option, + ) -> Self { + let basedir = basedir.as_ref().to_owned(); + Self { + basedir, + file_options: file_options.unwrap_or_else(|| CreateOptions::new()), + dir_options: dir_options.unwrap_or_else(|| CreateOptions::new()), + cache: RwLock::new(HashMap::new()), + } + } +} + +impl RRDCache { + + /// Create rrdd stat dir with correct permission + pub fn create_rrdb_dir(&self) -> Result<(), Error> { + + create_path(&self.basedir, Some(self.dir_options.clone()), Some(self.file_options.clone())) + .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; + + Ok(()) + } + + /// Update data in RAM and write file back to disk (if `save` is set) + pub fn update_value( + &self, + rel_path: &str, + value: f64, + dst: DST, + save: bool, + ) -> Result<(), Error> { + + let mut path = self.basedir.clone(); + path.push(rel_path); + + std::fs::create_dir_all(path.parent().unwrap())?; // fixme?? + + let mut map = self.cache.write().unwrap(); + let now = proxmox::tools::time::epoch_f64(); + + if let Some(rrd) = map.get_mut(rel_path) { + rrd.update(now, value); + if save { rrd.save(&path, self.file_options.clone())?; } + } else { + let mut rrd = match RRD::load(&path) { + Ok(rrd) => rrd, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err); + } + RRD::new(dst) + }, + }; + rrd.update(now, value); + if save { + rrd.save(&path, self.file_options.clone())?; + } + map.insert(rel_path.into(), rrd); + } + + Ok(()) + } + + /// Extract data from cached RRD + pub fn extract_cached_data( + &self, + base: &str, + name: &str, + now: f64, + timeframe: RRDTimeFrameResolution, + mode: RRDMode, + ) -> Option<(u64, u64, Vec>)> { + + let map = self.cache.read().unwrap(); + + match map.get(&format!("{}/{}", base, name)) { + Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), + None => None, + } + } + +} diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs new file mode 100644 index 00000000..d6ba54c9 --- /dev/null +++ b/proxmox-rrd/src/lib.rs @@ -0,0 +1,37 @@ +mod rrd; +pub use rrd::*; + +mod cache; +pub use cache::*; + +use serde::{Deserialize, Serialize}; +use proxmox::api::api; + +#[api()] +#[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +/// RRD consolidation mode +pub enum RRDMode { + /// Maximum + Max, + /// Average + Average, +} + +#[api()] +#[repr(u64)] +#[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +/// RRD time frame resolution +pub enum RRDTimeFrameResolution { + /// 1 min => last 70 minutes + Hour = 60, + /// 30 min => last 35 hours + Day = 60*30, + /// 3 hours => about 8 days + Week = 60*180, + /// 12 hours => last 35 days + Month = 60*720, + /// 1 week => last 490 days + Year = 60*10080, +} diff --git a/src/rrd/rrd.rs b/proxmox-rrd/src/rrd.rs similarity index 93% rename from src/rrd/rrd.rs rename to proxmox-rrd/src/rrd.rs index b1780307..19a6deba 100644 --- a/src/rrd/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -3,17 +3,21 @@ use std::path::Path; use anyhow::Error; -use pbs_api_types::{RRDMode, RRDTimeFrameResolution}; +use proxmox::tools::{fs::replace_file, fs::CreateOptions}; +use crate::{RRDMode, RRDTimeFrameResolution}; + +/// The number of data entries per RRA pub const RRD_DATA_ENTRIES: usize = 70; +/// Proxmox RRD file magic number // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; use bitflags::bitflags; bitflags!{ - pub struct RRAFlags: u64 { + struct RRAFlags: u64 { // Data Source Types const DST_GAUGE = 1; const DST_DERIVE = 2; @@ -27,6 +31,7 @@ bitflags!{ } } +/// RRD data source tyoe pub enum DST { Gauge, Derive, @@ -227,6 +232,7 @@ impl RRD { timeframe: RRDTimeFrameResolution, mode: RRDMode, ) -> (u64, u64, Vec>) { + let epoch = time as u64; let reso = timeframe as u64; @@ -296,25 +302,11 @@ impl RRD { Self::from_raw(&raw) } - pub fn save(&self, filename: &Path) -> Result<(), Error> { - use proxmox::tools::{fs::replace_file, fs::CreateOptions}; - + pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> { let rrd_slice = unsafe { std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) }; - - let backup_user = pbs_config::backup_user()?; - let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); - // set the correct owner/group/permissions while saving file - // owner(rw) = backup, group(r)= backup - let options = CreateOptions::new() - .perm(mode) - .owner(backup_user.uid) - .group(backup_user.gid); - - replace_file(filename, rrd_slice, options)?; - - Ok(()) + replace_file(filename, rrd_slice, options) } diff --git a/src/rrd/cache.rs b/src/rrd/cache.rs deleted file mode 100644 index d6b79ac0..00000000 --- a/src/rrd/cache.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::path::PathBuf; -use std::collections::HashMap; -use std::sync::{RwLock}; - -use anyhow::{format_err, Error}; -use lazy_static::lazy_static; - -use proxmox::tools::fs::{create_path, CreateOptions}; - -use pbs_api_types::{RRDMode, RRDTimeFrameResolution}; - -use super::*; - -const PBS_RRD_BASEDIR: &str = "/var/lib/proxmox-backup/rrdb"; - -lazy_static!{ - static ref RRD_CACHE: RwLock> = { - RwLock::new(HashMap::new()) - }; -} - -/// Create rrdd stat dir with correct permission -pub fn create_rrdb_dir() -> Result<(), Error> { - - let backup_user = pbs_config::backup_user()?; - let opts = CreateOptions::new() - .owner(backup_user.uid) - .group(backup_user.gid); - - create_path(PBS_RRD_BASEDIR, None, Some(opts)) - .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; - - Ok(()) -} - -pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result<(), Error> { - - let mut path = PathBuf::from(PBS_RRD_BASEDIR); - path.push(rel_path); - - std::fs::create_dir_all(path.parent().unwrap())?; - - let mut map = RRD_CACHE.write().unwrap(); - let now = proxmox::tools::time::epoch_f64(); - - if let Some(rrd) = map.get_mut(rel_path) { - rrd.update(now, value); - if save { rrd.save(&path)?; } - } else { - let mut rrd = match RRD::load(&path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - RRD::new(dst) - }, - }; - rrd.update(now, value); - if save { rrd.save(&path)?; } - map.insert(rel_path.into(), rrd); - } - - Ok(()) -} - -pub fn extract_cached_data( - base: &str, - name: &str, - now: f64, - timeframe: RRDTimeFrameResolution, - mode: RRDMode, -) -> Option<(u64, u64, Vec>)> { - - let map = RRD_CACHE.read().unwrap(); - - match map.get(&format!("{}/{}", base, name)) { - Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), - None => None, - } -} diff --git a/src/rrd/mod.rs b/src/rrd/mod.rs deleted file mode 100644 index 03e4c9de..00000000 --- a/src/rrd/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[allow(clippy::module_inception)] -mod rrd; -pub use rrd::*; -mod cache; -pub use cache::*; From ac17698e4a881f71808028f4c2d06eaf9b1cdbb2 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 08:37:14 +0200 Subject: [PATCH 028/111] proxmox-rrd: use create_path instead of std::fs::create_dir_all To ensure correct file ownership. --- proxmox-rrd/src/cache.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index c87e49fd..c0e46d35 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -37,9 +37,6 @@ impl RRDCache { cache: RwLock::new(HashMap::new()), } } -} - -impl RRDCache { /// Create rrdd stat dir with correct permission pub fn create_rrdb_dir(&self) -> Result<(), Error> { @@ -62,7 +59,7 @@ impl RRDCache { let mut path = self.basedir.clone(); path.push(rel_path); - std::fs::create_dir_all(path.parent().unwrap())?; // fixme?? + create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.file_options.clone()))?; let mut map = self.cache.write().unwrap(); let now = proxmox::tools::time::epoch_f64(); @@ -107,5 +104,4 @@ impl RRDCache { None => None, } } - } From 538e6f66f3ecb1a4b1b3bcb820c2db80a88c6f84 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 09:49:51 +0200 Subject: [PATCH 029/111] split out RRD api types into proxmox-rrd-api-types crate --- proxmox-rrd-api-types/Cargo.toml | 11 +++++++++++ proxmox-rrd-api-types/src/lib.rs | 31 +++++++++++++++++++++++++++++++ proxmox-rrd/Cargo.toml | 4 +++- proxmox-rrd/src/cache.rs | 2 +- proxmox-rrd/src/lib.rs | 32 -------------------------------- proxmox-rrd/src/rrd.rs | 2 +- 6 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 proxmox-rrd-api-types/Cargo.toml create mode 100644 proxmox-rrd-api-types/src/lib.rs diff --git a/proxmox-rrd-api-types/Cargo.toml b/proxmox-rrd-api-types/Cargo.toml new file mode 100644 index 00000000..eaae9a90 --- /dev/null +++ b/proxmox-rrd-api-types/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "proxmox-rrd-api-types" +version = "0.1.0" +authors = ["Proxmox Support Team "] +edition = "2018" +description = "API type definitions for proxmox-rrd crate." + + +[dependencies] +serde = { version = "1.0", features = [] } +proxmox = { version = "0.13.5", features = ["api-macro"] } diff --git a/proxmox-rrd-api-types/src/lib.rs b/proxmox-rrd-api-types/src/lib.rs new file mode 100644 index 00000000..4a0165ff --- /dev/null +++ b/proxmox-rrd-api-types/src/lib.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Serialize}; +use proxmox::api::api; + +#[api()] +#[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +/// RRD consolidation mode +pub enum RRDMode { + /// Maximum + Max, + /// Average + Average, +} + +#[api()] +#[repr(u64)] +#[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +/// RRD time frame resolution +pub enum RRDTimeFrameResolution { + /// 1 min => last 70 minutes + Hour = 60, + /// 30 min => last 35 hours + Day = 60*30, + /// 3 hours => about 8 days + Week = 60*180, + /// 12 hours => last 35 days + Month = 60*720, + /// 1 week => last 490 days + Year = 60*10080, +} diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index c2b2d213..e5a0e2b0 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "proxmox-rrd" version = "0.1.0" -authors = ["Dietmar Maurer "] +authors = ["Proxmox Support Team "] edition = "2018" description = "Simple RRD database implementation." @@ -11,3 +11,5 @@ bitflags = "1.2.1" serde = { version = "1.0", features = [] } proxmox = { version = "0.13.5", features = ["api-macro"] } + +proxmox-rrd-api-types = { path = "../proxmox-rrd-api-types" } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index c0e46d35..550eb30c 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -6,7 +6,7 @@ use anyhow::{format_err, Error}; use proxmox::tools::fs::{create_path, CreateOptions}; -use crate::{RRDMode, RRDTimeFrameResolution}; +use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; use super::*; diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs index d6ba54c9..446aff04 100644 --- a/proxmox-rrd/src/lib.rs +++ b/proxmox-rrd/src/lib.rs @@ -3,35 +3,3 @@ pub use rrd::*; mod cache; pub use cache::*; - -use serde::{Deserialize, Serialize}; -use proxmox::api::api; - -#[api()] -#[derive(Copy, Clone, Serialize, Deserialize)] -#[serde(rename_all = "UPPERCASE")] -/// RRD consolidation mode -pub enum RRDMode { - /// Maximum - Max, - /// Average - Average, -} - -#[api()] -#[repr(u64)] -#[derive(Copy, Clone, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -/// RRD time frame resolution -pub enum RRDTimeFrameResolution { - /// 1 min => last 70 minutes - Hour = 60, - /// 30 min => last 35 hours - Day = 60*30, - /// 3 hours => about 8 days - Week = 60*180, - /// 12 hours => last 35 days - Month = 60*720, - /// 1 week => last 490 days - Year = 60*10080, -} diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 19a6deba..526d36b3 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -5,7 +5,7 @@ use anyhow::Error; use proxmox::tools::{fs::replace_file, fs::CreateOptions}; -use crate::{RRDMode, RRDTimeFrameResolution}; +use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; /// The number of data entries per RRA pub const RRD_DATA_ENTRIES: usize = 70; From 54f7a80f9721d5c07c549b064e422c72d95bf4db Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 10:55:46 +0200 Subject: [PATCH 030/111] proxmox-rrd: remove serde dependency --- proxmox-rrd/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index e5a0e2b0..f273bbe3 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -8,7 +8,6 @@ description = "Simple RRD database implementation." [dependencies] anyhow = "1.0" bitflags = "1.2.1" -serde = { version = "1.0", features = [] } proxmox = { version = "0.13.5", features = ["api-macro"] } From 881d8f85eaed0984f8f4462fcf3ca833906fb910 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 12:19:54 +0200 Subject: [PATCH 031/111] proxmox-rrd: improve developer docs --- proxmox-rrd/src/cache.rs | 2 +- proxmox-rrd/src/lib.rs | 20 +++++++++- proxmox-rrd/src/rrd.rs | 81 ++++++++++++++++++++++++++-------------- 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 550eb30c..d884d01d 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -8,7 +8,7 @@ use proxmox::tools::fs::{create_path, CreateOptions}; use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; -use super::*; +use crate::{DST, rrd::RRD}; /// RRD cache - keep RRD data in RAM, but write updates to disk /// diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs index 446aff04..303cd55d 100644 --- a/proxmox-rrd/src/lib.rs +++ b/proxmox-rrd/src/lib.rs @@ -1,5 +1,21 @@ -mod rrd; -pub use rrd::*; +//! # Simple Round Robin Database files with fixed format +//! +//! ## Features +//! +//! * One file stores a single data source +//! * Small/constant file size (6008 bytes) +//! * Stores avarage and maximum values +//! * Stores data for different time resolution ([RRDTimeFrameResolution](proxmox_rrd_api_types::RRDTimeFrameResolution)) + +pub mod rrd; mod cache; pub use cache::*; + +/// RRD data source tyoe +pub enum DST { + /// Gauge values are stored unmodified. + Gauge, + /// Stores the difference to the previous value. + Derive, +} diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 526d36b3..8ea182c7 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -1,7 +1,10 @@ +//! # Round Robin Database file format + use std::io::Read; use std::path::Path; use anyhow::Error; +use bitflags::bitflags; use proxmox::tools::{fs::replace_file, fs::CreateOptions}; @@ -14,10 +17,11 @@ pub const RRD_DATA_ENTRIES: usize = 70; // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; -use bitflags::bitflags; +use crate::DST; bitflags!{ - struct RRAFlags: u64 { + /// Flags to specify the data soure type and consolidation function + pub struct RRAFlags: u64 { // Data Source Types const DST_GAUGE = 1; const DST_DERIVE = 2; @@ -31,20 +35,24 @@ bitflags!{ } } -/// RRD data source tyoe -pub enum DST { - Gauge, - Derive, -} - +/// Round Robin Archive with [RRD_DATA_ENTRIES] data slots. +/// +/// This data structure is used inside [RRD] and directly written to the +/// RRD files. #[repr(C)] -struct RRA { - flags: RRAFlags, - resolution: u64, - last_update: f64, - last_count: u64, - counter_value: f64, // used for derive/counters - data: [f64; RRD_DATA_ENTRIES], +pub struct RRA { + /// Defined the data soure type and consolidation function + pub flags: RRAFlags, + /// Resulution (seconds) from [RRDTimeFrameResolution] + pub resolution: u64, + /// Last update time (epoch) + pub last_update: f64, + /// Count values computed inside this update interval + pub last_count: u64, + /// Stores the last value, used to compute differential value for derive/counters + pub counter_value: f64, + /// Data slots + pub data: [f64; RRD_DATA_ENTRIES], } impl RRA { @@ -157,24 +165,37 @@ impl RRA { } } +/// Round Robin Database file format with fixed number of [RRA]s #[repr(C)] // Note: Avoid alignment problems by using 8byte types only pub struct RRD { - magic: [u8; 8], - hour_avg: RRA, - hour_max: RRA, - day_avg: RRA, - day_max: RRA, - week_avg: RRA, - week_max: RRA, - month_avg: RRA, - month_max: RRA, - year_avg: RRA, - year_max: RRA, + /// The magic number to identify the file type + pub magic: [u8; 8], + /// Hourly data (average values) + pub hour_avg: RRA, + /// Hourly data (maximum values) + pub hour_max: RRA, + /// Dayly data (average values) + pub day_avg: RRA, + /// Dayly data (maximum values) + pub day_max: RRA, + /// Weekly data (average values) + pub week_avg: RRA, + /// Weekly data (maximum values) + pub week_max: RRA, + /// Monthly data (average values) + pub month_avg: RRA, + /// Monthly data (maximum values) + pub month_max: RRA, + /// Yearly data (average values) + pub year_avg: RRA, + /// Yearly data (maximum values) + pub year_max: RRA, } impl RRD { + /// Create a new empty instance pub fn new(dst: DST) -> Self { let flags = match dst { DST::Gauge => RRAFlags::DST_GAUGE, @@ -226,6 +247,7 @@ impl RRD { } } + /// Extract data from the archive pub fn extract_data( &self, time: f64, @@ -276,6 +298,7 @@ impl RRD { (start, reso, list) } + /// Create instance from raw data, testing data len and magic number pub fn from_raw(mut raw: &[u8]) -> Result { let expected_len = std::mem::size_of::(); if raw.len() != expected_len { @@ -297,11 +320,13 @@ impl RRD { Ok(rrd) } + /// Load data from a file pub fn load(path: &Path) -> Result { let raw = std::fs::read(path)?; Self::from_raw(&raw) } + /// Store data into a file (atomic replace file) pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> { let rrd_slice = unsafe { std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) @@ -309,7 +334,9 @@ impl RRD { replace_file(filename, rrd_slice, options) } - + /// Update the value (in memory) + /// + /// Note: This does not call [Self::save]. pub fn update(&mut self, time: f64, value: f64) { self.hour_avg.update(time, value); self.hour_max.update(time, value); From 9c7fd3c9364d55e9527821edebc16a2779eb9111 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 18:00:37 +0200 Subject: [PATCH 032/111] proxmox-rrd: fix update (do not update) when time is in the past --- proxmox-rrd/src/rrd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 8ea182c7..4b71f67e 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -131,6 +131,7 @@ impl RRA { if time <= self.last_update { eprintln!("rrdb update failed - time in past ({} < {})", time, self.last_update); + return; } if value.is_nan() { From 5165bed8c235bf59ab8c1843ff1482dca8266986 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 6 Oct 2021 18:19:22 +0200 Subject: [PATCH 033/111] proxmox-rrd: use log crate instead of eprintln, avoid duplicate logs --- proxmox-rrd/Cargo.toml | 1 + proxmox-rrd/src/cache.rs | 2 +- proxmox-rrd/src/rrd.rs | 43 +++++++++++++++++++++++++--------------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index f273bbe3..19db5bf6 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -8,6 +8,7 @@ description = "Simple RRD database implementation." [dependencies] anyhow = "1.0" bitflags = "1.2.1" +log = "0.4" proxmox = { version = "0.13.5", features = ["api-macro"] } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index d884d01d..08e91c9c 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -72,7 +72,7 @@ impl RRDCache { Ok(rrd) => rrd, Err(err) => { if err.kind() != std::io::ErrorKind::NotFound { - eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err); + log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); } RRD::new(dst) }, diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 4b71f67e..c31572a4 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -118,7 +118,7 @@ impl RRA { (last_value*(self.last_count as f64))/(new_count as f64) + value/(new_count as f64) } else { - eprintln!("rrdb update failed - unknown CF"); + log::error!("rrdb update failed - unknown CF"); return; }; self.data[index] = new_value; @@ -127,15 +127,18 @@ impl RRA { self.last_update = time; } - fn update(&mut self, time: f64, mut value: f64) { + fn update(&mut self, time: f64, mut value: f64, log_time_in_past: &mut bool) { if time <= self.last_update { - eprintln!("rrdb update failed - time in past ({} < {})", time, self.last_update); + if *log_time_in_past { + log::warn!("rrdb update failed - time in past ({} < {})", time, self.last_update); + *log_time_in_past = false; // avoid logging this multiple times inside a RRD + } return; } if value.is_nan() { - eprintln!("rrdb update failed - new value is NAN"); + log::warn!("rrdb update failed - new value is NAN"); return; } @@ -147,12 +150,12 @@ impl RRA { let diff = if self.counter_value.is_nan() { 0.0 } else if is_counter && value < 0.0 { - eprintln!("rrdb update failed - got negative value for counter"); + log::warn!("rrdb update failed - got negative value for counter"); return; } else if is_counter && value < self.counter_value { // Note: We do not try automatic overflow corrections self.counter_value = value; - eprintln!("rrdb update failed - conter overflow/reset detected"); + log::warn!("rrdb update failed - conter overflow/reset detected"); return; } else { value - self.counter_value @@ -339,19 +342,27 @@ impl RRD { /// /// Note: This does not call [Self::save]. pub fn update(&mut self, time: f64, value: f64) { - self.hour_avg.update(time, value); - self.hour_max.update(time, value); - self.day_avg.update(time, value); - self.day_max.update(time, value); + if value.is_nan() { + log::warn!("rrdb update failed - new value is NAN"); + return; + } - self.week_avg.update(time, value); - self.week_max.update(time, value); + let mut log_time_in_past = true; - self.month_avg.update(time, value); - self.month_max.update(time, value); + self.hour_avg.update(time, value, &mut log_time_in_past); + self.hour_max.update(time, value, &mut log_time_in_past); - self.year_avg.update(time, value); - self.year_max.update(time, value); + self.day_avg.update(time, value, &mut log_time_in_past); + self.day_max.update(time, value, &mut log_time_in_past); + + self.week_avg.update(time, value, &mut log_time_in_past); + self.week_max.update(time, value, &mut log_time_in_past); + + self.month_avg.update(time, value, &mut log_time_in_past); + self.month_max.update(time, value, &mut log_time_in_past); + + self.year_avg.update(time, value, &mut log_time_in_past); + self.year_max.update(time, value, &mut log_time_in_past); } } From 9c64c09c9252bede9c7a8ba33bccab4d1811e10a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 7 Oct 2021 08:01:12 +0200 Subject: [PATCH 034/111] proxmox-rrd: cleanup error handling --- proxmox-rrd/src/rrd.rs | 62 ++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index c31572a4..f4c08909 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -3,7 +3,7 @@ use std::io::Read; use std::path::Path; -use anyhow::Error; +use anyhow::{bail, Error}; use bitflags::bitflags; use proxmox::tools::{fs::replace_file, fs::CreateOptions}; @@ -127,19 +127,15 @@ impl RRA { self.last_update = time; } - fn update(&mut self, time: f64, mut value: f64, log_time_in_past: &mut bool) { + // Note: This may update the state even in case of errors (see counter overflow) + fn update(&mut self, time: f64, mut value: f64) -> Result<(), Error> { if time <= self.last_update { - if *log_time_in_past { - log::warn!("rrdb update failed - time in past ({} < {})", time, self.last_update); - *log_time_in_past = false; // avoid logging this multiple times inside a RRD - } - return; + bail!("time in past ({} < {})", time, self.last_update); } if value.is_nan() { - log::warn!("rrdb update failed - new value is NAN"); - return; + bail!("new value is NAN"); } // derive counter value @@ -150,13 +146,13 @@ impl RRA { let diff = if self.counter_value.is_nan() { 0.0 } else if is_counter && value < 0.0 { - log::warn!("rrdb update failed - got negative value for counter"); - return; + bail!("got negative value for counter"); } else if is_counter && value < self.counter_value { - // Note: We do not try automatic overflow corrections + // Note: We do not try automatic overflow corrections, but + // we update counter_value anyways, so that we can compute the diff + // next time. self.counter_value = value; - log::warn!("rrdb update failed - conter overflow/reset detected"); - return; + bail!("conter overflow/reset detected"); } else { value - self.counter_value }; @@ -166,6 +162,8 @@ impl RRA { self.delete_old(time); self.compute_new_value(time, value); + + Ok(()) } } @@ -343,26 +341,32 @@ impl RRD { /// Note: This does not call [Self::save]. pub fn update(&mut self, time: f64, value: f64) { - if value.is_nan() { - log::warn!("rrdb update failed - new value is NAN"); - return; - } + let mut log_error = true; - let mut log_time_in_past = true; + let mut update_rra = |rra: &mut RRA| { + if let Err(err) = rra.update(time, value) { + if log_error { + log::error!("rrd update failed: {}", err); + // we only log the first error, because it is very + // likely other calls produce the same error + log_error = false; + } + } + }; - self.hour_avg.update(time, value, &mut log_time_in_past); - self.hour_max.update(time, value, &mut log_time_in_past); + update_rra(&mut self.hour_avg); + update_rra(&mut self.hour_max); - self.day_avg.update(time, value, &mut log_time_in_past); - self.day_max.update(time, value, &mut log_time_in_past); + update_rra(&mut self.day_avg); + update_rra(&mut self.day_max); - self.week_avg.update(time, value, &mut log_time_in_past); - self.week_max.update(time, value, &mut log_time_in_past); + update_rra(&mut self.week_avg); + update_rra(&mut self.week_max); - self.month_avg.update(time, value, &mut log_time_in_past); - self.month_max.update(time, value, &mut log_time_in_past); + update_rra(&mut self.month_avg); + update_rra(&mut self.month_max); - self.year_avg.update(time, value, &mut log_time_in_past); - self.year_max.update(time, value, &mut log_time_in_past); + update_rra(&mut self.year_avg); + update_rra(&mut self.year_max); } } From 071cb7aa8b147689aae54f3e6c5e7b145b33f626 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 7 Oct 2021 08:50:50 +0200 Subject: [PATCH 035/111] proxmox-rrd: use correct directory options in create_rrdb_dir --- proxmox-rrd/src/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 08e91c9c..1084a037 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -41,7 +41,7 @@ impl RRDCache { /// Create rrdd stat dir with correct permission pub fn create_rrdb_dir(&self) -> Result<(), Error> { - create_path(&self.basedir, Some(self.dir_options.clone()), Some(self.file_options.clone())) + create_path(&self.basedir, Some(self.dir_options.clone()), Some(self.dir_options.clone())) .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; Ok(()) From fa9757e67f541d18f885a1e76861b7f9a35edbe0 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Fri, 8 Oct 2021 11:18:22 +0200 Subject: [PATCH 036/111] bump proxmox dependency to 0.14.0 and proxmox-http to 0.5.0 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd-api-types/Cargo.toml | 2 +- proxmox-rrd/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd-api-types/Cargo.toml b/proxmox-rrd-api-types/Cargo.toml index eaae9a90..0eec1c49 100644 --- a/proxmox-rrd-api-types/Cargo.toml +++ b/proxmox-rrd-api-types/Cargo.toml @@ -8,4 +8,4 @@ description = "API type definitions for proxmox-rrd crate." [dependencies] serde = { version = "1.0", features = [] } -proxmox = { version = "0.13.5", features = ["api-macro"] } +proxmox = { version = "0.14.0", features = ["api-macro"] } diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 19db5bf6..de9dcfe3 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -10,6 +10,6 @@ anyhow = "1.0" bitflags = "1.2.1" log = "0.4" -proxmox = { version = "0.13.5", features = ["api-macro"] } +proxmox = { version = "0.14.0", features = ["api-macro"] } proxmox-rrd-api-types = { path = "../proxmox-rrd-api-types" } From e0ce41b03ac713c25f9f0fca576167aefe659820 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Fri, 8 Oct 2021 11:19:37 +0200 Subject: [PATCH 037/111] update to first proxmox crate split Signed-off-by: Wolfgang Bumiller --- proxmox-rrd-api-types/Cargo.toml | 4 ++-- proxmox-rrd-api-types/src/lib.rs | 3 ++- proxmox-rrd/Cargo.toml | 3 ++- proxmox-rrd/src/cache.rs | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/proxmox-rrd-api-types/Cargo.toml b/proxmox-rrd-api-types/Cargo.toml index 0eec1c49..816f7fde 100644 --- a/proxmox-rrd-api-types/Cargo.toml +++ b/proxmox-rrd-api-types/Cargo.toml @@ -7,5 +7,5 @@ description = "API type definitions for proxmox-rrd crate." [dependencies] -serde = { version = "1.0", features = [] } -proxmox = { version = "0.14.0", features = ["api-macro"] } +serde = { version = "1.0", features = ["derive"] } +proxmox-schema = { version = "1", features = ["api-macro"] } diff --git a/proxmox-rrd-api-types/src/lib.rs b/proxmox-rrd-api-types/src/lib.rs index 4a0165ff..b5e62e73 100644 --- a/proxmox-rrd-api-types/src/lib.rs +++ b/proxmox-rrd-api-types/src/lib.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; -use proxmox::api::api; + +use proxmox_schema::api; #[api()] #[derive(Copy, Clone, Serialize, Deserialize)] diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index de9dcfe3..7225be8e 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0" bitflags = "1.2.1" log = "0.4" -proxmox = { version = "0.14.0", features = ["api-macro"] } +proxmox = { version = "0.14.0" } +proxmox-time = "1" proxmox-rrd-api-types = { path = "../proxmox-rrd-api-types" } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 1084a037..fe28aeda 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -62,7 +62,7 @@ impl RRDCache { create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.file_options.clone()))?; let mut map = self.cache.write().unwrap(); - let now = proxmox::tools::time::epoch_f64(); + let now = proxmox_time::epoch_f64(); if let Some(rrd) = map.get_mut(rel_path) { rrd.update(now, value); From 03555549056991c0808c746840bf7c0f198b02f9 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:38 +0200 Subject: [PATCH 038/111] proxmox-rrd: use a journal to reduce amount of bytes written Append pending changes in a simple text based format that allows for lockless appends as long as we stay below 4 KiB data per write. Apply the journal every 30 minutes and on daemon startup. Note that we do not ensure that the journal is synced, this is a perfomance optimization we can make as the kernel defaults to writeback in-flight data every 30s (sysctl vm/dirty_expire_centisecs) anyway, so we lose at max half a minute of data on a crash, here one should have in mind that we normally expose 1 minute as finest granularity anyway, so not really much lost. Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/Cargo.toml | 1 + proxmox-rrd/src/cache.rs | 225 ++++++++++++++++++++++++++++++++++----- proxmox-rrd/src/lib.rs | 6 +- proxmox-rrd/src/rrd.rs | 30 ++++++ 4 files changed, 232 insertions(+), 30 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 7225be8e..c66344ac 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -9,6 +9,7 @@ description = "Simple RRD database implementation." anyhow = "1.0" bitflags = "1.2.1" log = "0.4" +nix = "0.19.1" proxmox = { version = "0.14.0" } proxmox-time = "1" diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index fe28aeda..7c56e047 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -1,24 +1,46 @@ +use std::fs::File; use std::path::{Path, PathBuf}; use std::collections::HashMap; -use std::sync::{RwLock}; +use std::sync::RwLock; +use std::io::Write; +use std::io::{BufRead, BufReader}; +use std::os::unix::io::AsRawFd; -use anyhow::{format_err, Error}; +use anyhow::{format_err, bail, Error}; +use nix::fcntl::OFlag; -use proxmox::tools::fs::{create_path, CreateOptions}; +use proxmox::tools::fs::{atomic_open_or_create_file, create_path, CreateOptions}; use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; use crate::{DST, rrd::RRD}; +const RRD_JOURNAL_NAME: &str = "rrd.journal"; + /// RRD cache - keep RRD data in RAM, but write updates to disk /// /// This cache is designed to run as single instance (no concurrent /// access from other processes). pub struct RRDCache { + apply_interval: f64, basedir: PathBuf, file_options: CreateOptions, dir_options: CreateOptions, - cache: RwLock>, + state: RwLock, +} + +// shared state behind RwLock +struct RRDCacheState { + rrd_map: HashMap, + journal: File, + last_journal_flush: f64, +} + +struct JournalEntry { + time: f64, + value: f64, + dst: DST, + rel_path: String, } impl RRDCache { @@ -28,21 +50,166 @@ impl RRDCache { basedir: P, file_options: Option, dir_options: Option, - ) -> Self { + apply_interval: f64, + ) -> Result { let basedir = basedir.as_ref().to_owned(); - Self { + + let file_options = file_options.unwrap_or_else(|| CreateOptions::new()); + let dir_options = dir_options.unwrap_or_else(|| CreateOptions::new()); + + create_path(&basedir, Some(dir_options.clone()), Some(dir_options.clone())) + .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; + + let mut journal_path = basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; + let journal = atomic_open_or_create_file(&journal_path, flags, &[], file_options.clone())?; + + let state = RRDCacheState { + journal, + rrd_map: HashMap::new(), + last_journal_flush: 0.0, + }; + + Ok(Self { basedir, - file_options: file_options.unwrap_or_else(|| CreateOptions::new()), - dir_options: dir_options.unwrap_or_else(|| CreateOptions::new()), - cache: RwLock::new(HashMap::new()), - } + file_options, + dir_options, + apply_interval, + state: RwLock::new(state), + }) } - /// Create rrdd stat dir with correct permission - pub fn create_rrdb_dir(&self) -> Result<(), Error> { + fn parse_journal_line(line: &str) -> Result { - create_path(&self.basedir, Some(self.dir_options.clone()), Some(self.dir_options.clone())) - .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; + let line = line.trim(); + + let parts: Vec<&str> = line.splitn(4, ':').collect(); + if parts.len() != 4 { + bail!("wrong numper of components"); + } + + let time: f64 = parts[0].parse() + .map_err(|_| format_err!("unable to parse time"))?; + let value: f64 = parts[1].parse() + .map_err(|_| format_err!("unable to parse value"))?; + let dst: u8 = parts[2].parse() + .map_err(|_| format_err!("unable to parse data source type"))?; + + let dst = match dst { + 0 => DST::Gauge, + 1 => DST::Derive, + _ => bail!("got strange value for data source type '{}'", dst), + }; + + let rel_path = parts[3].to_string(); + + Ok(JournalEntry { time, value, dst, rel_path }) + } + + fn append_journal_entry( + state: &mut RRDCacheState, + time: f64, + value: f64, + dst: DST, + rel_path: &str, + ) -> Result<(), Error> { + let journal_entry = format!("{}:{}:{}:{}\n", time, value, dst as u8, rel_path); + state.journal.write_all(journal_entry.as_bytes())?; + Ok(()) + } + + pub fn apply_journal(&self) -> Result<(), Error> { + let mut state = self.state.write().unwrap(); // block writers + self.apply_journal_locked(&mut state) + } + + fn apply_journal_locked(&self, state: &mut RRDCacheState) -> Result<(), Error> { + + log::info!("applying rrd journal"); + + state.last_journal_flush = proxmox_time::epoch_f64(); + + let mut journal_path = self.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let flags = OFlag::O_CLOEXEC|OFlag::O_RDONLY; + let journal = atomic_open_or_create_file(&journal_path, flags, &[], self.file_options.clone())?; + let mut journal = BufReader::new(journal); + + let mut last_update_map = HashMap::new(); + + let mut get_last_update = |rel_path: &str, rrd: &RRD| { + if let Some(time) = last_update_map.get(rel_path) { + return *time; + } + let last_update = rrd.last_update(); + last_update_map.insert(rel_path.to_string(), last_update); + last_update + }; + + let mut linenr = 0; + loop { + linenr += 1; + let mut line = String::new(); + let len = journal.read_line(&mut line)?; + if len == 0 { break; } + + let entry = match Self::parse_journal_line(&line) { + Ok(entry) => entry, + Err(err) => { + log::warn!("unable to parse rrd journal line {} (skip) - {}", linenr, err); + continue; // skip unparsable lines + } + }; + + if let Some(rrd) = state.rrd_map.get_mut(&entry.rel_path) { + if entry.time > get_last_update(&entry.rel_path, &rrd) { + rrd.update(entry.time, entry.value); + } + } else { + let mut path = self.basedir.clone(); + path.push(&entry.rel_path); + create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; + + let mut rrd = match RRD::load(&path) { + Ok(rrd) => rrd, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); + } + RRD::new(entry.dst) + }, + }; + if entry.time > get_last_update(&entry.rel_path, &rrd) { + rrd.update(entry.time, entry.value); + } + state.rrd_map.insert(entry.rel_path.clone(), rrd); + } + } + + // save all RRDs + + let mut errors = 0; + for (rel_path, rrd) in state.rrd_map.iter() { + let mut path = self.basedir.clone(); + path.push(&rel_path); + if let Err(err) = rrd.save(&path, self.file_options.clone()) { + errors += 1; + log::error!("unable to save {:?}: {}", path, err); + } + } + + // if everything went ok, commit the journal + + if errors == 0 { + nix::unistd::ftruncate(state.journal.as_raw_fd(), 0) + .map_err(|err| format_err!("unable to truncate journal - {}", err))?; + log::info!("rrd journal successfully committed"); + } else { + log::error!("errors during rrd flush - unable to commit rrd journal"); + } Ok(()) } @@ -53,21 +220,26 @@ impl RRDCache { rel_path: &str, value: f64, dst: DST, - save: bool, ) -> Result<(), Error> { - let mut path = self.basedir.clone(); - path.push(rel_path); + let mut state = self.state.write().unwrap(); // block other writers - create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.file_options.clone()))?; - - let mut map = self.cache.write().unwrap(); let now = proxmox_time::epoch_f64(); - if let Some(rrd) = map.get_mut(rel_path) { + if (now - state.last_journal_flush) > self.apply_interval { + if let Err(err) = self.apply_journal_locked(&mut state) { + log::error!("apply journal failed: {}", err); + } + } + + Self::append_journal_entry(&mut state, now, value, dst, rel_path)?; + + if let Some(rrd) = state.rrd_map.get_mut(rel_path) { rrd.update(now, value); - if save { rrd.save(&path, self.file_options.clone())?; } } else { + let mut path = self.basedir.clone(); + path.push(rel_path); + create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; let mut rrd = match RRD::load(&path) { Ok(rrd) => rrd, Err(err) => { @@ -78,10 +250,7 @@ impl RRDCache { }, }; rrd.update(now, value); - if save { - rrd.save(&path, self.file_options.clone())?; - } - map.insert(rel_path.into(), rrd); + state.rrd_map.insert(rel_path.into(), rrd); } Ok(()) @@ -97,9 +266,9 @@ impl RRDCache { mode: RRDMode, ) -> Option<(u64, u64, Vec>)> { - let map = self.cache.read().unwrap(); + let state = self.state.read().unwrap(); - match map.get(&format!("{}/{}", base, name)) { + match state.rrd_map.get(&format!("{}/{}", base, name)) { Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), None => None, } diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs index 303cd55d..d83e6ffc 100644 --- a/proxmox-rrd/src/lib.rs +++ b/proxmox-rrd/src/lib.rs @@ -13,9 +13,11 @@ mod cache; pub use cache::*; /// RRD data source tyoe +#[repr(u8)] +#[derive(Copy, Clone)] pub enum DST { /// Gauge values are stored unmodified. - Gauge, + Gauge = 0, /// Stores the difference to the previous value. - Derive, + Derive = 1, } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index f4c08909..026498ed 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -336,6 +336,36 @@ impl RRD { replace_file(filename, rrd_slice, options) } + pub fn last_update(&self) -> f64 { + + let mut last_update = 0.0; + + { + let mut check_last_update = |rra: &RRA| { + if rra.last_update > last_update { + last_update = rra.last_update; + } + }; + + check_last_update(&self.hour_avg); + check_last_update(&self.hour_max); + + check_last_update(&self.day_avg); + check_last_update(&self.day_max); + + check_last_update(&self.week_avg); + check_last_update(&self.week_max); + + check_last_update(&self.month_avg); + check_last_update(&self.month_max); + + check_last_update(&self.year_avg); + check_last_update(&self.year_max); + } + + last_update + } + /// Update the value (in memory) /// /// Note: This does not call [Self::save]. From bc68dee171759dffa27d1b08351bd3f7b8ab8eea Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:41 +0200 Subject: [PATCH 039/111] proxmox-rrd: implement new CBOR based format Storing much more data points now got get better graphs. Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd-api-types/src/lib.rs | 23 +- proxmox-rrd/Cargo.toml | 4 + proxmox-rrd/src/cache.rs | 51 ++- proxmox-rrd/src/lib.rs | 19 +- proxmox-rrd/src/rrd.rs | 554 +++++++++++++++---------------- proxmox-rrd/src/rrd_v1.rs | 296 +++++++++++++++++ 6 files changed, 623 insertions(+), 324 deletions(-) create mode 100644 proxmox-rrd/src/rrd_v1.rs diff --git a/proxmox-rrd-api-types/src/lib.rs b/proxmox-rrd-api-types/src/lib.rs index b5e62e73..32601477 100644 --- a/proxmox-rrd-api-types/src/lib.rs +++ b/proxmox-rrd-api-types/src/lib.rs @@ -14,19 +14,20 @@ pub enum RRDMode { } #[api()] -#[repr(u64)] #[derive(Copy, Clone, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] /// RRD time frame resolution pub enum RRDTimeFrameResolution { - /// 1 min => last 70 minutes - Hour = 60, - /// 30 min => last 35 hours - Day = 60*30, - /// 3 hours => about 8 days - Week = 60*180, - /// 12 hours => last 35 days - Month = 60*720, - /// 1 week => last 490 days - Year = 60*10080, + /// Hour + Hour, + /// Day + Day, + /// Week + Week, + /// Month + Month, + /// Year + Year, + /// Decade (10 years) + Decade, } diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index c66344ac..b3dd02c3 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -10,8 +10,12 @@ anyhow = "1.0" bitflags = "1.2.1" log = "0.4" nix = "0.19.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_cbor = "0.11.1" proxmox = { version = "0.14.0" } proxmox-time = "1" +proxmox-schema = { version = "1", features = [ "api-macro" ] } proxmox-rrd-api-types = { path = "../proxmox-rrd-api-types" } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 7c56e047..f14837fc 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -13,7 +13,7 @@ use proxmox::tools::fs::{atomic_open_or_create_file, create_path, CreateOptions} use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; -use crate::{DST, rrd::RRD}; +use crate::rrd::{DST, CF, RRD, RRA}; const RRD_JOURNAL_NAME: &str = "rrd.journal"; @@ -81,6 +81,29 @@ impl RRDCache { }) } + fn create_default_rrd(dst: DST) -> RRD { + + let mut rra_list = Vec::new(); + + // 1min * 1440 => 1day + rra_list.push(RRA::new(CF::Average, 60, 1440)); + rra_list.push(RRA::new(CF::Maximum, 60, 1440)); + + // 30min * 1440 => 30days = 1month + rra_list.push(RRA::new(CF::Average, 30*60, 1440)); + rra_list.push(RRA::new(CF::Maximum, 30*60, 1440)); + + // 6h * 1440 => 360days = 1year + rra_list.push(RRA::new(CF::Average, 6*3600, 1440)); + rra_list.push(RRA::new(CF::Maximum, 6*3600, 1440)); + + // 1week * 570 => 10years + rra_list.push(RRA::new(CF::Average, 7*86400, 570)); + rra_list.push(RRA::new(CF::Maximum, 7*86400, 570)); + + RRD::new(dst, rra_list) + } + fn parse_journal_line(line: &str) -> Result { let line = line.trim(); @@ -179,7 +202,7 @@ impl RRDCache { if err.kind() != std::io::ErrorKind::NotFound { log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); } - RRD::new(entry.dst) + Self::create_default_rrd(entry.dst) }, }; if entry.time > get_last_update(&entry.rel_path, &rrd) { @@ -246,7 +269,7 @@ impl RRDCache { if err.kind() != std::io::ErrorKind::NotFound { log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); } - RRD::new(dst) + Self::create_default_rrd(dst) }, }; rrd.update(now, value); @@ -264,13 +287,29 @@ impl RRDCache { now: f64, timeframe: RRDTimeFrameResolution, mode: RRDMode, - ) -> Option<(u64, u64, Vec>)> { + ) -> Result>)>, Error> { let state = self.state.read().unwrap(); + let cf = match mode { + RRDMode::Max => CF::Maximum, + RRDMode::Average => CF::Average, + }; + + let now = now as u64; + + let (start, resolution) = match timeframe { + RRDTimeFrameResolution::Hour => (now - 3600, 60), + RRDTimeFrameResolution::Day => (now - 3600*24, 60), + RRDTimeFrameResolution::Week => (now - 3600*24*7, 30*60), + RRDTimeFrameResolution::Month => (now - 3600*24*30, 30*60), + RRDTimeFrameResolution::Year => (now - 3600*24*365, 6*60*60), + RRDTimeFrameResolution::Decade => (now - 10*3600*24*366, 7*86400), + }; + match state.rrd_map.get(&format!("{}/{}", base, name)) { - Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)), - None => None, + Some(rrd) => Ok(Some(rrd.extract_data(start, now, cf, resolution)?)), + None => Ok(None), } } } diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs index d83e6ffc..2038170d 100644 --- a/proxmox-rrd/src/lib.rs +++ b/proxmox-rrd/src/lib.rs @@ -1,23 +1,14 @@ -//! # Simple Round Robin Database files with fixed format +//! # Round Robin Database files //! //! ## Features //! //! * One file stores a single data source -//! * Small/constant file size (6008 bytes) -//! * Stores avarage and maximum values -//! * Stores data for different time resolution ([RRDTimeFrameResolution](proxmox_rrd_api_types::RRDTimeFrameResolution)) +//! * Stores data for different time resolution +//! * Simple cache implementation with journal support + +mod rrd_v1; pub mod rrd; mod cache; pub use cache::*; - -/// RRD data source tyoe -#[repr(u8)] -#[derive(Copy, Clone)] -pub enum DST { - /// Gauge values are stored unmodified. - Gauge = 0, - /// Stores the difference to the previous value. - Derive = 1, -} diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 026498ed..82fa5a3a 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -1,82 +1,175 @@ -//! # Round Robin Database file format +//! # Proxmox RRD format version 2 +//! +//! The new format uses +//! [CBOR](https://datatracker.ietf.org/doc/html/rfc8949) as storage +//! format. This way we can use the serde serialization framework, +//! which make our code more flexible, much nicer and type safe. +//! +//! ## Features +//! +//! * Well defined data format [CBOR](https://datatracker.ietf.org/doc/html/rfc8949) +//! * Plattform independent (big endian f64, hopefully a standard format?) +//! * Arbitrary number of RRAs (dynamically changeable) -use std::io::Read; use std::path::Path; use anyhow::{bail, Error}; -use bitflags::bitflags; -use proxmox::tools::{fs::replace_file, fs::CreateOptions}; +use serde::{Serialize, Deserialize}; -use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; +use proxmox::tools::fs::{replace_file, CreateOptions}; +use proxmox_schema::api; -/// The number of data entries per RRA -pub const RRD_DATA_ENTRIES: usize = 70; +use crate::rrd_v1; -/// Proxmox RRD file magic number -// openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; -pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; +/// Proxmox RRD v2 file magic number +// openssl::sha::sha256(b"Proxmox Round Robin Database file v2.0")[0..8]; +pub const PROXMOX_RRD_MAGIC_2_0: [u8; 8] = [224, 200, 228, 27, 239, 112, 122, 159]; -use crate::DST; - -bitflags!{ - /// Flags to specify the data soure type and consolidation function - pub struct RRAFlags: u64 { - // Data Source Types - const DST_GAUGE = 1; - const DST_DERIVE = 2; - const DST_COUNTER = 4; - const DST_MASK = 255; // first 8 bits - - // Consolidation Functions - const CF_AVERAGE = 1 << 8; - const CF_MAX = 2 << 8; - const CF_MASK = 255 << 8; - } +#[api()] +#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq)] +#[serde(rename_all = "kebab-case")] +/// RRD data source type +pub enum DST { + /// Gauge values are stored unmodified. + Gauge, + /// Stores the difference to the previous value. + Derive, + /// Stores the difference to the previous value (like Derive), but + /// detect counter overflow (and ignores that value) + Counter, } -/// Round Robin Archive with [RRD_DATA_ENTRIES] data slots. -/// -/// This data structure is used inside [RRD] and directly written to the -/// RRD files. -#[repr(C)] -pub struct RRA { - /// Defined the data soure type and consolidation function - pub flags: RRAFlags, - /// Resulution (seconds) from [RRDTimeFrameResolution] - pub resolution: u64, +#[api()] +#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq)] +#[serde(rename_all = "kebab-case")] +/// Consolidation function +pub enum CF { + /// Average + Average, + /// Maximum + Maximum, + /// Minimum + Minimum, +} + +#[derive(Serialize, Deserialize)] +pub struct DataSource { + /// Data source type + pub dst: DST, /// Last update time (epoch) pub last_update: f64, - /// Count values computed inside this update interval - pub last_count: u64, - /// Stores the last value, used to compute differential value for derive/counters + /// Stores the last value, used to compute differential value for + /// derive/counters pub counter_value: f64, - /// Data slots - pub data: [f64; RRD_DATA_ENTRIES], } -impl RRA { - fn new(flags: RRAFlags, resolution: u64) -> Self { +impl DataSource { + + pub fn new(dst: DST) -> Self { Self { - flags, resolution, + dst, last_update: 0.0, - last_count: 0, counter_value: f64::NAN, - data: [f64::NAN; RRD_DATA_ENTRIES], } } - fn delete_old(&mut self, time: f64) { - let epoch = time as u64; - let last_update = self.last_update as u64; - let reso = self.resolution; + fn compute_new_value(&mut self, time: f64, mut value: f64) -> Result { + if time <= self.last_update { + bail!("time in past ({} < {})", time, self.last_update); + } - let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso; + if value.is_nan() { + bail!("new value is NAN"); + } + + // derive counter value + let is_counter = self.dst == DST::Counter; + + if is_counter || self.dst == DST::Derive { + let time_diff = time - self.last_update; + + let diff = if self.counter_value.is_nan() { + 0.0 + } else if is_counter && value < 0.0 { + bail!("got negative value for counter"); + } else if is_counter && value < self.counter_value { + // Note: We do not try automatic overflow corrections, but + // we update counter_value anyways, so that we can compute the diff + // next time. + self.counter_value = value; + bail!("conter overflow/reset detected"); + } else { + value - self.counter_value + }; + self.counter_value = value; + value = diff/time_diff; + } + + Ok(value) + } + + +} + +#[derive(Serialize, Deserialize)] +pub struct RRA { + pub resolution: u64, + pub cf: CF, + /// Count values computed inside this update interval + pub last_count: u64, + /// The actual data + pub data: Vec, +} + +impl RRA { + + pub fn new(cf: CF, resolution: u64, points: usize) -> Self { + Self { + cf, + resolution, + last_count: 0, + data: vec![f64::NAN; points], + } + } + + // directly overwrite data slots + // the caller need to set last_update value on the DataSource manually. + pub(crate) fn insert_data( + &mut self, + start: u64, + resolution: u64, + data: Vec>, + ) -> Result<(), Error> { + if resolution != self.resolution { + bail!("inser_data failed: got wrong resolution"); + } + let num_entries = self.data.len() as u64; + let mut index = ((start/self.resolution) % num_entries) as usize; + + for i in 0..data.len() { + if let Some(v) = data[i] { + self.data[index] = v; + } + index += 1; + if index >= self.data.len() { index = 0; } + } + Ok(()) + } + + fn delete_old_slots(&mut self, time: f64, last_update: f64) { + let epoch = time as u64; + let last_update = last_update as u64; + let reso = self.resolution; + let num_entries = self.data.len() as u64; + + let min_time = epoch - num_entries*reso; let min_time = (min_time/reso + 1)*reso; - let mut t = last_update.saturating_sub((RRD_DATA_ENTRIES as u64)*reso); - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + let mut t = last_update.saturating_sub(num_entries*reso); + let mut index = ((t/reso) % num_entries) as usize; + for _ in 0..num_entries { + t += reso; + index = (index + 1) % (num_entries as usize); if t < min_time { self.data[index] = f64::NAN; } else { @@ -85,13 +178,14 @@ impl RRA { } } - fn compute_new_value(&mut self, time: f64, value: f64) { + fn compute_new_value(&mut self, time: f64, last_update: f64, value: f64) { let epoch = time as u64; - let last_update = self.last_update as u64; + let last_update = last_update as u64; let reso = self.resolution; + let num_entries = self.data.len() as u64; - let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - let last_index = ((last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + let index = ((epoch/reso) % num_entries) as usize; + let last_index = ((last_update/reso) % num_entries) as usize; if (epoch - (last_update as u64)) > reso || index != last_index { self.last_count = 0; @@ -112,258 +206,111 @@ impl RRA { self.data[index] = value; self.last_count = 1; } else { - let new_value = if self.flags.contains(RRAFlags::CF_MAX) { - if last_value > value { last_value } else { value } - } else if self.flags.contains(RRAFlags::CF_AVERAGE) { - (last_value*(self.last_count as f64))/(new_count as f64) - + value/(new_count as f64) - } else { - log::error!("rrdb update failed - unknown CF"); - return; + let new_value = match self.cf { + CF::Maximum => if last_value > value { last_value } else { value }, + CF::Minimum => if last_value < value { last_value } else { value }, + CF::Average => { + (last_value*(self.last_count as f64))/(new_count as f64) + + value/(new_count as f64) + } }; self.data[index] = new_value; self.last_count = new_count; } - self.last_update = time; } - // Note: This may update the state even in case of errors (see counter overflow) - fn update(&mut self, time: f64, mut value: f64) -> Result<(), Error> { - - if time <= self.last_update { - bail!("time in past ({} < {})", time, self.last_update); - } - - if value.is_nan() { - bail!("new value is NAN"); - } - - // derive counter value - if self.flags.intersects(RRAFlags::DST_DERIVE | RRAFlags::DST_COUNTER) { - let time_diff = time - self.last_update; - let is_counter = self.flags.contains(RRAFlags::DST_COUNTER); - - let diff = if self.counter_value.is_nan() { - 0.0 - } else if is_counter && value < 0.0 { - bail!("got negative value for counter"); - } else if is_counter && value < self.counter_value { - // Note: We do not try automatic overflow corrections, but - // we update counter_value anyways, so that we can compute the diff - // next time. - self.counter_value = value; - bail!("conter overflow/reset detected"); - } else { - value - self.counter_value - }; - self.counter_value = value; - value = diff/time_diff; - } - - self.delete_old(time); - self.compute_new_value(time, value); - - Ok(()) - } -} - -/// Round Robin Database file format with fixed number of [RRA]s -#[repr(C)] -// Note: Avoid alignment problems by using 8byte types only -pub struct RRD { - /// The magic number to identify the file type - pub magic: [u8; 8], - /// Hourly data (average values) - pub hour_avg: RRA, - /// Hourly data (maximum values) - pub hour_max: RRA, - /// Dayly data (average values) - pub day_avg: RRA, - /// Dayly data (maximum values) - pub day_max: RRA, - /// Weekly data (average values) - pub week_avg: RRA, - /// Weekly data (maximum values) - pub week_max: RRA, - /// Monthly data (average values) - pub month_avg: RRA, - /// Monthly data (maximum values) - pub month_max: RRA, - /// Yearly data (average values) - pub year_avg: RRA, - /// Yearly data (maximum values) - pub year_max: RRA, -} - -impl RRD { - - /// Create a new empty instance - pub fn new(dst: DST) -> Self { - let flags = match dst { - DST::Gauge => RRAFlags::DST_GAUGE, - DST::Derive => RRAFlags::DST_DERIVE, - }; - - Self { - magic: PROXMOX_RRD_MAGIC_1_0, - hour_avg: RRA::new( - flags | RRAFlags::CF_AVERAGE, - RRDTimeFrameResolution::Hour as u64, - ), - hour_max: RRA::new( - flags | RRAFlags::CF_MAX, - RRDTimeFrameResolution::Hour as u64, - ), - day_avg: RRA::new( - flags | RRAFlags::CF_AVERAGE, - RRDTimeFrameResolution::Day as u64, - ), - day_max: RRA::new( - flags | RRAFlags::CF_MAX, - RRDTimeFrameResolution::Day as u64, - ), - week_avg: RRA::new( - flags | RRAFlags::CF_AVERAGE, - RRDTimeFrameResolution::Week as u64, - ), - week_max: RRA::new( - flags | RRAFlags::CF_MAX, - RRDTimeFrameResolution::Week as u64, - ), - month_avg: RRA::new( - flags | RRAFlags::CF_AVERAGE, - RRDTimeFrameResolution::Month as u64, - ), - month_max: RRA::new( - flags | RRAFlags::CF_MAX, - RRDTimeFrameResolution::Month as u64, - ), - year_avg: RRA::new( - flags | RRAFlags::CF_AVERAGE, - RRDTimeFrameResolution::Year as u64, - ), - year_max: RRA::new( - flags | RRAFlags::CF_MAX, - RRDTimeFrameResolution::Year as u64, - ), - } - } - - /// Extract data from the archive - pub fn extract_data( + fn extract_data( &self, - time: f64, - timeframe: RRDTimeFrameResolution, - mode: RRDMode, + start: u64, + end: u64, + last_update: f64, ) -> (u64, u64, Vec>) { - - let epoch = time as u64; - let reso = timeframe as u64; - - let end = reso*(epoch/reso + 1); - let start = end - reso*(RRD_DATA_ENTRIES as u64); + let last_update = last_update as u64; + let reso = self.resolution; + let num_entries = self.data.len() as u64; let mut list = Vec::new(); - let raa = match (mode, timeframe) { - (RRDMode::Average, RRDTimeFrameResolution::Hour) => &self.hour_avg, - (RRDMode::Max, RRDTimeFrameResolution::Hour) => &self.hour_max, - (RRDMode::Average, RRDTimeFrameResolution::Day) => &self.day_avg, - (RRDMode::Max, RRDTimeFrameResolution::Day) => &self.day_max, - (RRDMode::Average, RRDTimeFrameResolution::Week) => &self.week_avg, - (RRDMode::Max, RRDTimeFrameResolution::Week) => &self.week_max, - (RRDMode::Average, RRDTimeFrameResolution::Month) => &self.month_avg, - (RRDMode::Max, RRDTimeFrameResolution::Month) => &self.month_max, - (RRDMode::Average, RRDTimeFrameResolution::Year) => &self.year_avg, - (RRDMode::Max, RRDTimeFrameResolution::Year) => &self.year_max, - }; - - let rrd_end = reso*((raa.last_update as u64)/reso); - let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64); + let rrd_end = reso*(last_update/reso); + let rrd_start = rrd_end.saturating_sub(reso*num_entries); let mut t = start; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; - for _ in 0..RRD_DATA_ENTRIES { + let mut index = ((t/reso) % num_entries) as usize; + for _ in 0..num_entries { + if t > end { break; }; if t < rrd_start || t > rrd_end { list.push(None); } else { - let value = raa.data[index]; + let value = self.data[index]; if value.is_nan() { list.push(None); } else { list.push(Some(value)); } } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + t += reso; index = (index + 1) % (num_entries as usize); } (start, reso, list) } +} - /// Create instance from raw data, testing data len and magic number - pub fn from_raw(mut raw: &[u8]) -> Result { - let expected_len = std::mem::size_of::(); - if raw.len() != expected_len { - let msg = format!("wrong data size ({} != {})", raw.len(), expected_len); - return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); +#[derive(Serialize, Deserialize)] +pub struct RRD { + pub source: DataSource, + pub rra_list: Vec, +} + +impl RRD { + + pub fn new(dst: DST, rra_list: Vec) -> RRD { + + let source = DataSource::new(dst); + + RRD { + source, + rra_list, } - let mut rrd: RRD = unsafe { std::mem::zeroed() }; - unsafe { - let rrd_slice = std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len); - raw.read_exact(rrd_slice)?; - } - - if rrd.magic != PROXMOX_RRD_MAGIC_1_0 { - let msg = "wrong magic number".to_string(); - return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); - } - - Ok(rrd) } /// Load data from a file pub fn load(path: &Path) -> Result { let raw = std::fs::read(path)?; - Self::from_raw(&raw) + if raw.len() < 8 { + let msg = format!("not an rrd file - file is too small ({})", raw.len()); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + if raw[0..8] == rrd_v1::PROXMOX_RRD_MAGIC_1_0 { + let v1 = rrd_v1::RRDv1::from_raw(&raw)?; + v1.to_rrd_v2() + .map_err(|err| { + let msg = format!("unable to convert from old V1 format - {}", err); + std::io::Error::new(std::io::ErrorKind::Other, msg) + }) + } else if raw[0..8] == PROXMOX_RRD_MAGIC_2_0 { + serde_cbor::from_slice(&raw[8..]) + .map_err(|err| { + let msg = format!("unable to decode RRD file - {}", err); + std::io::Error::new(std::io::ErrorKind::Other, msg) + }) + } else { + let msg = format!("not an rrd file - unknown magic number"); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } } /// Store data into a file (atomic replace file) pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> { - let rrd_slice = unsafe { - std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) - }; - replace_file(filename, rrd_slice, options) + let mut data: Vec = Vec::new(); + data.extend(&PROXMOX_RRD_MAGIC_2_0); + serde_cbor::to_writer(&mut data, self)?; + replace_file(filename, &data, options) } pub fn last_update(&self) -> f64 { - - let mut last_update = 0.0; - - { - let mut check_last_update = |rra: &RRA| { - if rra.last_update > last_update { - last_update = rra.last_update; - } - }; - - check_last_update(&self.hour_avg); - check_last_update(&self.hour_max); - - check_last_update(&self.day_avg); - check_last_update(&self.day_max); - - check_last_update(&self.week_avg); - check_last_update(&self.week_max); - - check_last_update(&self.month_avg); - check_last_update(&self.month_max); - - check_last_update(&self.year_avg); - check_last_update(&self.year_max); - } - - last_update + self.source.last_update } /// Update the value (in memory) @@ -371,32 +318,53 @@ impl RRD { /// Note: This does not call [Self::save]. pub fn update(&mut self, time: f64, value: f64) { - let mut log_error = true; - - let mut update_rra = |rra: &mut RRA| { - if let Err(err) = rra.update(time, value) { - if log_error { - log::error!("rrd update failed: {}", err); - // we only log the first error, because it is very - // likely other calls produce the same error - log_error = false; - } + let value = match self.source.compute_new_value(time, value) { + Ok(value) => value, + Err(err) => { + log::error!("rrd update failed: {}", err); + return; } }; - update_rra(&mut self.hour_avg); - update_rra(&mut self.hour_max); + let last_update = self.source.last_update; + self.source.last_update = time; - update_rra(&mut self.day_avg); - update_rra(&mut self.day_max); - - update_rra(&mut self.week_avg); - update_rra(&mut self.week_max); - - update_rra(&mut self.month_avg); - update_rra(&mut self.month_max); - - update_rra(&mut self.year_avg); - update_rra(&mut self.year_max); + for rra in self.rra_list.iter_mut() { + rra.delete_old_slots(time, last_update); + rra.compute_new_value(time, last_update, value); + } } + + /// Extract data from the archive + /// + /// This selects the RRA with specified [CF] and (minimum) + /// resolution, and extract data from `start` to `end`. + pub fn extract_data( + &self, + start: u64, + end: u64, + cf: CF, + resolution: u64, + ) -> Result<(u64, u64, Vec>), Error> { + + let mut rra: Option<&RRA> = None; + for item in self.rra_list.iter() { + if item.cf != cf { continue; } + if item.resolution > resolution { continue; } + + if let Some(current) = rra { + if item.resolution > current.resolution { + rra = Some(item); + } + } else { + rra = Some(item); + } + } + + match rra { + Some(rra) => Ok(rra.extract_data(start, end, self.source.last_update)), + None => bail!("unable to find RRA suitable ({:?}:{})", cf, resolution), + } + } + } diff --git a/proxmox-rrd/src/rrd_v1.rs b/proxmox-rrd/src/rrd_v1.rs new file mode 100644 index 00000000..919896f0 --- /dev/null +++ b/proxmox-rrd/src/rrd_v1.rs @@ -0,0 +1,296 @@ +use std::io::Read; + +use anyhow::Error; +use bitflags::bitflags; + +/// The number of data entries per RRA +pub const RRD_DATA_ENTRIES: usize = 70; + +/// Proxmox RRD file magic number +// openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; +pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; + +use crate::rrd::{RRD, RRA, CF, DST, DataSource}; + +bitflags!{ + /// Flags to specify the data soure type and consolidation function + pub struct RRAFlags: u64 { + // Data Source Types + const DST_GAUGE = 1; + const DST_DERIVE = 2; + const DST_COUNTER = 4; + const DST_MASK = 255; // first 8 bits + + // Consolidation Functions + const CF_AVERAGE = 1 << 8; + const CF_MAX = 2 << 8; + const CF_MASK = 255 << 8; + } +} + +/// Round Robin Archive with [RRD_DATA_ENTRIES] data slots. +/// +/// This data structure is used inside [RRD] and directly written to the +/// RRD files. +#[repr(C)] +pub struct RRAv1 { + /// Defined the data soure type and consolidation function + pub flags: RRAFlags, + /// Resulution (seconds) from [RRDTimeFrameResolution] + pub resolution: u64, + /// Last update time (epoch) + pub last_update: f64, + /// Count values computed inside this update interval + pub last_count: u64, + /// Stores the last value, used to compute differential value for derive/counters + pub counter_value: f64, + /// Data slots + pub data: [f64; RRD_DATA_ENTRIES], +} + +impl RRAv1 { + + fn extract_data( + &self, + ) -> (u64, u64, Vec>) { + let reso = self.resolution; + + let mut list = Vec::new(); + + let rra_end = reso*((self.last_update as u64)/reso); + let rra_start = rra_end - reso*(RRD_DATA_ENTRIES as u64); + + let mut t = rra_start; + let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + for _ in 0..RRD_DATA_ENTRIES { + let value = self.data[index]; + if value.is_nan() { + list.push(None); + } else { + list.push(Some(value)); + } + + t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + } + + (rra_start, reso, list) + } +} + +/// Round Robin Database file format with fixed number of [RRA]s +#[repr(C)] +// Note: Avoid alignment problems by using 8byte types only +pub struct RRDv1 { + /// The magic number to identify the file type + pub magic: [u8; 8], + /// Hourly data (average values) + pub hour_avg: RRAv1, + /// Hourly data (maximum values) + pub hour_max: RRAv1, + /// Dayly data (average values) + pub day_avg: RRAv1, + /// Dayly data (maximum values) + pub day_max: RRAv1, + /// Weekly data (average values) + pub week_avg: RRAv1, + /// Weekly data (maximum values) + pub week_max: RRAv1, + /// Monthly data (average values) + pub month_avg: RRAv1, + /// Monthly data (maximum values) + pub month_max: RRAv1, + /// Yearly data (average values) + pub year_avg: RRAv1, + /// Yearly data (maximum values) + pub year_max: RRAv1, +} + +impl RRDv1 { + + pub fn from_raw(mut raw: &[u8]) -> Result { + + let expected_len = std::mem::size_of::(); + + if raw.len() != expected_len { + let msg = format!("wrong data size ({} != {})", raw.len(), expected_len); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + let mut rrd: RRDv1 = unsafe { std::mem::zeroed() }; + unsafe { + let rrd_slice = std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len); + raw.read_exact(rrd_slice)?; + } + + if rrd.magic != PROXMOX_RRD_MAGIC_1_0 { + let msg = "wrong magic number".to_string(); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + Ok(rrd) + } + + pub fn to_rrd_v2(&self) -> Result { + + let mut rra_list = Vec::new(); + + // old format v1: + // + // hour 1 min, 70 points + // day 30 min, 70 points + // week 3 hours, 70 points + // month 12 hours, 70 points + // year 1 week, 70 points + // + // new default for RRD v2: + // + // day 1 min, 1440 points + // month 30 min, 1440 points + // year 365 min (6h), 1440 points + // decade 1 week, 570 points + + // Linear extrapolation + fn extrapolate_data(start: u64, reso: u64, factor: u64, data: Vec>) -> (u64, u64, Vec>) { + + let mut new = Vec::new(); + + for i in 0..data.len() { + let mut next = i + 1; + if next >= data.len() { next = 0 }; + let v = data[i]; + let v1 = data[next]; + match (v, v1) { + (Some(v), Some(v1)) => { + let diff = (v1 - v)/(factor as f64); + for j in 0..factor { + new.push(Some(v + diff*(j as f64))); + } + } + (Some(v), None) => { + new.push(Some(v)); + for _ in 0..factor-1 { + new.push(None); + } + } + (None, Some(v1)) => { + for _ in 0..factor-1 { + new.push(None); + } + new.push(Some(v1)); + } + (None, None) => { + for _ in 0..factor { + new.push(None); + } + } + } + } + + (start, reso/factor, new) + } + + // Try to convert to new, higher capacity format + + // compute daily average (merge old self.day_avg and self.hour_avg + let mut day_avg = RRA::new(CF::Average, 60, 1440); + + let (start, reso, data) = self.day_avg.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 30, data); + day_avg.insert_data(start, reso, data)?; + + let (start, reso, data) = self.hour_avg.extract_data(); + day_avg.insert_data(start, reso, data)?; + + // compute daily maximum (merge old self.day_max and self.hour_max + let mut day_max = RRA::new(CF::Maximum, 60, 1440); + + let (start, reso, data) = self.day_max.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 30, data); + day_max.insert_data(start, reso, data)?; + + let (start, reso, data) = self.hour_max.extract_data(); + day_max.insert_data(start, reso, data)?; + + // compute montly average (merge old self.month_avg, + // self.week_avg and self.day_avg) + let mut month_avg = RRA::new(CF::Average, 30*60, 1440); + + let (start, reso, data) = self.month_avg.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 24, data); + month_avg.insert_data(start, reso, data)?; + + let (start, reso, data) = self.week_avg.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 6, data); + month_avg.insert_data(start, reso, data)?; + + let (start, reso, data) = self.day_avg.extract_data(); + month_avg.insert_data(start, reso, data)?; + + // compute montly maximum (merge old self.month_max, + // self.week_max and self.day_max) + let mut month_max = RRA::new(CF::Maximum, 30*60, 1440); + + let (start, reso, data) = self.month_max.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 24, data); + month_max.insert_data(start, reso, data)?; + + let (start, reso, data) = self.week_max.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 6, data); + month_max.insert_data(start, reso, data)?; + + let (start, reso, data) = self.day_max.extract_data(); + month_max.insert_data(start, reso, data)?; + + // compute yearly average (merge old self.year_avg) + let mut year_avg = RRA::new(CF::Average, 6*3600, 1440); + + let (start, reso, data) = self.year_avg.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 28, data); + year_avg.insert_data(start, reso, data)?; + + // compute yearly maximum (merge old self.year_avg) + let mut year_max = RRA::new(CF::Maximum, 6*3600, 1440); + + let (start, reso, data) = self.year_max.extract_data(); + let (start, reso, data) = extrapolate_data(start, reso, 28, data); + year_max.insert_data(start, reso, data)?; + + // compute decade average (merge old self.year_avg) + let mut decade_avg = RRA::new(CF::Average, 7*86400, 570); + let (start, reso, data) = self.year_avg.extract_data(); + decade_avg.insert_data(start, reso, data)?; + + // compute decade maximum (merge old self.year_max) + let mut decade_max = RRA::new(CF::Maximum, 7*86400, 570); + let (start, reso, data) = self.year_max.extract_data(); + decade_max.insert_data(start, reso, data)?; + + rra_list.push(day_avg); + rra_list.push(day_max); + rra_list.push(month_avg); + rra_list.push(month_max); + rra_list.push(year_avg); + rra_list.push(year_max); + rra_list.push(decade_avg); + rra_list.push(decade_max); + + // use values from hour_avg for source (all RRAv1 must have the same config) + let dst = if self.hour_avg.flags.contains(RRAFlags::DST_COUNTER) { + DST::Counter + } else if self.hour_avg.flags.contains(RRAFlags::DST_DERIVE) { + DST::Derive + } else { + DST::Gauge + }; + + let source = DataSource { + dst, + counter_value: f64::NAN, + last_update: self.hour_avg.last_update, // IMPORTANT! + }; + Ok(RRD { + source, + rra_list, + }) + } +} From cf097c5a89f0fba32913418d12bdbaf3f023b28c Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:42 +0200 Subject: [PATCH 040/111] proxmox-rrd: remove dependency to proxmox-rrd-api-types Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/Cargo.toml | 2 -- proxmox-rrd/src/cache.rs | 30 ++++++++---------------------- proxmox-rrd/src/rrd.rs | 13 ++++++++++--- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index b3dd02c3..69a68530 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -17,5 +17,3 @@ serde_cbor = "0.11.1" proxmox = { version = "0.14.0" } proxmox-time = "1" proxmox-schema = { version = "1", features = [ "api-macro" ] } - -proxmox-rrd-api-types = { path = "../proxmox-rrd-api-types" } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index f14837fc..5e359d9b 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -11,8 +11,6 @@ use nix::fcntl::OFlag; use proxmox::tools::fs::{atomic_open_or_create_file, create_path, CreateOptions}; -use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution}; - use crate::rrd::{DST, CF, RRD, RRA}; const RRD_JOURNAL_NAME: &str = "rrd.journal"; @@ -280,35 +278,23 @@ impl RRDCache { } /// Extract data from cached RRD + /// + /// `start`: Start time. If not sepecified, we simply extract 10 data points. + /// `end`: End time. Default is to use the current time. pub fn extract_cached_data( &self, base: &str, name: &str, - now: f64, - timeframe: RRDTimeFrameResolution, - mode: RRDMode, + cf: CF, + resolution: u64, + start: Option, + end: Option, ) -> Result>)>, Error> { let state = self.state.read().unwrap(); - let cf = match mode { - RRDMode::Max => CF::Maximum, - RRDMode::Average => CF::Average, - }; - - let now = now as u64; - - let (start, resolution) = match timeframe { - RRDTimeFrameResolution::Hour => (now - 3600, 60), - RRDTimeFrameResolution::Day => (now - 3600*24, 60), - RRDTimeFrameResolution::Week => (now - 3600*24*7, 30*60), - RRDTimeFrameResolution::Month => (now - 3600*24*30, 30*60), - RRDTimeFrameResolution::Year => (now - 3600*24*365, 6*60*60), - RRDTimeFrameResolution::Decade => (now - 10*3600*24*366, 7*86400), - }; - match state.rrd_map.get(&format!("{}/{}", base, name)) { - Some(rrd) => Ok(Some(rrd.extract_data(start, now, cf, resolution)?)), + Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)), None => Ok(None), } } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 82fa5a3a..c97be96d 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -339,12 +339,15 @@ impl RRD { /// /// This selects the RRA with specified [CF] and (minimum) /// resolution, and extract data from `start` to `end`. + /// + /// `start`: Start time. If not sepecified, we simply extract 10 data points. + /// `end`: End time. Default is to use the current time. pub fn extract_data( &self, - start: u64, - end: u64, cf: CF, resolution: u64, + start: Option, + end: Option, ) -> Result<(u64, u64, Vec>), Error> { let mut rra: Option<&RRA> = None; @@ -362,7 +365,11 @@ impl RRD { } match rra { - Some(rra) => Ok(rra.extract_data(start, end, self.source.last_update)), + Some(rra) => { + let end = end.unwrap_or_else(|| proxmox_time::epoch_f64() as u64); + let start = start.unwrap_or(end - 10*rra.resolution); + Ok(rra.extract_data(start, end, self.source.last_update)) + } None => bail!("unable to find RRA suitable ({:?}:{})", cf, resolution), } } From ab567561b537f7fb736c28c3234fcfa757423073 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:43 +0200 Subject: [PATCH 041/111] proxmox-rrd: extract_data: include values from current slot Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index c97be96d..328ac9f2 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -231,7 +231,7 @@ impl RRA { let mut list = Vec::new(); - let rrd_end = reso*(last_update/reso); + let rrd_end = reso*(last_update/reso + 1); let rrd_start = rrd_end.saturating_sub(reso*num_entries); let mut t = start; From 4bf2db86664d2c70602e2072e888575765842df1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:44 +0200 Subject: [PATCH 042/111] remove proxmox-rrd-api-types crate, s/RRDTimeFrameResolution/RRDTimeFrame/ Because the types used inside the RRD have other requirements than the API types: - other serialization format - the API may not support all RRD features Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd-api-types/Cargo.toml | 11 ----------- proxmox-rrd-api-types/src/lib.rs | 33 -------------------------------- proxmox-rrd/src/rrd_v1.rs | 2 +- 3 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 proxmox-rrd-api-types/Cargo.toml delete mode 100644 proxmox-rrd-api-types/src/lib.rs diff --git a/proxmox-rrd-api-types/Cargo.toml b/proxmox-rrd-api-types/Cargo.toml deleted file mode 100644 index 816f7fde..00000000 --- a/proxmox-rrd-api-types/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "proxmox-rrd-api-types" -version = "0.1.0" -authors = ["Proxmox Support Team "] -edition = "2018" -description = "API type definitions for proxmox-rrd crate." - - -[dependencies] -serde = { version = "1.0", features = ["derive"] } -proxmox-schema = { version = "1", features = ["api-macro"] } diff --git a/proxmox-rrd-api-types/src/lib.rs b/proxmox-rrd-api-types/src/lib.rs deleted file mode 100644 index 32601477..00000000 --- a/proxmox-rrd-api-types/src/lib.rs +++ /dev/null @@ -1,33 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use proxmox_schema::api; - -#[api()] -#[derive(Copy, Clone, Serialize, Deserialize)] -#[serde(rename_all = "UPPERCASE")] -/// RRD consolidation mode -pub enum RRDMode { - /// Maximum - Max, - /// Average - Average, -} - -#[api()] -#[derive(Copy, Clone, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -/// RRD time frame resolution -pub enum RRDTimeFrameResolution { - /// Hour - Hour, - /// Day - Day, - /// Week - Week, - /// Month - Month, - /// Year - Year, - /// Decade (10 years) - Decade, -} diff --git a/proxmox-rrd/src/rrd_v1.rs b/proxmox-rrd/src/rrd_v1.rs index 919896f0..511b510b 100644 --- a/proxmox-rrd/src/rrd_v1.rs +++ b/proxmox-rrd/src/rrd_v1.rs @@ -36,7 +36,7 @@ bitflags!{ pub struct RRAv1 { /// Defined the data soure type and consolidation function pub flags: RRAFlags, - /// Resulution (seconds) from [RRDTimeFrameResolution] + /// Resulution (seconds) pub resolution: u64, /// Last update time (epoch) pub last_update: f64, From e928c24948f2dfed911d041998e2f643cb95aaf0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:45 +0200 Subject: [PATCH 043/111] proxmox-rrd: support CF::Last Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 328ac9f2..7a9ce94a 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -51,6 +51,8 @@ pub enum CF { Maximum, /// Minimum Minimum, + /// Use the last value + Last, } #[derive(Serialize, Deserialize)] @@ -209,6 +211,7 @@ impl RRA { let new_value = match self.cf { CF::Maximum => if last_value > value { last_value } else { value }, CF::Minimum => if last_value < value { last_value } else { value }, + CF::Last => value, CF::Average => { (last_value*(self.last_count as f64))/(new_count as f64) + value/(new_count as f64) From 2c72c6a7baf650bb61eb7d41386fb3e05c338f53 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:46 +0200 Subject: [PATCH 044/111] proxmox-rrd: split out load_rrd (cleanup) Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/cache.rs | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 5e359d9b..5225bd49 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -194,15 +194,8 @@ impl RRDCache { path.push(&entry.rel_path); create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - let mut rrd = match RRD::load(&path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - Self::create_default_rrd(entry.dst) - }, - }; + let mut rrd = Self::load_rrd(&path, entry.dst); + if entry.time > get_last_update(&entry.rel_path, &rrd) { rrd.update(entry.time, entry.value); } @@ -235,7 +228,19 @@ impl RRDCache { Ok(()) } - /// Update data in RAM and write file back to disk (if `save` is set) + fn load_rrd(path: &Path, dst: DST) -> RRD { + match RRD::load(path) { + Ok(rrd) => rrd, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); + } + Self::create_default_rrd(dst) + }, + } + } + + /// Update data in RAM and write file back to disk (journal) pub fn update_value( &self, rel_path: &str, @@ -261,15 +266,9 @@ impl RRDCache { let mut path = self.basedir.clone(); path.push(rel_path); create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - let mut rrd = match RRD::load(&path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - Self::create_default_rrd(dst) - }, - }; + + let mut rrd = Self::load_rrd(&path, dst); + rrd.update(now, value); state.rrd_map.insert(rel_path.into(), rrd); } From 4cd28918e248bebeef05cf5d13c89351a2383fd8 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:47 +0200 Subject: [PATCH 045/111] proxmox-rrd: add binary to create/manage rrd files Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/Cargo.toml | 1 + proxmox-rrd/src/bin/rrd.rs | 215 +++++++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 proxmox-rrd/src/bin/rrd.rs diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 69a68530..e3ab5380 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -16,4 +16,5 @@ serde_cbor = "0.11.1" proxmox = { version = "0.14.0" } proxmox-time = "1" +proxmox-router = "1" proxmox-schema = { version = "1", features = [ "api-macro" ] } diff --git a/proxmox-rrd/src/bin/rrd.rs b/proxmox-rrd/src/bin/rrd.rs new file mode 100644 index 00000000..fdb61ffd --- /dev/null +++ b/proxmox-rrd/src/bin/rrd.rs @@ -0,0 +1,215 @@ +//! RRD toolkit - create/manage/update proxmox RRD (v2) file + +use std::path::PathBuf; + +use anyhow::{bail, Error}; +use serde::{Serialize, Deserialize}; + +use proxmox_router::RpcEnvironment; +use proxmox_router::cli::{run_cli_command, CliCommand, CliCommandMap, CliEnvironment}; +use proxmox_schema::{api, parse_property_string}; +use proxmox_schema::{ApiStringFormat, ApiType, Schema, StringSchema}; + +use proxmox::tools::fs::CreateOptions; + +use proxmox_rrd::rrd::{CF, DST, RRA, RRD}; + +pub const RRA_CONFIG_STRING_SCHEMA: Schema = StringSchema::new( + "RRA configuration") + .format(&ApiStringFormat::PropertyString(&RRAConfig::API_SCHEMA)) + .schema(); + +#[api( + properties: {}, + default_key: "cf", +)] +#[derive(Debug, Serialize, Deserialize)] +/// RRA configuration +pub struct RRAConfig { + /// Time resolution + pub r: u64, + pub cf: CF, + /// Number of data points + pub n: u64, +} + +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + }, + }, +)] +/// Dump the RRDB database in JSON format +pub fn dump_rrdb(path: String) -> Result<(), Error> { + + let rrd = RRD::load(&PathBuf::from(path))?; + serde_json::to_writer_pretty(std::io::stdout(), &rrd)?; + Ok(()) +} + +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + time: { + description: "Update time.", + optional: true, + }, + value: { + description: "Update value.", + }, + }, + }, +)] +/// Update the RRDB database +pub fn update_rrdb( + path: String, + time: Option, + value: f64, +) -> Result<(), Error> { + + let path = PathBuf::from(path); + + let time = time.map(|v| v as f64) + .unwrap_or_else(proxmox_time::epoch_f64); + + let mut rrd = RRD::load(&path)?; + rrd.update(time, value); + + rrd.save(&path, CreateOptions::new())?; + + Ok(()) +} + +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + cf: { + type: CF, + }, + resolution: { + description: "Time resulution", + }, + start: { + description: "Start time. If not sepecified, we simply extract 10 data points.", + optional: true, + }, + end: { + description: "End time (Unix Epoch). Default is the last update time.", + optional: true, + }, + }, + }, +)] +/// Fetch data from the RRDB database +pub fn fetch_rrdb( + path: String, + cf: CF, + resolution: u64, + start: Option, + end: Option, +) -> Result<(), Error> { + + let rrd = RRD::load(&PathBuf::from(path))?; + + let data = rrd.extract_data(cf, resolution, start, end)?; + + println!("{}", serde_json::to_string_pretty(&data)?); + + Ok(()) +} + +#[api( + input: { + properties: { + dst: { + type: DST, + }, + path: { + description: "The filename to create." + }, + rra: { + description: "Configuration of contained RRAs.", + type: Array, + items: { + schema: RRA_CONFIG_STRING_SCHEMA, + } + }, + }, + }, +)] +/// Create a new RRDB database file +pub fn create_rrdb( + dst: DST, + path: String, + rra: Vec, +) -> Result<(), Error> { + + let mut rra_list = Vec::new(); + + for item in rra.iter() { + let rra: RRAConfig = serde_json::from_value( + parse_property_string(item, &RRAConfig::API_SCHEMA)? + )?; + println!("GOT {:?}", rra); + rra_list.push(RRA::new(rra.cf, rra.r, rra.n as usize)); + } + + let path = PathBuf::from(path); + + let rrd = RRD::new(dst, rra_list); + + rrd.save(&path, CreateOptions::new())?; + + Ok(()) +} + + +fn main() -> Result<(), Error> { + + let uid = nix::unistd::Uid::current(); + + let username = match nix::unistd::User::from_uid(uid)? { + Some(user) => user.name, + None => bail!("unable to get user name"), + }; + + let cmd_def = CliCommandMap::new() + .insert( + "create", + CliCommand::new(&API_METHOD_CREATE_RRDB) + .arg_param(&["path"]) + ) + .insert( + "update", + CliCommand::new(&API_METHOD_UPDATE_RRDB) + .arg_param(&["path"]) + ) + .insert( + "fetch", + CliCommand::new(&API_METHOD_FETCH_RRDB) + .arg_param(&["path"]) + ) + .insert( + "dump", + CliCommand::new(&API_METHOD_DUMP_RRDB) + .arg_param(&["path"]) + ) + ; + + let mut rpcenv = CliEnvironment::new(); + rpcenv.set_auth_id(Some(format!("{}@pam", username))); + + run_cli_command(cmd_def, rpcenv, None); + + Ok(()) + +} From 334eb9ce48c20b836c6ac6c7b3fb78493798091b Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:48 +0200 Subject: [PATCH 046/111] proxmox-rrd: avoid expensive modulo (%) inside loop Modulo is very slow, so we try to avoid it inside loops. Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 7a9ce94a..54cb8b48 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -153,8 +153,7 @@ impl RRA { if let Some(v) = data[i] { self.data[index] = v; } - index += 1; - if index >= self.data.len() { index = 0; } + index += 1; if index >= self.data.len() { index = 0; } } Ok(()) } @@ -171,7 +170,7 @@ impl RRA { let mut index = ((t/reso) % num_entries) as usize; for _ in 0..num_entries { t += reso; - index = (index + 1) % (num_entries as usize); + index += 1; if index >= self.data.len() { index = 0; } if t < min_time { self.data[index] = f64::NAN; } else { @@ -251,7 +250,8 @@ impl RRA { list.push(Some(value)); } } - t += reso; index = (index + 1) % (num_entries as usize); + t += reso; + index += 1; if index >= self.data.len() { index = 0; } } (start, reso, list) From 507f19dd33b6243e2f27a81f95cfcd9d31c4cb44 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:49 +0200 Subject: [PATCH 047/111] proxmox-rrd: new helpers: slot, slot_start_time & slot_end_time Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 54cb8b48..ce97c339 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -135,6 +135,18 @@ impl RRA { } } + pub fn slot_end_time(&self, time: u64) -> u64 { + self.resolution * (time / self.resolution + 1) + } + + pub fn slot_start_time(&self, time: u64) -> u64 { + self.resolution * (time / self.resolution) + } + + pub fn slot(&self, time: u64) -> usize { + ((time / self.resolution) as usize) % self.data.len() + } + // directly overwrite data slots // the caller need to set last_update value on the DataSource manually. pub(crate) fn insert_data( @@ -146,8 +158,8 @@ impl RRA { if resolution != self.resolution { bail!("inser_data failed: got wrong resolution"); } - let num_entries = self.data.len() as u64; - let mut index = ((start/self.resolution) % num_entries) as usize; + + let mut index = self.slot(start); for i in 0..data.len() { if let Some(v) = data[i] { @@ -167,7 +179,9 @@ impl RRA { let min_time = epoch - num_entries*reso; let min_time = (min_time/reso + 1)*reso; let mut t = last_update.saturating_sub(num_entries*reso); - let mut index = ((t/reso) % num_entries) as usize; + + let mut index = self.slot(t); + for _ in 0..num_entries { t += reso; index += 1; if index >= self.data.len() { index = 0; } @@ -183,12 +197,11 @@ impl RRA { let epoch = time as u64; let last_update = last_update as u64; let reso = self.resolution; - let num_entries = self.data.len() as u64; - let index = ((epoch/reso) % num_entries) as usize; - let last_index = ((last_update/reso) % num_entries) as usize; + let index = self.slot(epoch); + let last_index = self.slot(last_update); - if (epoch - (last_update as u64)) > reso || index != last_index { + if (epoch - last_update) > reso || index != last_index { self.last_count = 0; } @@ -233,11 +246,11 @@ impl RRA { let mut list = Vec::new(); - let rrd_end = reso*(last_update/reso + 1); + let rrd_end = self.slot_end_time(last_update); let rrd_start = rrd_end.saturating_sub(reso*num_entries); let mut t = start; - let mut index = ((t/reso) % num_entries) as usize; + let mut index = self.slot(t); for _ in 0..num_entries { if t > end { break; }; if t < rrd_start || t > rrd_end { From 66dfd1f08fd6e10173766ab3b493244c8a396c51 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:50 +0200 Subject: [PATCH 048/111] proxmox-rrd: protect against negative update time Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 51 +++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index ce97c339..f39b0471 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -13,7 +13,7 @@ use std::path::Path; -use anyhow::{bail, Error}; +use anyhow::{bail, format_err, Error}; use serde::{Serialize, Deserialize}; @@ -77,6 +77,9 @@ impl DataSource { } fn compute_new_value(&mut self, time: f64, mut value: f64) -> Result { + if time < 0.0 { + bail!("got negative time"); + } if time <= self.last_update { bail!("time in past ({} < {})", time, self.last_update); } @@ -290,30 +293,36 @@ impl RRD { } + fn from_raw(raw: &[u8]) -> Result { + if raw.len() < 8 { + bail!("not an rrd file - file is too small ({})", raw.len()); + } + + let rrd = if raw[0..8] == rrd_v1::PROXMOX_RRD_MAGIC_1_0 { + let v1 = rrd_v1::RRDv1::from_raw(&raw)?; + v1.to_rrd_v2() + .map_err(|err| format_err!("unable to convert from old V1 format - {}", err))? + } else if raw[0..8] == PROXMOX_RRD_MAGIC_2_0 { + serde_cbor::from_slice(&raw[8..]) + .map_err(|err| format_err!("unable to decode RRD file - {}", err))? + } else { + bail!("not an rrd file - unknown magic number"); + }; + + if rrd.source.last_update < 0.0 { + bail!("rrd file has negative last_update time"); + } + + Ok(rrd) + } + /// Load data from a file pub fn load(path: &Path) -> Result { let raw = std::fs::read(path)?; - if raw.len() < 8 { - let msg = format!("not an rrd file - file is too small ({})", raw.len()); - return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); - } - if raw[0..8] == rrd_v1::PROXMOX_RRD_MAGIC_1_0 { - let v1 = rrd_v1::RRDv1::from_raw(&raw)?; - v1.to_rrd_v2() - .map_err(|err| { - let msg = format!("unable to convert from old V1 format - {}", err); - std::io::Error::new(std::io::ErrorKind::Other, msg) - }) - } else if raw[0..8] == PROXMOX_RRD_MAGIC_2_0 { - serde_cbor::from_slice(&raw[8..]) - .map_err(|err| { - let msg = format!("unable to decode RRD file - {}", err); - std::io::Error::new(std::io::ErrorKind::Other, msg) - }) - } else { - let msg = format!("not an rrd file - unknown magic number"); - return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + match Self::from_raw(&raw) { + Ok(rrd) => Ok(rrd), + Err(err) => Err(std::io::Error::new(std::io::ErrorKind::Other, err.to_string())), } } From 7edea7e08c0c28fbe559ca09855cdf85a10854e7 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:51 +0200 Subject: [PATCH 049/111] proxmox-rrd: rename last_counter to last_value Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 18 ++++++++++-------- proxmox-rrd/src/rrd_v1.rs | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index f39b0471..73734076 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -63,7 +63,7 @@ pub struct DataSource { pub last_update: f64, /// Stores the last value, used to compute differential value for /// derive/counters - pub counter_value: f64, + pub last_value: f64, } impl DataSource { @@ -72,7 +72,7 @@ impl DataSource { Self { dst, last_update: 0.0, - counter_value: f64::NAN, + last_value: f64::NAN, } } @@ -94,21 +94,23 @@ impl DataSource { if is_counter || self.dst == DST::Derive { let time_diff = time - self.last_update; - let diff = if self.counter_value.is_nan() { + let diff = if self.last_value.is_nan() { 0.0 } else if is_counter && value < 0.0 { bail!("got negative value for counter"); - } else if is_counter && value < self.counter_value { + } else if is_counter && value < self.last_value { // Note: We do not try automatic overflow corrections, but - // we update counter_value anyways, so that we can compute the diff + // we update last_value anyways, so that we can compute the diff // next time. - self.counter_value = value; + self.last_value = value; bail!("conter overflow/reset detected"); } else { - value - self.counter_value + value - self.last_value }; - self.counter_value = value; + self.last_value = value; value = diff/time_diff; + } else { + self.last_value = value; } Ok(value) diff --git a/proxmox-rrd/src/rrd_v1.rs b/proxmox-rrd/src/rrd_v1.rs index 511b510b..7e4b97c2 100644 --- a/proxmox-rrd/src/rrd_v1.rs +++ b/proxmox-rrd/src/rrd_v1.rs @@ -285,7 +285,7 @@ impl RRDv1 { let source = DataSource { dst, - counter_value: f64::NAN, + last_value: f64::NAN, last_update: self.hour_avg.last_update, // IMPORTANT! }; Ok(RRD { From 392d646f7b1db27bb0e37c8d358203fb57d9e349 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 10:24:52 +0200 Subject: [PATCH 050/111] proxmox-rrd: add more commands to the rrd cli tool Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/bin/rrd.rs | 239 +++++++++++++++++++++++++++++++++---- proxmox-rrd/src/rrd.rs | 4 +- 2 files changed, 220 insertions(+), 23 deletions(-) diff --git a/proxmox-rrd/src/bin/rrd.rs b/proxmox-rrd/src/bin/rrd.rs index fdb61ffd..bf2817c4 100644 --- a/proxmox-rrd/src/bin/rrd.rs +++ b/proxmox-rrd/src/bin/rrd.rs @@ -4,16 +4,22 @@ use std::path::PathBuf; use anyhow::{bail, Error}; use serde::{Serialize, Deserialize}; +use serde_json::json; use proxmox_router::RpcEnvironment; use proxmox_router::cli::{run_cli_command, CliCommand, CliCommandMap, CliEnvironment}; use proxmox_schema::{api, parse_property_string}; -use proxmox_schema::{ApiStringFormat, ApiType, Schema, StringSchema}; +use proxmox_schema::{ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema}; use proxmox::tools::fs::CreateOptions; use proxmox_rrd::rrd::{CF, DST, RRA, RRD}; +pub const RRA_INDEX_SCHEMA: Schema = IntegerSchema::new( + "Index of the RRA.") + .minimum(0) + .schema(); + pub const RRA_CONFIG_STRING_SCHEMA: Schema = StringSchema::new( "RRA configuration") .format(&ApiStringFormat::PropertyString(&RRAConfig::API_SCHEMA)) @@ -42,11 +48,36 @@ pub struct RRAConfig { }, }, )] -/// Dump the RRDB database in JSON format -pub fn dump_rrdb(path: String) -> Result<(), Error> { +/// Dump the RRD file in JSON format +pub fn dump_rrd(path: String) -> Result<(), Error> { let rrd = RRD::load(&PathBuf::from(path))?; serde_json::to_writer_pretty(std::io::stdout(), &rrd)?; + println!(""); + Ok(()) +} + +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + }, + }, +)] +/// RRD file information +pub fn rrd_info(path: String) -> Result<(), Error> { + + let rrd = RRD::load(&PathBuf::from(path))?; + + println!("DST: {:?}", rrd.source.dst); + + for (i, rra) in rrd.rra_list.iter().enumerate() { + // use RRAConfig property string format + println!("RRA[{}]: {:?},r={},n={}", i, rra.cf, rra.resolution, rra.data.len()); + } + Ok(()) } @@ -66,8 +97,8 @@ pub fn dump_rrdb(path: String) -> Result<(), Error> { }, }, )] -/// Update the RRDB database -pub fn update_rrdb( +/// Update the RRD database +pub fn update_rrd( path: String, time: Option, value: f64, @@ -109,8 +140,8 @@ pub fn update_rrdb( }, }, )] -/// Fetch data from the RRDB database -pub fn fetch_rrdb( +/// Fetch data from the RRD file +pub fn fetch_rrd( path: String, cf: CF, resolution: u64, @@ -127,6 +158,80 @@ pub fn fetch_rrdb( Ok(()) } +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + "rra-index": { + schema: RRA_INDEX_SCHEMA, + }, + }, + }, +)] +/// Return the Unix timestamp of the first time slot inside the +/// specified RRA (slot start time) +pub fn first_update_time( + path: String, + rra_index: usize, +) -> Result<(), Error> { + + let rrd = RRD::load(&PathBuf::from(path))?; + + if rra_index >= rrd.rra_list.len() { + bail!("rra-index is out of range"); + } + let rra = &rrd.rra_list[rra_index]; + let duration = (rra.data.len() as u64)*rra.resolution; + let first = rra.slot_start_time((rrd.source.last_update as u64).saturating_sub(duration)); + + println!("{}", first); + Ok(()) +} + +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + }, + }, +)] +/// Return the Unix timestamp of the last update +pub fn last_update_time(path: String) -> Result<(), Error> { + + let rrd = RRD::load(&PathBuf::from(path))?; + + println!("{}", rrd.source.last_update); + Ok(()) +} + +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + }, + }, +)] +/// Return the time and value from the last update +pub fn last_update(path: String) -> Result<(), Error> { + + let rrd = RRD::load(&PathBuf::from(path))?; + + let result = json!({ + "time": rrd.source.last_update, + "value": rrd.source.last_value, + }); + + println!("{}", serde_json::to_string_pretty(&result)?); + + Ok(()) +} + #[api( input: { properties: { @@ -146,8 +251,8 @@ pub fn fetch_rrdb( }, }, )] -/// Create a new RRDB database file -pub fn create_rrdb( +/// Create a new RRD file +pub fn create_rrd( dst: DST, path: String, rra: Vec, @@ -172,6 +277,64 @@ pub fn create_rrdb( Ok(()) } +#[api( + input: { + properties: { + path: { + description: "The filename." + }, + "rra-index": { + schema: RRA_INDEX_SCHEMA, + }, + slots: { + description: "The number of slots you want to add or remove.", + type: i64, + }, + }, + }, +)] +/// Resize. Change the number of data slots for the specified RRA. +pub fn resize_rrd( + path: String, + rra_index: usize, + slots: i64, +) -> Result<(), Error> { + + let path = PathBuf::from(&path); + + let mut rrd = RRD::load(&path)?; + + if rra_index >= rrd.rra_list.len() { + bail!("rra-index is out of range"); + } + + let rra = &rrd.rra_list[rra_index]; + + let new_slots = (rra.data.len() as i64) + slots; + + if new_slots < 1 { + bail!("numer of new slots is too small ('{}' < 1)", new_slots); + } + + if new_slots > 1024*1024 { + bail!("numer of new slots is too big ('{}' > 1M)", new_slots); + } + + let rra_end = rra.slot_end_time(rrd.source.last_update as u64); + let rra_start = rra_end - rra.resolution*(rra.data.len() as u64); + let (start, reso, data) = rra.extract_data(rra_start, rra_end, rrd.source.last_update); + + let mut new_rra = RRA::new(rra.cf, rra.resolution, new_slots as usize); + new_rra.last_count = rra.last_count; + + new_rra.insert_data(start, reso, data)?; + + rrd.rra_list[rra_index] = new_rra; + + rrd.save(&path, CreateOptions::new())?; + + Ok(()) +} fn main() -> Result<(), Error> { @@ -185,23 +348,57 @@ fn main() -> Result<(), Error> { let cmd_def = CliCommandMap::new() .insert( "create", - CliCommand::new(&API_METHOD_CREATE_RRDB) - .arg_param(&["path"]) - ) - .insert( - "update", - CliCommand::new(&API_METHOD_UPDATE_RRDB) - .arg_param(&["path"]) - ) - .insert( - "fetch", - CliCommand::new(&API_METHOD_FETCH_RRDB) + CliCommand::new(&API_METHOD_CREATE_RRD) .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) ) .insert( "dump", - CliCommand::new(&API_METHOD_DUMP_RRDB) + CliCommand::new(&API_METHOD_DUMP_RRD) .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "fetch", + CliCommand::new(&API_METHOD_FETCH_RRD) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "first", + CliCommand::new(&API_METHOD_FIRST_UPDATE_TIME) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "info", + CliCommand::new(&API_METHOD_RRD_INFO) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "last", + CliCommand::new(&API_METHOD_LAST_UPDATE_TIME) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "lastupdate", + CliCommand::new(&API_METHOD_LAST_UPDATE) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "resize", + CliCommand::new(&API_METHOD_RESIZE_RRD) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) + ) + .insert( + "update", + CliCommand::new(&API_METHOD_UPDATE_RRD) + .arg_param(&["path"]) + //.completion_cb("path", pbs_tools::fs::complete_file_name) ) ; diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 73734076..96d2c366 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -154,7 +154,7 @@ impl RRA { // directly overwrite data slots // the caller need to set last_update value on the DataSource manually. - pub(crate) fn insert_data( + pub fn insert_data( &mut self, start: u64, resolution: u64, @@ -239,7 +239,7 @@ impl RRA { } } - fn extract_data( + pub fn extract_data( &self, start: u64, end: u64, From 919ccf713aab2ce81027d8cc194ea93b06c4c04e Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Wed, 13 Oct 2021 13:33:15 +0200 Subject: [PATCH 051/111] proxmox-rrd: move unshipped cli tool to examples it's a rather low-level tool mostly useful for debugging and some of it is rather "dumb" (by design) anyway, e.g., it does not transparently applies journal but really only operates on the DB files as is (which can conflict with daemon operations). In summary, not (yet) a tool meant for end user consumption. Move it to examples folder to avoid compilation on packaging (we do not ship it anyway) which allows us to move the rather expensive proxmox-router (pulls in hyper) to the dev-dependencies section. Signed-off-by: Thomas Lamprecht --- proxmox-rrd/Cargo.toml | 4 +++- proxmox-rrd/{src/bin/rrd.rs => examples/prrd.rs} | 0 2 files changed, 3 insertions(+), 1 deletion(-) rename proxmox-rrd/{src/bin/rrd.rs => examples/prrd.rs} (100%) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index e3ab5380..0ede97f5 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -5,6 +5,9 @@ authors = ["Proxmox Support Team "] edition = "2018" description = "Simple RRD database implementation." +[dev-dependencies] +proxmox-router = "1" + [dependencies] anyhow = "1.0" bitflags = "1.2.1" @@ -16,5 +19,4 @@ serde_cbor = "0.11.1" proxmox = { version = "0.14.0" } proxmox-time = "1" -proxmox-router = "1" proxmox-schema = { version = "1", features = [ "api-macro" ] } diff --git a/proxmox-rrd/src/bin/rrd.rs b/proxmox-rrd/examples/prrd.rs similarity index 100% rename from proxmox-rrd/src/bin/rrd.rs rename to proxmox-rrd/examples/prrd.rs From 432374d0242cc369559be2766b334dddc0f926f0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 12:55:51 +0200 Subject: [PATCH 052/111] use complete_file_name from proxmox-router 1.1 --- proxmox-rrd/Cargo.toml | 2 +- proxmox-rrd/examples/prrd.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 0ede97f5..31473962 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" description = "Simple RRD database implementation." [dev-dependencies] -proxmox-router = "1" +proxmox-router = "1.1" [dependencies] anyhow = "1.0" diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index bf2817c4..59e75d3c 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -7,7 +7,7 @@ use serde::{Serialize, Deserialize}; use serde_json::json; use proxmox_router::RpcEnvironment; -use proxmox_router::cli::{run_cli_command, CliCommand, CliCommandMap, CliEnvironment}; +use proxmox_router::cli::{run_cli_command, complete_file_name, CliCommand, CliCommandMap, CliEnvironment}; use proxmox_schema::{api, parse_property_string}; use proxmox_schema::{ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema}; @@ -350,55 +350,55 @@ fn main() -> Result<(), Error> { "create", CliCommand::new(&API_METHOD_CREATE_RRD) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "dump", CliCommand::new(&API_METHOD_DUMP_RRD) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "fetch", CliCommand::new(&API_METHOD_FETCH_RRD) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "first", CliCommand::new(&API_METHOD_FIRST_UPDATE_TIME) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "info", CliCommand::new(&API_METHOD_RRD_INFO) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "last", CliCommand::new(&API_METHOD_LAST_UPDATE_TIME) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "lastupdate", CliCommand::new(&API_METHOD_LAST_UPDATE) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "resize", CliCommand::new(&API_METHOD_RESIZE_RRD) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) .insert( "update", CliCommand::new(&API_METHOD_UPDATE_RRD) .arg_param(&["path"]) - //.completion_cb("path", pbs_tools::fs::complete_file_name) + .completion_cb("path", complete_file_name) ) ; From 4231e2d5f505286d5641df8f02aa27f8a567ade1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Oct 2021 18:20:06 +0200 Subject: [PATCH 053/111] proxmox-rrd: add some integration tests (file format tests) --- proxmox-rrd/tests/file_format_test.rs | 59 ++++++++++++++++++++++++++ proxmox-rrd/tests/testdata/cpu.rrd_v1 | Bin 0 -> 6008 bytes proxmox-rrd/tests/testdata/cpu.rrd_v2 | Bin 0 -> 82078 bytes 3 files changed, 59 insertions(+) create mode 100644 proxmox-rrd/tests/file_format_test.rs create mode 100644 proxmox-rrd/tests/testdata/cpu.rrd_v1 create mode 100644 proxmox-rrd/tests/testdata/cpu.rrd_v2 diff --git a/proxmox-rrd/tests/file_format_test.rs b/proxmox-rrd/tests/file_format_test.rs new file mode 100644 index 00000000..cecb242d --- /dev/null +++ b/proxmox-rrd/tests/file_format_test.rs @@ -0,0 +1,59 @@ +use std::path::Path; +use std::process::Command; + +use anyhow::{bail, Error}; + +use proxmox_rrd::rrd::RRD; +use proxmox::tools::fs::CreateOptions; + +fn compare_file(fn1: &str, fn2: &str) -> Result<(), Error> { + + let status = Command::new("/usr/bin/cmp") + .arg(fn1) + .arg(fn2) + .status() + .expect("failed to execute process"); + + if !status.success() { + bail!("file compare failed"); + } + + Ok(()) +} + +const RRD_V1_FN: &str = "./tests/testdata/cpu.rrd_v1"; +const RRD_V2_FN: &str = "./tests/testdata/cpu.rrd_v2"; + +// make sure we can load and convert RRD v1 +#[test] +fn upgrade_from_rrd_v1() -> Result<(), Error> { + + let rrd = RRD::load(Path::new(RRD_V1_FN))?; + + const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.upgraded"; + let new_path = Path::new(RRD_V2_NEW_FN); + rrd.save(new_path, CreateOptions::new())?; + + let result = compare_file(RRD_V2_FN, RRD_V2_NEW_FN); + let _ = std::fs::remove_file(RRD_V2_NEW_FN); + result?; + + Ok(()) +} + +// make sure we can load and save RRD v2 +#[test] +fn load_and_save_rrd_v2() -> Result<(), Error> { + + let rrd = RRD::load(Path::new(RRD_V2_FN))?; + + const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.saved"; + let new_path = Path::new(RRD_V2_NEW_FN); + rrd.save(new_path, CreateOptions::new())?; + + let result = compare_file(RRD_V2_FN, RRD_V2_NEW_FN); + let _ = std::fs::remove_file(RRD_V2_NEW_FN); + result?; + + Ok(()) +} diff --git a/proxmox-rrd/tests/testdata/cpu.rrd_v1 b/proxmox-rrd/tests/testdata/cpu.rrd_v1 new file mode 100644 index 0000000000000000000000000000000000000000..99d43d34b2d8d068f078263ef57673580263ace5 GIT binary patch literal 6008 zcmb`Kc|6oz`^QH`2w6&{g*MreR$1yZX`zxRw8%|Ki=}L1A4bWVPzs6cB`Qmav5l=1 zS_osSN!i9Sicq5G``q_Ep5OD={rltj{&T&)=X=g|UFSOI^ZuUmT3PCSMjS7lhllj% zqxqL1Jzp-{t#^#ML)j6uqb+-FNEFOlR(|wKmLsg4opZ#FJL0YAm)YveG9bQB zU`+U9CX8-V7TOqPp^|1Iym9I}I-OQFHECZ5n+5DH3E;@mx z-Ww`I_#>}xu`t|Af%RuiL#0FB=sU*>IP}^PiboF0-+1l>V=4A>p>{_!e7}~{fI=Y~5n`qCTBu7ZD^v`;I*bT;S{A4_d_{N(op44CUK%c8{dtIU<<{P}}wDKkD zb8Iimi|h;r=&>3qM7&)_ebNm?9i>N%4iWhqZ@;ldo2Va|Ji}eSmV)%x%Tad81&T`? zCN(A8G2doZ>X$Z0WZd*FGa$wpvx006niF|m`ekuEil__k=EY&eeCKyp{ywfxymv1X zP7bGlwoP_P;(Av|n(E!Aopb_ue`@XA@!8JcneqSJS7FBEr=>50%)9dzS;7ajxQ8BVHu!<2FSV^)$P0|a zZ_d<((ZHZ@I<{GmXiwEul~AXF?7?o_47DrTtDz{oD);g^N2VuuXGJH;~?)pD|Xo7cU z`H2(l{(Vn#CvSngKw)ZhyAMb|QzF&nd_iur-~ajF-(OswAo=)$RI;($+M4)o{;Jj> zhuF6shqX!@NT-Y+aL6$Bci_js)(F^`O> zS1Q`XxINErrU}y!uW?e$gMI_#KHrKcB0g<=51+roC6KoZ<_QfD?jV@Vf$;lY-k#`L2Wf$?1800THiBKHMsk$m4P8 zckN*!?$p?8naBW;#yX7^?1=jA_Gdg0A;vT5D_>l`0)~O*;=qz?)Ab^0jMrM+BH{_8 z=guT7NjTk(^^M5y`P|M}Vm(PhdJV!w{vgSVoq0&)MLM~yT#QWgA2v}A>ZXCFS6V7Z z)R8nwRq~A_+W#I$E_X>vx=GBBeW2pqIl_{c5hH!TMt|y-4QZq ztRP20r%roU;wB0dPSTGp`{<8%bY_U3i5rf75Dh4IpdecDk+!@W1w3AQmEPr7af2Z) zq}SyJgErGmCi^HDON&!$A*JE^{WkmTybwHKU;c5h!5tQPmj2zIpP*Aw==G*C1AaNi zFYVj!gLN_P#7KKG>UA48`F14W*V6tg-RD!$av)M#&^8Tmfq6H}$C6OC$hVG9ARbK} z1_$}I?%?*8=C`RjDKMN7AbfpaGDhSdFby6>z%M^5qH6XHi1F;2xx^&_3me7{HU!+k zjK(YLm_p7tU$UTc$tVSNeA!OwJ^py*bT;2_xeHXC-}?j<21D|&m3MTc8w4x6Z$*WC z!Yx%!xXRmPEZ?;7oyeXbh#q$EKi1}kNd2-EZK}=)v~RO+Jw-v`&V2g#d_Npp`lFo{ z>jqm%uUhd)XOz@!Xbm-?pvB&Q(UIf+7@K1^*e~FQk@fBig4C#R?k!lEqd`Gt?Dws- z0zbrF<874w;EIyw5>Z|!XAI`9HIuENAm!T<-;3#fP-}7GB&E6`V9C7_^B+!dJz*v= zJIon+#z8FWH$k`%s-=fLmjNs8V!KD)vfe{B90JHgkPQ zVZGZlul{LY$>?-mQ{NsAmRF*36z_FpT1xHiZp(*^+KhCj(QU{$Zt0M1ia>mYP>J~B z>(GhF3DT3T0Bz#ZS#_a1VD&cs%G8MlU2nzMnP2q?u-c$}d1fKZ`SvxZE5*P_RzRql z%|V91($Ng*R*Xohy^;th!@}+-sXm`_!AfJvFs^4|>rh}y{nc#HPS7{~wkd^v#eoZ5&h4mO)-yP}{ z{#Ag05RJjCyeA+x%>2D5_x?0@xIE=@Ft4UBRY;%+WLrHtnH>d(blOHF6zC>F%bXN>4E~CHL4^Gy$Sr z+CJeGNiaXLEBnH+o0z3FGkL&d%%5V1D*AZJ?vi_pXxL2R9@IyK6RWq~XMqc?xHqyJ76u8|{yCXyCo~ zJ!x`>2kb;bJ1dL5Fjy?SeW&>i^gMLQYYMKxm{8xmthfT`-zw5PtDl2XIoHbFXV|F-o2qS07qLD))m#zF>9;zf(>f!u=mUpwj$zn zIop{kQ{2%iGMK%zfd=uJGR^ir9+*%xnz?>H4I%RS!|mf&v35b|X_*VfxT^Y8Y|FtU z2yLij4+`JKvCM6iEo>?@O|Ga-4tSx_*Mfx1AVeIwHm~t>F-T2&*Ld!5hm)z<;La>> zype1$Uz6pA*!vCgY`Qo4Q!>q{`#sT-WH`La#v8T5c|}asaKwuYrENT3iklJ;rREgL0a22w0<qIX)<+;~ERfpGcAaMd*w z2h^Mwyk3giCFkd>5$8K8w#uNV6Ah-ceb#a_J<#?gZ|&Dc8Y&m*MJ!n3i9=yLWZO0$ zTqu^%vYi(O%F;@c%%&1tYphz|#_NuaDWQ#VMl@^&yD3f00|`sSQoBe#_!Z95mJ0O1 z$anrZ8`@~oy!xkowP&_LaL`me-tQN>KR(2PYq5q`(%k|?9lA*kkgtS!2-)kTR{`Rk zo_2o~E&z!(siQMcfVWGEKVH`=0MlQ;cFUp`MBI$b8Y9O4ad2+;!Irl$oQj<%a-tuetzMa!i~WXv!A*PRut?r1sG;9c=|!w-OQpe!*>v0rzr07cw+XE75ATh>ad33;t=m?kWhi{7 zt6k|=gMa`Oou2ayjAhFV3pN#lCK~F?xL<^!Q@bv{7cWHr`@>oaX-&{1)t1`!av;;z zzu?sFe4ML`U)=b)4CIzAt`!dDxPPQC)~cDPyV3iF@^;Vg#eSz{g&G6#r3tPqiAMOS z7a3&_b3i-2x~_M39-@L~%M2J+;GEpDYt5hOU<4TP9I7lv4NI`U+M5CM&1d5WZ!mDq z=-?T}hfKT^d7~M@;o$Aao{eq#xfp6?3LOfnM32F?hlO%Qu$#YO+57YoglR?eI#Tjs zr@Jz^*t2k&SB(4r&8u?3zuvjLQVcvhcVj6PbKlL6zR;e6)K9}2$MmSUHMr8Ko1KDM zLCbOF*;F*lo$Kb@@ECzh3T#%#cw%g)PN~S}MDR||JNKl?6JL_TI>k?K0L%(5=Xx^WjK5c*^CeYmQS9^n7E^>FOkuvbXT{ zWk;a$VjMeoRSG;tZFe{qQ!%w&@9u8hB;0La=9qd?5hqrzcGxWeX@?wnoQZl`o-Q#> zs(6HG(}ePvK~G4-YJXTw3g$gMx*+Yw)2(8<15TEZ*TKL}_XCU_%?OQ(gS zUt;iqqG?ta2Se5(O?z0?m=&(#`|s}=%>utV_^YS&&eb_rzg*pN^~lv9S6^IRarMI0 z373~#PICFjxa9U`o5NG`yJ*^KOD-r!Y9eRZ{X4fI~=2dod2M0CI zLFCPi3^a$T7B;@%pl7bp#i5qxPz<$axPKt@ZuQ;VyP*WTCf6M4tmJ^SF!PT(F9ufS z@9oiNa&Thj$~N7O3V0{Fq%e9oFqE~q%rY#)-nZYHPY(TUHSH%(^(mmK!TaWB=ab&;|}{c-p2?wX1N*SmUl{H3v!;Ss&Z9ULtX6 zoSPiczfH}JrBnSJ_Ag%zc=QqVn4ieW601b?x1T#cz2qR`cSPNc&?>Aiw-zNeb8tpQ z&v%_t^Yna^6`OT*>)wKAu}jp0uN*|JNctKfRRg2bLJr@F_2=FPY3c%hqKSJS^vd~f zUOkkT$l9Iy2uF?v7)eJzhC#AdsmIq`RK3_?qcxh1!6|jK+S)8gFJ4Y763NAb&#A>U z$%p7@f0?mp$1^l+SCY=ne}H=v)Pf&tAEC5xYgk8$ngXmwcd-V_*ATt+;c#EeIit|-zc5+%0Tuw z+p#?<0kvnG+h(p!LZ!0E>VpbNP@k;RQoMH;4ZC{Ubixy{*2yL;&^``Rt(r?Ct?ojl zsjY)ACLTi`vp&Ab&c>vFX7UndG{kSme~V$>LeLwwVYNjR?m2q-@GHkdHA9~5E*_0r zGPkWeHb>%o+QadmK2Z>8Np=m^zl}-lq~xlSSR`6?`ANn_WB%I6eRJibAh!9=fX>nw ztlf9g@}g!exc`CwwKZE@uy^spc*F(hu0QD-P5hsq?X6#)f^wB5Nl{1BPeS z9zK9z5Ra|C=pzg(CBNXUPespazmVjNWHi^TiwYRcfmPkB)03(N)4ckpeI>)!ysfaE z(Al5Q5Q66LMyC_HzFb`%B2mbJ#~*t>t<&OQRa@q^O$YzeZ-Mg3n)9S^fknQ>z2q+(q)ul3(L>y~xYpe}o1^%H&;?l~ z)}A;1&_ z=XpZG;K#duSlG(O7n${eu2J!wG|-HiK*sn+D>kH0=Rf=8{-65KF_LI@jq9IY5AOOt z>B`S6Gx*OsD*cT1%ON-$%;%ihUDyk+t^D_#nFH9sT(E;qJ|Q7OGNs8@Ns)vQWlYGJu+8%vWgaunv&_jnrKFIVA}J~g zMW!vzwZFgLIscsV-}|-Jv)1$6_kCUWv-YxT;`LYdd28pCbYt5GL>oh+48wc2c19)! zL=z)(3jn=8 z1<}s*ffehyyN1Rl2KGia1|~-T-vx#bh*owK_i!tNc&bzqA#KuEd{D8EL`bLM&vxSH z0-We(Jw){Ypkeo@;d2APjg4f`KnKWZwA}IZHsH;@@f`bHz;9Y*1=VE|A%o>v^RE-K z0BLoxu-y@Wd+S-E-#!C^Q<(?aP5{a-2Rf#@0fuMR*KC>G{;r&)e8fm1WFJaQ&oe;WXwgbDrT=YS;TGXq8s0nc|tE6@l6daCCe3a0@xmI`9ZX(U3<4zA3cC2hce z16zmUm;tAPw8GE62dGF_P<_I}a?BR@I{U~0d~@#4^KAnp-)S{`Qwyl3|8LmJ9`L@+ z@j9ajVAip>xpJ07$fXhzlGMru9R3`T{|`&em5QF^NMi@6Dx8aw9RQeb#_swa2JkEU z?e_Q@AjRa`9}z0R3-)Z|z9v9#55)%uXTVS2JId@5BtjmwRApPk0)XQe{pq{;fYWJa z#neWCo9Zu*CaE833XBcU;bT* zLp=cX=%l1YlrooZyJ*VZErhdb^(s{$7SQoQJ55d)kZ|R9xq5JDuA#|8^zSFzpwG&{U6iew>dI)JEv#@LnGR`bxxKP0^ z1MqPhtdtxAsLhyHslo-BytgQxTTw%+E3MJhkCFj{;w`5C?f@#X=Gh;z0wP)1-+o0c zG9CtAb=Qml+*l)?czXjNY?k&}2Hwe_`_L6p{u-8yiN-M z5~^!N3%>!J`KP~fW&$)5_D4q<08Y`#T-kLPaKO!C)de+8U7U;0n@3Snd-dAy7$U2w zB|VFDLkxh>iyRFKT>#5+#Rm;gZ>r)^Coa*e0D?4m#u|FNcrtMw-$Q_ z(6b>mvkpb2l-PQI)P-nMLTBE-Yk@CP9^7Salt%PR9w7-S!piw+zuWSo^_4_O`o0#julFh7we|2eNo7FJcdkEAvF4-zJ@YUB;J75?!O8&xm^w-B z`d0f(bcrOMm-5xe#sF03BJ&Lg~QKXiLu3GiX7eZX!G zxEoHT@4*GQxa)?>BxI9VcT~#ZTnFwWDZ`Z*L!PTdo3?qOh`dyX#cv8JfXv7-y3BCI ze)w7Y$xWy?C9d@}!Yn!~GrViQ_G#enMtCQ88XFV=lc8&HezZzE$%*ecz8OKa`m93XFxQt&H? zBz>Cw(qX#^D3hE2R4EsFW`_PL!Ao7efTYWugD#zb(VGWMkF~)CECw{BfqJqptTscu zs{rAyi(Dt30lI>|m4AeZvUbOPlzW7pkaab?XG9Vu$dVDx71s*@oT~B&4YUQEkk*(f z)d%pl3tt{p1RPePN~1gtI56;WLlBW>?a`^Yr4F@Z?U)KKb3^7cH!KbJWW(>7i+|7D z?;Zxsx-qn_z65;P@q>e_6fpX@$yovR%+awu&K+2_zRa0g_OsC7u2;?v`HVMaFYa-kZEC4!mgVRFkBC?4+sI=@z#A`;sUryD=9c)0#Jz96Sy7z%J`ehOMRmW zklG;an}n?-<68f%?K5$x#q!JpFR2}Xbc%p2KWo5s=2W*{Oq#YTSiQjs>!oE}eD7>o z0Z`KXv0BRsC9FNz?y!q&&rH9Af7Mq2Hxk5bGYp}qzjtH5d`31>u7$jwaJ7f((rVVm zSd<{hm2UzgWyo`q(*8SxiklE^CgE}Kw?PsiVcopG+q)Q`96R^+KYtS8X?7dO>uOyR zA%0^)`Q=Rsz|BKm&2xK5gt)w`WetH-Btq<#-IrQ!Xd_lVjp=zqG>H&X@a}Sj7Ftux zwl({bC0P<7T9YTIDD)7C5LK))oX@`mNIfSmy8~5@ir@_$8$Su~rJr@EM$e2QlCCgB zSOH9?(hDpe1GKs~FUu7GuGMIL;)K7V&SzFSu73xd2xVX#payU`Srk6w0qi$?E%)dW zfJ%vb!tgGM5V<1d;e6f`@Z-n_CKh-w^1~j;@=_zbUbw z1ITdjDyX1qMT*jReLa30z`Z!;PKA~dd0_lB%cuK*ov))xUm(wsq4vvnC?v6F79PzV za?^n1z86JLVvsssZ+4A-IY6<98^yuD$g0Hnc%DHPp#F)BYAzNQd--e{X`Tx3c7`&G ztjsvII3p!jBfv+w_LWv96wdJPu{0WZIes=HNI@AXT9P3Bjy^Ej8 z9rcAIXm#l4&BXx`yKUl1zd|i+FZwU6Mgh|9OUx2vp(uXl*eJgxKuO!$`L`Jmo0RSC zZ~Jconz-u1Zo@>W+~4;d>bVUtX(ar%Edvz9Shus#0Ol>zp4O1}CNbIG2RbhR4u>dZ zw~-s#o%aJFW@uk&G66Zm{m4cd&v|EV=^g;hqM~Ik`fBQ2-r;J-Y(Ssx@9nN(fNG{z z#c5|i;+yDLWyGH9VYe-toAqBrKdv{iXXG?PX-F=Z2lj_(4E?EgO9)T1t{3Z+(_u8$dAP?isCkUSkP8fm5y`&K!>!aK!0w9$(;M z4~bw;H+;fxa61{8B_hvasrI|o&Zeea2hc`bD5l1vkv&b%m0F89GVHznx>p*oOY!F5 zsV{(?L0Vsu9s#Id>;0xYK?Z80&HKohJ(aA*?k>p2euuXWn>cn@d&(w9-uRb*?VIjm zTV`Z9=X=2jaoKNk4pMqk1)!)7(-yfyA`rKhV@zYVkwHAso&-S>H*L~=4UjS7zmnYW zc8G+yF;|=-7)u7hio%F<0FqhFihdT!Gz9VCIcHr?LeiXET0cJ%G!u|B}+-QM{S zN5D$z@W#?%62a%%!H1?Po|xVBh20r$v=rjrTr3S*a`%EuAq+gmTcVK6Z~bI;*#IO2FR z{SYs?QQH}KK2_3T#^7|)mm^}B%Z|C1S(|MDV}5qtrw)k0k>Hwh%cn6r@$Gz)B@tlT zWhoc5GzcgynD(HX1+2|b`f`xXRAzRvYI(47OprIv8rygt?9{(Q+}9BfalYOm9v}c7z z0sCO7x&b-)i?3hWfw>6Wm>VNg!G8<1{js$z588u%3%ab6H%XzqbK6Bb$ z@*Ut+-Nib-KY*0JlT`}< zrrryyH2UcL#7{lDL;ZFFe(7=?$h!tu|0$aAE)*f}a+c%zjSgVXyh|&WnHj(xV<~** zHsIt@&kEgSfOJ{Z8E-TYdqtU?rW_uCR#&ZDrzyZhyX)jX+F7m3X zcbY_S>poTY(m0(&a37W~?OR82Jw9K~o(#bb;z{Jaz8dxmu$K4QW3&@p-CgL?&Z8)| z_x6(d5G$0xJ3w6OK(#UY*zV_LN!u?2!p;gERXmDKfvF*|3fB^Nw!`thkW>nkJkzQvh8YjFqZ{k7B4-H6>sHrXI2 z1M>6k{Qc;KV+G)*Y-#89CrIXK?8w*=q|VD=cH7(v6z=&$T7cp$EaPb%uF9B-DtpYo zUb;FPf$D5NkhO@G1?)Kx6($XXxUtuME&rZ_QVQG^ABxaIaZm5^C7q*1xvvxt9G#7T5OasPG{#c7|q%@?~7h-7vkzWf`M>G)c`$@ex2=a@P0P3`SEz)NE< z51|C0GbsMI3@dazqj9I52a<4P9`ufheGjoMYUG*F!om*SgFUC*;BJRPjg_7A0S@1akgw^0XxE2c92D{Z4Bh%PlM2z=mk(3dGR~0o ztSJ=imkO{LR=cx`&So#K=^|)O2jCe_+ZbyGY}b7KPT2we8Xuld`kx)PR*Tt60^T9U zj+~dyjRV+e-P^|wB@mTIf_qQ= zGwa|iK%5wBYOy|`D7?MrJv*Rffk?492pGP>eUcePA^s|QZ*clL)5w5)pz-lhk z?A{6x>iFym`LqozZ6Sfx@Nt*>q^`2b5kTNhW8&1xEvYGN!};u`lTu zjw53}ySGKT+uQ`WOCOr8{sQ>zoTa&i#rxiSF4QiE3i=K$R*5~1g7vP6NT>Ee&wdRS zulBN4z@8`bZn)^n0y3wDzGwHt!mLO1)qmXs1nYip6f16SC^8QWy1}8|};a5OwBx<02?(TDd$g|01B|Y3qSsLjdQm zz1}CWRHw_rpXZ2p#&+80`1^Ks40JfsO3}V?9FVz(UX0KLu)4*O;-dzTN;|Q;{Rd$C zth6I7Y~}Dy;`&!5D9Rz$Lq|Cri+3>Su&5g#HzL9PuKcXMu*|YfaJ04wpeZL|TgnC? z=y$&5moR|(*%zaHD5X7zUx%Do4ZQrLd$_Fl2B3oN`{CbT0q%w?!4gh@tHsnSPJ00g zQPrUzAseEii`g;4L4eX-_NH_$fb!LMT2$C>h^i;tf8U2}h-%Ctk2;}nqWZ?<5n*%y z;;k={HQ_P^+BG-NBfE`R!*CEHEa}#3$*2JH`Y~P*Ge8#h5@pJ*SHOP(q-!jWp^Fy1zra_AIyEuMV@PUe&`uN zY@Yg_ce{Ta0Z@&8l(CpVQ8dg73_hPhnbJ6tFMh8;;b!~8xff0VBp>>+^Bp~2xy4nT&!BEKK1>|*}C+1Ug#b`gIl*-3uXcivbl zJS5Nxca8z7TfGz030hm zt7_Z@TsW_hAw>tEdD*!{-W?o!l`hDVbOBEXtD?uK0fu_VryEKD$G@eWN`>PbmaLfr zy|)AER)a?j3jlsz+XODF0hDO$U3k%r9hk#s-|otW_&*&qI5dNHVV{#eXxX_6us(jR zp(qU?Ro=WWQ~|KA{qAB&8K1uu%{R=ckdP!FH%c~>%SWrsGr6|wsIRmKI9}(VV zDJa}|zVqFW*Vu(!ewo;fX+buwvy|*TKb}Br-;-O~oJi2&^cmA?S~Do~%OHKh(_v`U zy81(RbOvDF|CzkbD1>dL=H1H60Qhx8)~({n9=V`*q_}Zu%7>2-IueTF0+-8osKm*)i-*?Kbauo_?jp$jHP-|#N?Ij#R}cIUuFIwZ-{PktEne9upU=4{)C5%Scc1!+cBjx znBBP~IQ;kLSk%3*?#anv%;i)tsd?h5BfxhTw~)3pKwD_;_~9J@@tZG>eL@MG=$w3c z(?TJeuPHZUXdKbZ?Z<K;Fy6FCs{v5n5xnsB5a5`A#@v%hGDNj-2|Xvnme9?2PXPR;asHN= zoygx$`C^Y0KtS4RwrCF-az_s{Wt>06|s%?{CRh z3D*5#x7Z61;?q&uO}9;I7_^CoP5=Z ztO>U_UjZWY*GkODSBW@ReW68Bh$2%*4&;-sI(g&CRRwZOIr(DMd1fC#^r&IcBl1(<^YW?uZopbF zRRkR-b@;Hw^wTN>#ci-XD%XQZol?JvrHML1T0VC!lA@3(7o%=f^<_oK@9HIk5=}`6 zQsDVR+8RzsbkA{5qeHZS`IOK`w(Ywe8^FD_*)F|&0G0h@ zmR=Ad%~~9B)iVXiS@+!3lSekDf4vgW;|BzoHEh$P0q}jR7}ouP)V;n}@JzP@aQ}0< zziu9YMmJtZ_Yv}Z^Ev1Jx?W^Hs&p@N9U83LKt8FhP8Gn6cPVv@sMoFJA#7Y$O-CZiTx(ST2Op;j)S|09-SK}iT!R+pbVi_Bl>w=9V(psjG(qQgT(R9F+n@^= zWI5Hq1cf_R)ZRU1ejgC2_qe3_C%}gCxd5v*Kso1ghSgud2}Qa?FP#B<79?ZXH=(hg zQ3m^LP&kK9so=nloq)WLRogjGQ-?s$2^V{EmoVX>U3wb@xZELO#mxpdYN_|lDH%Y? z5TxI81fu<1Sux5>esXBi`gGInI3RhG=51dIz%5-|PCy)>EvMV)2}RkTnRz5Ngx@mu zY$4@kLM?E?s_6J}Uu+Y^!J%E5qo@wC(naj>$s2$u4xQKlY#cM+-^5h$FEK@Tqu-gry+ymrLwD6$WZ)& zm*_`^DVwM1&^(cNh<0z!HZrKDdH*AKHsT|x6#+LgH1?R^swKm+$tT}cfW6?>%`!T(Oqh}m>v7w+X5_zy-P3jZfh5{#%QEkt=Q!N(tC|^M>IjB%zn@sKwrI zyLPV*Y-O)gUtP=Z&X; zhb3$l7DRx;9dVtH=&LS-h&wUt(5mOtd(9?7^(aa-g*xSRNZmg);d_duDwGg(%JuA% zLlBa0-@YO8H&&!+Ke~yE7xt*@s&>C z!bfrEtN7^XN^CQ`H_Vt#EuFnRNC?n274##51D_`PJbuvglo4MxUKCS%p1uR$LsE>M zPR?7w5wj^TVbe8?>0fx=z`suZpmBr#`i~7SJ_L|aOQw z5#T#VZWjTFKlPN4Iql3|e4lXXKO+e>EH#-$Y)P#jSxuZ(@Uc^ai4qzO7$f^3wx^K} zY2|9Dd%Q`ANN>L|(t9?u@sHX`Bz#}(4TgSkl;F$z@sny&D9ZEimdgES@r6L}=7zNz z3>%?;AUL%D0xBpjc<`AT`cK%di+Nq-_q|W5%N~BrjQ|LRF1t;Sq3-FHE*w0V^f8OV zR+FI(8pz{gz6NRafNj(?+b0lL5b?{C%)>XKmYHV8%i3sPkMxpbrk{RC-{^N`4;T>x z$ZMtYv$&&yG@Ls6M*THFj497NiV0%N`6Eo%s|z^PT~%6pWei|B6#P#V9<+*X0U(ONrHy&Yxpl8V>X zx&WJd7THg%d%(h;N7RfG>L<`Zgbn+hRv{aA*280Ub~9)o+eW50e|y3gvr!8dtPY@o z4BS}rnaM|@qHnw}i{R62I!z+_ZQQ4mvQj-x0(Wjy9$!7efVs+(cRelmMIwC2nH4=R z(ScT_lxXv6{ueIgCPx_iR5}`T2grH9TjHHn_nn$AT9*I!1 z($U#C3$azb-n3EJFH0hnJr!f^KLd>w4tBfOwOs{NOs$smzX3#S*kn#A0v;*q8`b|#_0$%wj>0H$UWW~Q$(fk1LDY**H~A%BBb;6Q;9V3aeHruG}4`qY2_-kJhpuE+lVI18}5 z$1wHF5}@iFJo0ZIAQDd1wf7-lZ;nUH;U#pxpY>~1C+q=l`fc+GtAN~@U&+cyLSDc& zlL-9{fDy}Q|A%mH-X-BX&fXLNF4+OA2ygVvtvfgFX6yirJ?zjZhr)Af)wbD`UIxTV zo(_42ewgdP`Y^w+1)#Q`*Pf4tpDX(D^JFf3kxSpmwwbkwPCJ*TdN2bmCFiYAkYt)G zAm6A(H3g;23B0myl}K*O#sZ>2Ppbj4bY|Ic4gjvX^p+S&wB@ay(eX%xlRZ|>xE@{( zsExkH9ExU@9q-^H`a}fapjE3Bgd}9EpItNygdMWQ4j=dTM}D#yC=Anm(N44GCKDRG z&i~gPXDfAIUPh@B)m@U4<;Z(d-?FEBm=%2ei9QO8~I^e3(*RIiB0O5~P zJyo@U1E!CABdxHl9`B0l6T0v=s z1#3=mIG~_~DOltOpn{d&svIGg)>+Z1O2h-&+IC4i{{wh;a0lBN1;D_At&KV?wRG$y zX+nA(FwMTw+Kf%Abnf9|&Lw2NboteMP&*`Ay7k|z)zv$2_fFeyDsL(P`#QczTt~*r zSUEnicB2)PaS}&2R3Oo^W1WMa`VRn3a`m=qKtE-t9lElIUjiei6(EDAEIw8o`;PF@7hGm#fER?uJZUJrQucQHNrq>DZsObO zDIs*N+hKh&)rhoes$k^xW9a9(b^C;}9<=eC^m%$%40W&e-}t3Z4H>^+WLf++dIXRp zy8fUj72WTc62*!a!l@}Y-RYr*=2 z%dhkWBx>Z(rCb7h{wgf`Xgk_6-@{USn`VH;mUxY}CqUwFfYC1;Y1*6}W$-`+n_f){ z*3-)Z!iGKWr}+W2x_X&JUjdH1Jlfa>i8c<|&kg3ojE$y-Q9|!f!3L7@Y2SH7(eUKl z_(w)|00+OXn0PW4U&_oMbc+*kX?xdarwjzxKW`_Mhziyje`E`7$VC!XUpGvDJ`M<~ zyeCbGmQs5tohF(WwWui$DgJpCji^S(MeUNgB(mCTvi9^zHQ=6F^6wIKw(8{zvR8W> z0D(e3QvS*S4jv3yVQEEm3U`Vtok6>JetvNzOZzn{*gfIC;c*RM@b2i%G$^Wa@kK{| zn=-lF?3-+MP;X_vyJ+B1V}Rd;@p3=hU1@MJ!N}JQa4xgEUmVh^WZk*r;#VZ0V$oEp zGzyVcbah&C%H{*|h0;QnDgl0>1OGCc0Q!GuEz~*z=hUP>>=*>FRN2Z`OhSi?tU1;9 zegV3iKk_lH0`kV_qhF#H<$i?AJBWJ$`e{zKJWxydIm)8X0}xv|i_xUAAM#W7=QVSq z1d>qJC2&n>3XzuOJ@!b9Rt5OZRqVd31JGCg?)3LQ;B5J!?^(70mIErc)ZJm&KMsEF zROnHq??!6GE29B1IwXqJm6lO@&2zzMr4HkSd_@ggR_MgcXQ2hMy%QA+DJ*7tOh8%U$@_(Tm9RoblITApVLXpPAA zkGl?d+4e*$;3VMHvW=iU6X4A;jjemgM(NwzXM1m=QI~c-=FGZ?mRs7>K;?K6LMrW@ zpSi{v02tuvKD^HyFm$tgVN(e(;uqKYOBC?oxm(Of*s65w$6eDmB%3UHeOAcV(XX2n%CYvJF@T%kWX?F z+!F@S=D#E%AO+ZSYmcvLP$;Dj?($ueu#Piot^L>g5c=Bu}dNj$Wm;+e%70IQKVLde|PcLOF0d!6T$mBI4oGth7F1POiWRqKn z23HXBqlUx>_ffd|u5o+Q2YUhgHz@!vZ?4kY8=G^p0CTFj~zQopF_*M8Gk( z>TBViD3fJU#N{|c_^Yhmf{=m^`Fv$e?sPU1_59pFPO&1~u*&<;x={IV81~hA7B6k->+}%L@i4Hi0?Sr{2n*d z+)~J>hI-2y--I71hBnH1qlW@A5o!4oga6hNQ~^tZ_sq#p<`uVB^2Y+vdn=k-e&2An z#hQZ>Cl`eO3L=rCYo&y?zWDL})19cJ>!6}6tM{UdKNd+o); zib)DseC-bIHvQL#y{4W2+|kPmfI#8$Z>l%?r_+j>7Ok5@_GB)j5t<5zE$n_WDNQnlR1?PPE?XiSmS_P7HNSfSb+iuda}{8#4df|GS+h^tlBXznf3csRJVnW) zPxj;FDM~K7vPF@nc(#1fF*3UXU}9FkS-*~#GsBsbrUNnn`LV10KmAFBqFUw<8OpZ+ z!5e)A>Lc^{zr}yta)dAP?^+4BkiVzR>&vkd zB2SSgJMkipJVkC5+ZH8xiriBsBt}kg{2HyUT;Vk51Dv5;6s}^&@5)-0`SaWG+?#za zW%imPC4M2+|N5pC10iJ*&VDiZ3?*bXAs)eTz=fd?z7DUF&J?sjKlv6!E4jzfY#J zS`2R#iylbzDj|y4lHDS6U$^65A(iFu?{>nPqbo0Q9oq-v|DS)%5cZkKDq3&|Q627|MycuglA*XBEm zQ{FZK)N-qyIP#0GnpPS|!Y!{Xp2#8}Vf=FK;YS?tGE+JqyAJTAOSjwd1_Yd$oF(=& zVRrxhg6%&&@rILHN1E1v6Ju)o+U-7Lj9-aaIR6m9d-2NO=4Tia7|YqUve|?q)&d$+ zD-^iT-u$M3%K$&0SKKwj7!%U?h}}p9V?67od9=L%UNMI6R?lLLR}5uU0gVhU|1)7- zu!nqPf6~;xb{q+)-!q^@M?PX!WX!-%1{sFd^9BIVZ$ZL$7BR+)@mO=Sk|!Y6SI}*l zJm!C2?iHPp;m}GRbN_wY^%?RQTzpst;KOlmRU?diWWP+zB`S>Z7v*y)O2!d)y4Ldt zl>we%y~f=)n{bL&LQA{`;CW{MDaCA@@;<0IGUFmc4kxJ4jgkTo+A<;a3`eS0mp-)Y z&_r|S2^y$Y9yH_vGUOfd7_N?mn`xQwiONBRl~w zJWoU)FbDKDh3$@02mGX1@Mf2R3#cwAuO_3nBpvoDeZX}9aH@r2Dw6`BvfW0TcNSoF zv9~vG0N~>zCNGG&k`h}Z8=qmWq$F;}v2d#vp%@?7ha>+$a78-{G@G)m&CByRSmuX8w+ z_+08JZ5B=?UQ-JY(ZQ*NPu4RU2XHFE_Q`dZw>XusGp{7g7pI;kwC{g)4yT@;n6&@7 zh*R;co1=^c*yG|g+0RQE;#AzvQ>nTKaVpMJjnbzVr{b8bjq`$VDmM3VNB0GsianPn zxU_~-F+DHC4wm9n%)QTxGWT&RdUaF7m=mX>pB#JFIxM; z|6p=s3#TH-9?Ko6!l}rIxp`Nua4M4OC7l&7PDR9jeh@i^QxQV{dS682RCvoliBH#Y zDyrt^*b!4Sz1Z?bUnBougi{!CJU2rbY00|RJKMyJ&US;np2Z5?IIYu1(FPj55cyox>L=6eBOIV`M{k*Z&P4;}(bd(OD?^jBlI3VFb{oPh3E zu9)%ul_45q1?{j)oB(=eMaLeV`@h{`nScx3r>pS>rs}f26RpPtwqweIBOk6{t9{=5 z>22ZOOE8)zlfCPJ9d>+adCu#J*?@u^Tr_=Y@?$p+kzyq)FQuW3w!Z*$l*zsp}V3>$m$rwMWvHiYF=P=(U}uiVXpW& zR$d`2Ec?@YAI9TYPgbz;uTB0clu2?!bA}JIr%#5r4)X3mDMR>#TaIF`)YEnG#Ya?8 z?l+k<;xf2%0$0~3y7QPlf$!(DSKCGa!nB@V?RW-A5I?4JE|?Z@MviOcMmj*o@J|`j zS%92ZtHp31KrtnXFUbO+T7PfyF6K(m{2-CUi;X@(ca8eitOUS_d5G~Do-Pv1#dF@- z83P`wItEH@0UWI5FaC=KxP`F&YY_qX6wFlncK{xB)Y@J%0EB*fEW8g70|_xy+TZ&l z0Eyf}8S%#f=`x$TI<0_Q-EOuc_zjX!?3&UyO^&PLsdY%f6F^P%1$k>8K+_=O4sl2{ zp?z_(f$YJAcMRoD_49zf6Taf!&|$*Jb+!2`P;b&hb;GpZh%|*p$3`sy9UxV?tjQ-G z5=}c)_3GmaOqBM$rd#rV{zssShQZJ@=rF^xW$gR%97>?rK6UXL2}NOj^D`k0V$1s4 zwYap*jCz&yl9aA-!#l1++jCM8a_-d+v=o1(QCy}^47U}o0Vck%mZxZ6VbO z0>@cCV|iOhLUvxbk-QTX(JxA}E`|U;uAH~IngIRel4LuRa$z*PEybgMpo9`mue?U? zNr1mkk$KmBz)!zYR!4+fsv1!7lG`3o{iyokyA4S6aB$sWM;Nxu_eo-@jX1uO4~{EOj*5u!f-=ijW7c^-{DkIJ;|(e$cLNF&I%LD}I9ExXEYde~6`-FQ zXDDj|s7c@Ylk5lYFlCu|t->-@Ryo}aswiC5>wL*n^8Y#W{77N4$W97?>oclG0}jAo zsijga(o!u_-amc-#jOsiI%oO71@N^xW256K+1<6fA9JH9cwDq)cZ0iM{AnD`A6kaH z6KqHFpNF9V*qf_QOG0? z50ka6`UDXZxS-bJs`8qoH{gKC-4-g8vZjI6E~sAtU_#-4Gea1#?|a-KoBddPbx+=u z%nHDuy0&=V2LNsAJNxk_gj^o@iBN>pRcl-S-sKtypwim@{u-?Jy!Zms(}T)@Tl~Ct zf536iDHx=VSF<6#xoh%E{{H|fpYPPGLkX4Z9hUw_+W{G+9!iV(0LA!lmKF%9;;&cw zM7Rwg*|Z{0{}$kiO1sU8vncn%>ESb*oPc<)8OnFCdHF>e(%XdZ5Xt8T$%x0~U%p;8 z4>jEm0JwMy4_g*OT1}(VUv)15^h3GwmC##CDK2SSlRxb!Nm+@N6xj(N6ujTyM53NG zG_m;}xepm{9t)+Xi~*bv-;wYYA}MlRbLM-w0P)uu%$MgJh8;+6j0k~nO2Jvt(N0rG zIK??Z`IbU9yi-H@^2>3wrrb4)vk%BmjJZ<%CF^s@bB^PAUjObp@Zj^TM8*>2Fl%Km zLvlErk|k*$AYk(a&Lxg-RH@O!%avDbE=oXt=}XVPbupU*#94K;R^niMThHF9VdOBi z?8nuIFBZ`yzGk?PBxL~C#HW8Qq1z;vwyTVn!57J_x9WNxVDU-rKloaNAd=ZNyz zT7cs2e_45{Xgg&|_hU``0c^s3k9aWGQ;#NT=P$~5x|mkUvq+N!D1Kqm)nms)MY)}a z@_&dc?jY67h0$s3=AN{N3 z)0_>sabGjgBN!j3m@oP-IZxo}{WP=o&@#&Xg!;5-t&KgtSm@KhU-+}^>JG2afmO-9{AT)fVSSMF3HcB8?XLT*SRDNA&Oeg@qyzo@(a5Cd5M=CF>qoR_hO4)^aS#MSQnmC|>VMVX1IyuP=_RioB;nyg4 z56v`tgf#>(|NQUuW!NEqpYaY)Ei9wp1k?VqdNU|WspZHN7pzxgWi86$i=q^V9KIm` z9SeI_^j6tb5jHO|u0Jt!YZR~?b;|k?+F+@_If16u2U2IgD3|^VOD)T0Q#!{X2wzAn zs1K-60A9Y*vGVSRiPV#gc2?x#$Ye5A<{K}-gQV;W6I!@xBS~(2SrqUn@tZ3hnoJpU zVjvqAbXb~^c&zFa)>A5y&~fv66lSkaFkLXT04OBTK5@E?6+V;DP(OSIYwi$_4VC|i zD87gdcM##G!hd21DbthyETVVLzr|KtaPnlL!z#3qe@$ep>?k(YJR_0AKb0AgACD9I zye=?WPP}lMevTb-SS~dAv=1`QdMCu)PyUzwncoDB_6cB%%-nW7TV6v7C14l$=0pB7 zZn}iPvHGF}l<5Y)>B@*Ad|`U5Ky(M%P73vn;b8-`?xZtHcND7|P-QE{v1WRdDM6{- zdaeh3BjJS1jzeMNfPF7w6;#ec{0m|Njtq%_p2GU9_as0muX10+Z9sU+*KMjV05)t6 zC-$=dR3mA%`=Gj~VsuGCk?_~k{r)1=YT}UA;#TW4QzW3@>DK=KKY)rqGZ&&&0MQn1 z57d#3c;YmD67xQQhF->-K`Vgdh~#Pv_PBT!)i=DiP}8`Tw>oMp(0ttReqB!!YVLH+)KpTW&dsy&(1@ zae?0T_vkN^8GK?dNwcu}YT3XeT~+vI$04Ggl<{P=q3GN9^Msm@0+fL-XIY)q*e zB8f;$*sePPg-6`k&S$lNdPT6ADtb^jqg@R3JPf6;0>qq(N#R@ss3q4vJ1GL#Pc_j< zeoYtl-jb?YB_9ylFYw7=8lZIc#=H&k6ZSu^8uT$~xXYAHU^}wGtgb#4DukLQ2)AoS zW2woUyCgdk)}gUp>%J5f+&MjJEO@vdA!puFkY&6T1NE|1ZCoRN&71R)yTVTg&o+6< zVHORaF}Ohg-_g~37+l0{I(G6K1{Y5aMw^@AfwM%7E4$Zz6ONP6+&(XbK3~QhqPKgU zY|oXp+GDk7u;s%fmdkEv4iz;gZSP8BaAkr?w+96VSGgu1PiQ%b%~z*CpxOt6s|mDU z1};MhFE}p{{!n9Z%{GU`z5k1__YTMU4d2IQ_BI$VEO66-(?ea$!mL;`~L7{B}Vw;NU)_@LT4m{Ff&PzF(R7=#+<%^Ia}j z=D1?yyw?d-qgOHVo0=7EX7MO+$e9)WcmS;CKDjyg$sRms&mTA_wGWM7Dz@K9&LhO) z)st9aWkPiJNDSZp0uIw3tT8t}B}A|P)vqD|chS#XcJ2VuhjWC#Tmc{=cZf7*Keo-2r=J|ck3Mv3M`?E8r`T?Z$$>p_NGxb+w7uX1`{`eu%(GbA(ZN%mZKY(PMk*@g$Qs$4BK-G9H zK()mtyLJG8>$P{Domv2rX9H&x*a2iecdW5p2DtKMFYyMJ+K_8}*Y>u>0%&)Hv8Q9P z3%P06d*bmDfGIPBr22kJ{{|#CUJ_m4rbI|b(_zCgV(DCyb z19-j_7iB;W5S}=idHMuEoGfb*HTH)gDGLVgrrZIt0+YJSu~LV;5gwli=>jMk;`nZk zJPoOIz4cd`6X4yEblL+~0N%IE99p&o=(OVIZ$zCB8Q5=jHXfxiQ-IB@x!#@Fw1@mz{TlY71dNeI3#Ph`BLrj7+W&JyepE@c z=b5Xrs2jo2xg;VESb*+C_qDU2P$zRJxwU$r5T!hC4bNYPU#Z(lFQd$pAxP8KYUj~2 zSoM08dTyz7L$p$@^YZU3uter6{XQ;63Sl#~?c0}~fJ*Y@D)%Pg?~mM-wsW6CI*WL!~{8My@|60P{^i#?+a`AA`sITjUHSAcS!qxEMLk>=KB zLL#eJ2g|r$oPCiq1u!~yiPrnlQcXei-OCiRj`braw?q^{}>3f!yn-2i!FKM+(X2aC`bs;ZUF-5`0^Q|7g zSpc}pDfC7);Gn0dj9A4pxcsTWWnU~pnI|q-^3@Tw_sucQnc#vuD7oXxx^mXTU@S_m z^VbO&pQ9`#m;V);oNO|&O0!&uKdVyc6srX!@xqq(Mb9y6lnXu%R=+u{jOh~x!z2m+ zWv4~&+r4INfWWCxx@L1?6PQ9yympiD`*V`xMlZjVA-2|Yb2X8bxDu;5m_sbi4#2%O zPVJ_SWqorm17o8P!29Eh98~21v6tQW)u6ft<7oHQD)XOc9DR*;)%H9ZM<3zmaq&Xqs5N;#-}h)7)p$E3ln#xf zBHfzewa_@qEN&+=9*v{UlnUid5*q*4Yf{0B#*xF{V;il|I5L;IqoW9oBVCS>4(*_E zr1B;4=?iEa$!27=?2E<`-(Au++t4_oE@p^?9*rYHi)m?e&^Y3D_r(LrXdEH3{D@t^!34nLw-UpSJxR&z~<0X=Cx^2*iW+;>W$0bQ%fZLJDGf% zg~pb!^ruE8uQGY^LVnxa{Z*8Qp-jEp3Zqa=`m*L!GCA{q&@)G=cXkT)*sr}M@2DJG zhxl!8ac>QxhUcee98@~=9huTMPnBqhB3wwp_xpSLEE4U^z3(#f*aQ~e$(xK27)4gb z{^*^v9zepjiZ+WEqW+Zqb}S3nY{Nn?SeY{>(Skj|&A*94?(eWG2$2c)B>du9UH`Xd~6D zxG(qltG!Pjpe#3VD?fc3u7E4i^0S_Vr@h`$UXW2c2cGMl&nB`H&fTqB7T#K(30YZ&25wdzs-)!p!`N`&b`P#zJCXQJFXe*Gx6 zD?7jQKKwxiDEs`~+&lxj?$UQt?Bq(}AmK&dK-!N|&=SxRbRqT=h`L`P_gMy|sZjmZ zALj)g6e6+YV&-7jT)+}`uT;z$g#Yz9syP;cYSXk*Hb-TLY7;ncjPnNg$Fc|*bfMFTvcok^#R);gT~e&U=L7^0Pkq+RbH^Q$SWB<# z4)+51M0beuIJ6qq=;&b6^c3LLRA=4I=Kzt){ChTo0bEO;7>0xam~np2BRu&Vr}A9< z?>o#DC-yU3MK1w>-Do*+9jc4l-Kwx0n1<4{BAO=2lL;`C{QG$=Ds5~%)%8y|UIAp= z7YMB80fdfII|sZ4c&upJ$_093Zx`0FRFwlLv9VZcRRM_j+n0Z<1vvDhlhO|&i6t}O z)8S}B9sAni{j&T6K>w++E%i2l>I89xFP#7{D8izj^Z*3ee4Rhs5AbkQL97GU&9v82;keKWdiDFnoO#6&f+Ku%OTG|WLDU^@rfu+3 zM3uEEcdrgHb(_CyJHClO*`-fRkyjApj{ItyBAvuqEC_b)q^1%9OZ^O$RG;X`i` zg3{^3L6bp~3UEMM)!xJ%hN-b}2RI>@+ijoc3h-gTwuoORTZb`DYpF%aJ%4onI6QQo zRvf)P{FbkOy`TrcR;-{B!w3>&xZ1ueAf_$GVd}w~2tl*&uA!4zvS93`wL$mwKj84= z*t9JLrf68cYO%k>7GO`sw?ebyAY7QwUXu}Q)N4j>uQ#P27LO?0BOP=RomdBQF>dU~ zY8>Amc$1Tiq^{27T%n>zC`r5oPg(e(Mk(JH&?NjcT^^&$_oy35R5mTo8&7Txa6*{7 z=Bzo=+=-2A(#jB$XrwwqmWoi8(Ea4#?FY}rI#X;)lnRhdMhEMCNonN!_X?KK2WJ4z zy*OBuDTKuL2|F-2%nOZmxiS2rWd}H9NzW$3gi336eV+{BxzW6Wi+fD6ccT<+8?f%J zBtlsZ7Coqa6eWwC(pRhcA>P3*qj?|>xwG%V9QL(37?jJuQhQLkx zJmWK>BHfXi33|zsIJUMUH5=W;HCiyeBQ=*Y#mVf`yCXH9L&T=7)3778P{?{HQV)Kb&+1AF^Xa=&!g(W_)5VZRQB?0xQPzazCg@tXGIHIp5wZ;KzP|KrvK zP(F6$(M1M)l<)g=?q(kV{mcg~=LqzAB>P`u4BC7dNxdvmj5c4Si2TyJ(Pk;tpk?1T zv{@pnqVS|fo5edUax2HsW}(&5$M6!`ESwl~Dy-SJ7t1n?}Zh4sE9AEIo$!(PmoleVu{=+DyIXB`PsNo2k7% z(%hbCGif*N7LkZH6N54}q!nm0aW46f?I7BWC;mM@xsEpD#QHAR_M*+$$A^xd`T}@1 zCscUTFftd>(fD$@_{q#C_A4#$#iXZP9_m;gunAkjqqppL#l486_jgrmj1{Rf`;pLS5wU@|!t`a@~z58+X)b09Z**J=lO5wd3wq z&T*~)X}`05q>BN}oMwY&mI3%SIb#BlDK(>uRdR;a0JmpO-^dZfa! zHKFHOao{yE03Bb!lz1d;@q|Za;T-B%aiQops(dWI#XeueZaIU5;u|S$W%4%x1hiWA z?Bhg*{7b#ZbQF17)K#Zl^&%4>-64hk0ls7vJ>*}oBs{EIba~=rJq6e(Iuz-|)QzAN ze!EhCKj8sD{VvKT^V0y~#pKP+fj4<)HuP!~f5RSq@U{{{MY7 zi%Xp_otZy;_L%KF7ql>kyI8)Gf4fU3j+=yjoKI5_!9w_S%kDbYjZyqd@i&Kx$PH2m z_Wb_Y>KYUoN=_}VXFQK@8vZm_q;N8FoSu z;XB0j;4Y3SPvd(pGA%ytsViHaKg`+tSL`AwL zdhXCe$Kl(m(}Is^!H)yUC8H6jEo7xAGeQdD2)<`d(ULjKI*%FO0&Edn9NCq45%|9cp0aBcZfHQxe1*`@(Q z!HZxlYN5I36=aj5lYF^6(P_@0w);-sc^6traU z3iX`-uNz=-PV>leBmhs>_cK=v(8jnjKq-qGy;5Fq8<8R#Q^!K;?2SqQSe*6ztKBe{ zw&~AA7J1Abp%z;32;ojjf28~T zp;|?grCQv$><-`E9wvK|fcfw(rT*;76!2rqN!{asEl9|hpEAErz>8qhR!1&EHa@Sr zXb^X)0X!Fe{qUu98q8-!i%ANAdGC0cw4fvq?h*8rZekzO+l4}K(}WUP?IlLKbm}YI z@m+kNBdjrl$CVR4zZb4d$H-Zk@yz=Wf-quz^;o&R2#Ov(6|>_Ce&8tjP#^oM6W>PY zJe-&(^5D*$GbiI1g5hBO@&e1{`vk3C>%2;M+9#z#P=IDI8vM-v=i9c5r7@j;dMc1` z9$&_VS}A32GwjT);^JtfuVb$k=djVP1M0F`u3ZbX6T^4B_2aLV17Tqf=YN#6S&aB{ zt~_nhUQdqA-{YFPz>y8?%(CKGmDgvnLF{vIq@(L2T%p<%UWIVKa!Sf#zIznCu8K${ zGb>>nww8gTEl}^vReh@*hj`4@uu6WCs{yk|`6Lu~e}O#~ET<(rpw$eCH?8$2Bw#b0 ztg-y3+wfvyaHlN*3t>v(y)bq0xA4@Lm~U{7AiuZAy{ia>$7^f7ZSm7UOnUQCchjry{4e|;+yJcn{6?IQe<|I8{b{-CijVo~~aTZk3oc#%kJ z%k`nVbKKFp6^HzB-{J63S;pHOd}PjE`E>|Ed9rR+d-~5K+?VP3`km_$l&=7~^`@rF znBsYY#0Od{WK%H1h1nd%{~#OP8FLlAO3a?>xA5aW!W6b3U>W6!)QdRL@%3CdsEc&4 zwfg+R4>ngZe)lRr2QNsbfpKOdjN+UD0~@U5j7aeta8MReO6t5i~P)~27HDFi?nbLReI zA*@n|mbb4OAHd357@~cf68T=PBwYL`p8>mysebyv>*2UUGC}_NHY!Qck) zkMo6Qw7uY|Q@+3v#x_)A$9x{cKE@7QDQF(Z_O?NrLUU!eReJO)-rcBqvl!!)yp-T; z)5geUm){bd-Nszyi^ux1uorRT)!xlT*qXaWG|t-o#)iz5M7ZG{55nYPoszas%^Bm2 z(Rmm>E(EyZax*Uj3F6$joVt+_fi_w)#aWwb|)T!buaelVEYw%A%X7BFYNHrG#)*C2SwHC2*3PJ zn~cDP$xg@LM@cDvz-g~tXNAp{8WB0eeiZzI6Q3fxOyV&htAh&5eJCMqX~#i3+r-9L{B@vBnQ-M%k^uc#|#Qj{&L;B3zNp{1{SgG%p%;3C-=SOwnfTxM;DI$ zus}8i5nZROMJR)>m~DwZ2jNfYl0+K{!Lyg2)#F3TPZ1{K$F6goD+rT7FuB2#6p*Qv z{=Dz38aQ+>W4Q8-7}RCA#n-15fV$pEE^+N!NPM@)nuFx10B*me6C~!?(Pryu;QKm3 z^xEik*NQ_rIIkVHnX<=1_ISnowzUd;cUmsc_f z_2Zrx`PiCYc@$W6?~gOQO!%$b!%IeNFdB;Tdfb|R_bo<#Vt0KnQ5!0W@7)QP@OdT# z&ct%@7%|8U(BIIiI<$&vqvifiP0SF~ol|G=K8wuF;UFg$;XzF(@X~wqfN*wOiNooy zABZq=$VJdI&!8qZnF-4Q5CK-HNF2V1UCct#w71mfR*=`mMn+4Y`0KN}p$`f{0?&lsJ`#N6>;OgOCDt`$d{dpRh zunSq~MzoLry&a*}?TV>gpOePOBph0tI)s`)nzf|dMhADuh1?%YZQu&y8&cnoeQ3jT z+%Nok5qgQ`QYTM^V;lv(;8*vdI(Gq$r+U9%0u+X|-o6-vc+n`kv;<8eUZ($@I6f+X zcvZg}45-|^$S>EJ zmrP}0$giUMl8YVz$geL;jaHYvkYAyccUt{0PE}yYk26^zi0KyHzCo!>MEUl!QxE$Q z?w9fV(gjCAUHs(1rqTVd*-1#8;Z98~98@8hIR6eSk@LOwS9_Zt!NF%Sk7nKz>PC*S zY*`yb;`ZUhp68v7@XOA5S!j_NSH5bk3Co>Bo0URp(oz}p+Sq%fpAY(R-nKgTB3=a} z|M{ryNQ>@|Nq8+4-SzPiTYS|{38AsX-aWa?h`X!yliLPIDKVhkRxzFKH%y^WLTY#! z5p?fgIbJ$F3SdOJYEO8W+^zqzmB};`-*wUX=jRo;?D8kig?)uiVkSQWAvW%uHb9f zm`S=hP)y#zYLy(4ODp1zv?5NMr1yzJS{WBJF%P~*S~ayObMRq`VETJ9`>Qc&imJJa zT?`+7>WYl{{aO7mGG*DM|0p?ZE)H>hrE}X8-)2Rkyn||;cku9bo^B&61W?}o@XH^a z7OY>5&{FgvtY4KBbk=?MvF$NQxS(;=8wD*>yZO1a2I56Fda|WI5M*-Pwb`|Rv~rgC zm3Q_xim;3Jvu}2LoZzV)tKh36@8N0Ct9!Fzs0r?xiTyzt!njhuZLrh=dt8T0_LR!y zpx3Hxy>~-90Hu(fn#^|$$nlk%wQe7#5PvzaCyk122z!P$&W7s0=BCj-p(2!A z0`%5cbIVHZ(rP1U zL;HgGu%H9L=qak*>uZSD4e9-CULHuReq|QXy@+yvy513`#CT*=rv?9Q0R$z4yYgV= z=qR>WGZ$G~j-+A*=~8u|+=ZBCo%pMNVg<^46Zl$Ia{@*9t;6exs(Cvk5jXjNs^i)S zQ=h)`)F7l@oMPwx^wa{<>Ry1i&~tRpwY^05s`DxKqExnw+fQb(Lz9=02y{T;9LF35 zk_b0LIIUkdCSg^DgVd+vOp6NP;F0~^ACwUtSMgQKlWFc)wBdWn<(C`Da1*Q2`vAO!?>w^>C$UZ6>Es z5p5>CQe+CnFzL5_dP9Xr(fwBnlXc;Kj7)i=m!gmeb5VVk@3`?3S1#LDy*k;9xgPT_ zRlGr!aLFAz_~AGdSO(4P9GNP=W7%TP>Fx7!MF?c#$gb_L$6n&+IJ>w>6MpLI z4^0Txox$&7Cr|T}lA>z01`L!wd^L~X&JG+Hd3bFXekn`6{>OoEZ(`%|Xby*1sIInTSeRNA4hts^rQP{PA!yuE7an4N*oyR{0b zS6I+8-pGft^H$YNqAC)>%`+?;Qk&O71(;-7YPCaBXAoC&tusM&iE>Sq=N!i&Kfk?# z&+2!Bx}#QJk9e^}IzQ*Pt|EmQE^G4!yFPa#7J3<1m6QE`5A}v|^+Sofj7c zc)$GL)68u`qOrJ9Fo9loQH3=xf25V`;Fq4i>(GzOCHd;jL0?RfpZap80`YQU{T;rT z0NFSV>%5X!LBc+cENKk7q>0(p=|4@xK!+}LHXFiUh5XJHbDOQ>D>U&=f zLXerHD(S%j14fM#Z>UsDsss z=h2aKjg&~MPN9%*Vv$%Yl&GSj}< z4U6ym@2h?Oz18O53tj%b%jDl{JpR4G;@`^~{=K*0e~;|{zo*szJE;BN8Rh?u=l*vx z^}j=z|D8wtzen-@-&1J+9l-kU?9u-{cJu$9IQj3e#ee4v{ySpt-|2Gy4p#ejrr5vZ zwEmqW_3sd+f9D7Nd$seww;KO@q3^$U+5UTt>%TXc{(E`nzxP)Ddu8Llw-x?-QQyCJ z)ct!soBhV2nFBRgZhYs@+7*x^zcOE*7FN%+M0&2f4@I${P-k3dbsG*wl+y#5xDS&R z`tVA(aXlx2L zu<@ki*!yHZNedD-oP=}d`mbD6lC|2Y+><0I zm9qw|L?$}eiVXZJ)QHK)m$k6d;%f0okmn9I|80|2BfoCs4yWF`i=B+be2!v2jQ3^d zdi+BYHR=h8p;FunXx?)+o>g`a7xMl6z}GK0B~r>ohsWjEJGktjfAGHnWj`c7&0)=h z6@d_c)sbX9USy^JJ>I3n2$YLwtb(Ew#Hf&gZNi>M?q|cE>Uh(FW#nKeV-mFn*C>j) zRZ9N4E~Egk^roLzibU^S^^!2I{xAy~Mq-H{nh z3cxiis&Wo~CFttL|0I<6_>fN|$zEmftHr?LGEXu*{qwnVTQjJNMO3s6y!5PbNT|_5}&a_+NFEyyUG=F9$^#`=!d?zVTZWyclxyH2IKQx^4lvc;7(LcP=hKsOmf|IuWW`JzMtaA7^os9uptxP z?Q0Ewm~MNusbR583GjZ+3y_kb{zn)LFyx0QnX>X&E3PgDYS$k#H(Xy2b0aLsb=MoY&>TK!y68y0IpwdIKAV z)bj`E0$#*pE-&%Hru9(R^Ik8N_##+MrgO7AAL0Vzb7aj|s%PPjnE#Hs4q_4auH*UG z1`LZ;Wnk9~%>qB8It8tQ;b6=>`I5{BcaXU@Gf-xVauM#qZy`hsd%|d}e`O}hAoXI3 zgILcBLh46K_dVL^d_O*?IrY z82MH0aN*v@V_bQ-6*IR7YAdZ}#|i zI~-KCO{d=tNyNl=kLT{^hwlrgSY=s<5bo0Lq6 zhlP`WJN|Gsz~+eWYG0nBm`5v&_YAMnLg71In(uFbpNQ9W10{8cPPkFQjZEGhutA?X z5+0lkewxGdJvU{+n3LCMo163CP-nirCI|@|Q2V|q-T<-i=g$7%csv)(KMQEr{Dng8 zwR*SvRI4PSqkXlXE)KM~IP1S1>_b|4P&zdp*}s7{T1|O3st%)yrtOOhHpLXa-wb@3 z=@6Z#=dx{nmqLfnd^hw(jLI=-uZ)5vB_t6XEPt?bm=uN;-=n)Lhz(5WXc!rV7h(}k zT)Le)Pz`r@$G%GnT!Vuuq%%YA2tkxR--unGBYclj?b<^)Wi_VGp()1|4#v*KSIYH> zfH9i-{7If>a40C(?l4qqQ*UdB#6i0RO zcY6~FS4b!+o+6u)uN4d{k-Q?Ld7~!#79sJqxz=AUVfKWR-jSz1fsJ^|Y0-*~Xv{u+ zaX_{XagV7=d)W527#2p6Fll8VVWX^Xm41%ug%_9WZ!t`QSVSijaD*Su&HMZA<9cG+HgiI4c-HooA!j$U~JPs3k; zmS-XvDxTbWm`g8+QjGAgZD``hyRSn}A_Nl@9Y@umsHh{dC*${m)ff}^@O_;{C>Jj( zH~fP!a>Bya4I-6N5XH#+slqn_9Eyq4Zp`UJ^V-*?Psu|PX^!?b_cA}DYQ=`7z9gK5 zno*J8Gi_>!9mr@W8Cw%Peeru)RQKovew<;WHj8|M1j&}>i|(4imysMJg%32Bj^GQX zr|FJDI;eY<_C&y&Gy{L5G^FX*7@H8TkQt21>_!_8vzuFNP;bu7_eKoAk&M}r{I^aG zuVUmw&u^=gMqsY(Z^m~c&tUc;H4}f^QP@-R)-+Si4HiZ+n7uo}37hR6nvYXA!Ha7{ zcYZA*(Na#Hvpm=Wu_e5zo#Ul>`POy{=I zA?T-n(emdc5-m)p!{wRr4VW=j7{osF6Y{$ilU!s55~5~Z3ob7R!)A-y@5U^;q26VM zz6XS-ec~S+pQErsNlDn)hgX9U#P7ICDP)7KTCywNXyJ&el_EXCm|HFjeu#UL9ClE0 zQyZFiFPxhMbv~u}RL+o1xwqKj?t>#moTre-}no&T7wgB{IxV=M7fVw~t$9u7T%-rN^V45#xFK*%+pDZ^V8|YWgP}x-U?`98$U8Mwm@1i{u$BUe zhHH)(9$f*?5%v7 z_LN96mwhsYg~>VX^|uH%KRxLAf$%p$gsrb=kL4furMI>?@_huJ%H2wwwgTpF?1cszScW}C2|zpiXcxL?`jjLE&-xt|b05@nK`nTVj5#U0hN8`$za>GDos zlqbgb_cKWgD^Vg4KvIQh>zpwrCGYpjB76Z4y0_Bl<=X;#7JUNWSR(a857VD!r3V{f zYAGX6E04pSCnr2Mjpg8AX%(+2^-Z|^MX|l7@FCb>8??Wm7Yu&XO%d{Thxt>DVd#08j`rs-u$ssbUu2R&1?DyRbbGF%3?|F!?P7OI1Q_G&y{?>t1bMjd zB9L$%MC#r=r-qGpxY9}X9(OPS1nM$XRbqFMqLgP=&V~w^LPb_w>;V}kzt<%zn0^M} zo2T(cUKv1+EG;J+IhahUvzkTSjY*#vdMSM71|X*5xR9O=dt8fZ&RPf}?mLG1!Y2v< z9#QV_ub+m^Yeg|!<rtD+9R108HWJ^bdxW~>J?xpC&x*Tgv`GzR+ z3P>0)QrgmO2hhK{xXN)2{0vk2Ezb4>Xcci!%IYl#a5?M4sja&-zo8O7E}C{j3`Nn*Lh z1l1<}q0Lm0IMkBfpE;OH9Rk2vQb2Ql3XQE=0v_E5nW^PN##70@0OX63%3Y8|%C(&X zwARo`3#)l_ z`{aJrQcxFVM$*K?;s|B_J^JXk>tSd$N}j)G9Hlbsg!x$ZMN?#4^V1ObOiC#HX0Gh@ z-Qe)~x431UesCDT_F01>ZW#%p%2`%uggw9$cT+YaZg(VF-peO6OD*`}u5PH${uV~| zbdWyi?SPbdLKUWliHc`T5T3Yu_VHP7&NB!p$Z(v*{@Phgx^pmw zjd0g&=%sf;-d1QF7JRpPLCqYdjxgR-7U4%W9j)1T%Lp@~tnUOK*h2T{iv35OM;+iY zS!Lx!4aSL6HPw(yL3+l=(Jke8Bc=&6WnO#Cyg{ANZDxB#1(12(NBb+I55VLISCa1> zc=kK-U2W?!m_P1aNos-oO6?xV-e{ZK|gMhhcn_BkDX5ZOGvBZ__fn zz?kn|QhPmHY!oedPf>qGb_cWw>Ym_)ME#G7c&ica5b*Il`^xs&BhO9 za(_!Uq)`t~#i|u*ferUzgA1Ngsc6G;Ux!bF1A)_Xf6#57jKC#?1UR`Mrv788>6{_) z2prqn$`6?=nAEVL??HJi%urqYNn-|=Lu_oljBGw4f+4@?iku;&umbP+|9lkTMGpAKL;?=?`BZ%h(OyMKarNyg{*w(L8bERT+X3YhNs=#=ij;n!*-i@T zaAf<-L0OPFp!W3lq!+~hbNGPT7pOOx`lMR-jt4-zebjFv*qkCzHpWASxF^T|bpA>~ zhz`wZmbbg@(QA9gZgqbm#u+gzaAicPDywzUtIk4oo@#EIi*bd-li>Av&^0nz#eu#h)6ZYsX~Z*XOf|WBfniX-(0ngK&DMAl^jCdkwMc;!luS*#s51`CvvJ&h{kBpHHWT)bcUE0BO(=kS8FG5=RvW0~E z!3!@1M(W%^@Z<3xwQob%1H>zwoUH653cmZ#m9%1YnBjGFTsD~Y0(z;p&)jKQ!#EyQ zJ{P@SAYN~Bc~~WNAc;@0%^O?Pn4QYsyz?z+2|iRw+9xIs3x(fm$`Br04^_<0cujcZ zAj~LbgQY_bem#nj^UFoej0g&jT=(BZ>Sg-Q-7*7XQB`hY9kTE=y4S%^f)iV{n5BDz zS$_?{A+gCZhEdea_=9?mp5=SNvyl4xZ&8+DUj7QRo?Qd-!a&C2{WT~m`E2~BIKs_2 z$xTd8x_rz5#KUhApQS|`9d>Jz5mWTKz{J@slaFzD>H2tIg1UG*3g4%Dp_bV7zv4@e zLNWX3#uuf}gk|F0a^h=wURantt48-4c@Y^hMqT4cGJ9*cSdK)4f2ojH35pt(Iii8_|iL=4yHg)j7Al*uU=^_5dEI!eywW?;&1) zG`64r$A_%^!0?u~YYqcO^jikUrb7}&QC$0O8)5e2*GIzzz)$e6XNpg^tzhA;yO|%y zT48g`RTAoV{BTF`_|2;?kWG={G!=mX^Kf}<`9Oy?Xoz8z+uRJzn)!ngxpb?*+0t;!Udz*vrVJ`Cj68?3}CL5@JG%|g!2POz$w!U(*}kJE|JiuOZ^96Ph5m({WnY+FS->hy&L%++pWJ)#d!!`+^fHM z_8cM@{U3LHiN*(bn*MY{#EJvHOMO$x_eDVq8&Q8LnA`z=97?f6wVS+rh1EViHW4c9*?_U76k{WkzewbXRqx? zm2fXV%hWUo*?4wVha0~DW8QNY_YgZYp!@IUfaFUp7?56xLXsYa1@Luxj!d;fEf+NY zI2Lqc_Ul7V2RwQa!4kpk;jFtbtls9#%b>3?we79LqlZ{Z!Uoq1zmVwnd3;|$nDB|g$!8HqRIZ`c>Y3diPqWdBC64A0Adyl)l zf$H=pXZ(B#I(#TwHWhRmUIb?bN=BWBU)j5F#UvrhVec$N*D_b(`)Dhb%4^6tWaIef z?BYxSx}ZX`YIhLDyW(fqgklxwomuuU-W-7|vAA_S7fN`#C`dU~(Fwv2t-Ou-=#A7< zDbiCJIfJ6-=}1Pl_yv2CI~~RT2Wk-uHEwfi*>H?=#^qp;4aNx$TA1-yK{kbwL`W*m z_`#ls(oxipVNdwt(zo_~dNAG~N+Bq#3GQ^uq^L;=!P84$xA*nM!uPV6YrS^~NpxH` zCz2Q`oRIvT^m_CF$fS?cKX~E-=#@R9^YvCg!0#;zWkE`azoh-`>n&-dgH5_2U4tDm zh2M$VIw%`pMy0e%2O&sLI%;x?n;u_^wRiT0t|;JliG7__KPsW#%+^c`4c`I)?dv5J z-hyZ|{U1rW`%8fP<*N5wj-vaHWdgfn3V`pSg&qeGmA`!$oB93V4w zu*+S87>Sm4?sc5Plp2A|fa3EF$3X8c^R81Fy8w(Yec^dB1kikh;*e=Rg2G3{xmVsF zS8TG=E5AsfSD7Y{dGI#?qQBYnr&BSqLbgE7um)xi)_7y-MghYH{+>E;q#S_lWg#9k zhCA(9R-?m^O@jV!JCP$ukhrhc90TrAgBIr(u9L+!V2tXQxA)z44H?WxT(;^(7MN|PUn)I|8nv9HewIWMnL=|}e?Agg z_2d6x9>ojwK2c1ty;PBmqIXB$Wlb5Sz>RKd@1=UM>bcu4esl!Y&)e^3K4ZT!a_~Q= zM%7M8!r%Y*(9@5%kg$~QUmF^cjDZ29yZ+Rod<8#w7jQ`59cDCK8O-v(a`hZoIM=U=G|LmL!I%^$9^!BhD_wK>8IykmVkMoHygf{m8F zrW~@%;Ai)>?~*dYU`!$0NmuF-1mM?C`beS*WEuxFD<&XJDWtj2Zm>X6Dd}x$X0dVz zf$TD)1JgXf4DE<_Q~2zLq8iM?`WX*CL%U@jXag(k8`3=qdzV5&SMcB-huU6z(kLZm5kPq_^hhGV& z&)fJxBoQ|r3abS3!gq$UW5=Vwb7z6r0+|Qz{|Cgs({y*Q%@*%~2 zWXb3|9=#fr47`;(4!XxlJb}3p^Y)loiKf3#NwFDbH8N za^Y#SJ(lD8bwqd&jla$F&)g?Gnsy{Aw39*~_b!}NFlLYahzHsvw@%v#LUvP1+2nO~ zS$O(QM`v)KCLH+7Nh%mEWT0{Hz$zh8C_ zu8ezhDgWJ#Hl1{>4C+YBFEt5GBQKCeOKcMlt(HLH&-oHzpkl8T-C|oEEHFZ_j%4VTwLg#!*@q zjNEhLR974~l+;D5+|1LCaoUFa9L<-}{e29Lv8U7?Tp403_hy17x~OAz%WN5=mq4oU zMZ$@LeM&h7vOUrm`EH^ArCLGE^{lE;?iDL$&u#iFAB#2MQ)?%sg3lH#{4#XO8$~whY^%v3U z&uz{joS5=$Hea7V8(F7uvoa=u>NZW zbLISJt47#W%%?DMyiz6TCyKjs_cp-n^V6=z=|5nzm#jdo@*2Ewyf(tJ2{3iv$p2yL zIy^P87VA;kfbY7lvw|A{<7&^tpC$v01)X`)8UcGoA8Sr}Enw1NGh3;1=K%)QgX301 zFh#%AtId})0KNPlHI2kEa`%Dx0;cBxouuTn9TOPm!Nl9L z|D~gspzdc*TPKWj@t`izCsmAmy}$pnJU8aL8-A@S51Bveq1GPH+zopo=@gy3vtXg7 z>1~0BQn2}EBL_<^AH1+Q@|kji0e;ze#&0ts8K+$*bQTns;Jd#x)1X^F*a%DPZ!W9@ zKMB9XiwTcl&1dOd%RC$f4ht&wM5ud$)vAN--hNntmYN?uvn4$E{-tX`@z&}^#9~x* z_ca~}X?Zrh>4M(D|H&_Jp%7ABnbtB*+YUpUfwId7j5*Qk<01a1Z73j1)owI@#Gu2) z{E6>2ggyE~hHUo?r2%G-eo>@t3-!)Dqm26a$r%>9-E)$Xr-aQ9I`s6NL3F>}nt`swGlEH%t7W=(zJ z!8zEYM$8^2fi2qjX+`D-Ub|tl%zmyoGsJXKc~r1L39^~e%{cS*By65Gx4Ov4I0E18 z#jZ;3bb<{}qU&bkSS;rP-x~!tmV>dlP|LFHH{dW!*M2ex`dKXEa;LJ30nc?`pNiZF z1oQ31;rfzZh{cFoqR$bOl;wGatT($6(|>1ym)TsUzdXr<2Syh%l)7q=1?5J+pWH}j z02tP+ns|&*R$P0vapwk#aK%S*!4)wf++3J?MBl;>D0w!XJu;Oi~8bMh)}x+IFj>^(mV>)?1$_wrqn#N-*&v5$Xb_n9FUow9a2 zIh50|`N!)v+ERF0*?iys&>%!x9Vc{&ZxfAc#U8oGKcmKd;T+myX>nk_o=RLIJLe0) z`oY-zictW{1u?e6$!Nn~+VjlA7`;UO?r%*)b=A>%OzqDHF>-^7+Dmmq%=K;gsW)Nk zP~laVxL@uvJ~|MdpK6-}kYm<(G6i#0Nu3?6Z3p31Jv3ibcA)uc)A7HbkAcIgH$Kc) zB~XznH~B>>REq$lJ5`Pu!}tn+`#VvOAe-`TW^b~9-vG=tiMC0#03sAQRSY`-S5?Z$ zpPdFUjcQUQhE~hnKKH~t!tCWSJmZtgGXMqmmjz7N5icfIlRIRZxEXYSxTj)}8S8t* zybNvj7~;ieIw(|)lG69zP>>F<2$qQjB{{E72r$4Z86zW3dCzwvv+K69Pzpn@U!oFA>uV&&-}6358bbCp05|v#d4FV zS7)P)jO%{(p29$o9E5LJ#743pObvmKB{#*Z@z6H6UoWpV7j90ij{3gh@Dv);dO=ej z{vG$NE*JB8y~XUUK8^B9U6uGKUxxjO3Jhx+sott;Lm6x!Ki9o+g$NN8D$NyasRygY zmXz(!F;@wdag~%G>?xJ_!C$t4HLlFcBUZ%Z5{OFPv-@>VI+*WDe0WR-d%h|Xftj>C z$gf(YLgVmZFkj7TIx4h+FjbT9GrrgWo~xFtzGx9PZB+wz?Cw}Z0Myd|ay}mbkX7p) zw8tI50SAhbrv(^uckVtP~*j2nWR~JAl^X7H&QmR=KbDzU(b{`Ys|w z_K~$P9ONifQ+~aSc)j{;!1Xl?)UlGBc8NTN^E{Brg_G2G#a>~0$^>L=W_Uv}x%fiI)5xkC3K>(Q+9@XNZB)4&eF zt?}{YiIRqTYuniv%?U5%t{dErA-$`EC{OjXBzKzw>}0tcyP}f3+wU~P%ZFa?_^)1G z>em3c$oBe;=T!im)%~`D*i^iG(C|o<@Xe>*_kdi^Z}?RgXTiY1eh5>%shoi-e`?AgY4(G)3hAig3B%UzoFE+~|UJuRG+}$z7w;u(X zRcAdAFOIRUW1K^XmnxM~1eFltrF^B6q;yjbUNii~T}_{vW4G#Vi|aiC(1 z!xta@Cv&~9^dWjp-Te`u0(WX2pE&%k6}7kOgtwOUHg?_RQ$vA=c@gCb|9dIdXb_9a z6T1qZ{D6g(y%Ei2gW+iGbZq!*)h0M(t6Jtr(*tO{woB6wHrH_JXRu4a(;BZk3X3bd z0VeIf6lDltQubN}X#vnut18EQ)CbF7t=q>hl;fD9){BRdS@tY|k0+@-M$x(-9#A|FZps;m6LU4rs=Z+qV%z~Me@rADs z5U-p&oqn#Ih}XApi4NK>#EYKjuz5Z{nmu!uh**Oj@zNWZh}$Seyoh~$EKj|LS|Y2% zB)-A-V%1S$xwTGcwW)jPH3JsJDv9^*45#~G{0+nXlKfz!o?h3~rUGlqyAG|gj4SBg z`1Bg};0Usy!ARpEmn#cCI-_;enjJ}0Pp5sdq8_u?&gw`~7{lcnr_%6Dnh=c4Tlwm| z+B`tmlJuVDUVy&`yQj}17S$FCCM9ok0s0&^1CBunHPRUY21y{ZCa3rCDF$nNl-j^> z@6bAW#S~I!aX~+o!RaJZ0z?piZ`jUZ;U|!UtJms{bI8*&d)tL@nP7m898wRfaeVY} zlFcek2w>~rS>0xL^m0&X<7 z5Ml1(i&c^~7R2lMn$QnrTb!P2{(5Gl74e#w%PMV^Lbg1by^%1y3R9!v?5>tsKnR?(%{D-msIP@nZjR02 zcu#iuTO;XcoMoyKsazkAad_8LZrnk8r z$w~$g+sWd4W*AUS%~#n5!Ally-u&x84A|UWWV{3iOY${OPpf?c1TFZh65IKgSX+^{ z{YRXoPWJp}24SJ-o1wC!&B#NlCi5+A=u>NR&yMN60~9k|_#C~<712q{uDrkn4Le6o zYRXk6hVCm=ZUZ)YO z)2&{NI*ZQZcKhlVjSlKvX!tJ!H)4W$5YrmHV51XgjGy-ja`UGKAbT@UMpHe8JEgP7 zX)pX%L@3W3b(8Zt4bVEYVa^17K19+z3(oxl=-oeFavfD!c981+K(_|*9%09P;`tJ# znZ~IeV>qkicYK@X6AV})-nqfMfCMOhyZbbi2^O;VLsxH)uV>?k3jgzqR!e}MEB>~B zIq+r$zM@)t13(*ZAN{s83@F31sqHiaXy7@KO~!&ewEpQ48kLXhLqFLassiw|Au+>_ zsT&rysO_*`=C4KIrgQmP-5C%#6ZIc?U3yr*l=)Z{*pc(2$K5z>gh#P{y%YxM z|Lwc^Z_CMlyDk3PXz<@2d;hkV`?s^&zfHyd?Wgr`E2)3GDE-?&=-;!Q|2@|D-xGcR zJ+4%2qh5w$^_wONf|DMnG?;&IV?rNu`|6Jc5yNsS7 z`pu;0C$e{jXCgNH7J9<$x!xAnNUTK2nXFiCcPLuH!tX54 zp@l)*FE7TI6p9ofS4Tr;Jp*vDy;$;)s+tW+yWr1xX?yt;PB&Or6jJcVfY(1;Tg?k& z3P#VULPvCo={1EfEA+v*ZErq4&5p-cRybi%RVLVPzm|G2Wf{IC%&GsZ4S zSRUPu^>mctK!jpQ4BFTDz^Aw8rE%e5eWxzM9mP1nK2A$pAcdYeXrc0Y7|TSrFGa%k zy&I@Xk>~|0Ubxft{qp5ytxQyIe2v``Z!%n6q?dSoG%iIqrXFX^s6i|EYC@3}{y!Je zotK3@%|TO`ODU$0wfsxs+9~7y1Mp2)^L|0*ZZL+sH8)gN9>bnp|>spz3awuX$YX zV(z@zIc^$M_xwFC@BC#bzhIs4{c_7TL?@<9u`nB#MoU$lrMy09=8HMCKiulkKo-J2 za`x_a#aS+SPuRp&G0xMZ7#k;UjI0#a+fcp-Npj7XmT_YQlX5uUes%@zVv63v^^>_b z4BKWDUJF?$W z=U0VsdD-8{9UEBe2BBh)(Ou!jdeLpZ^HpC4dS<7@!f(5?zPO<~)@gV=Mi7E8mkoVk zn8mftYw7qp;0{z3EN7)E0-c)*cVHgktvg z(^^LEl35s8PGv)QBNB5tD!Q*{;Pl=sS^8`>B!B;aRKgQ#J%nlRjY~}BO0YRRdRYG* zGHN6*^6U-2Bk-#>%;==<8+bYroWCUsea6=V{`4_8LJjJ-qJ=BJpwAg!_cwLVpxA$2 z%|>x3hZ*_T^tWu(p{l-{+8Oyy=={-``!T&%LS3@FS; zyauhxpZl=kNdCrJ@Ym0H&ue?mj~hsvd97_aQ(UO!IXzVWn7yEci8(xu97wL(6R3;9m{%Ati92M;hEpGt>=^8pGs8Lz?R zFZ5ndiCmW3f5zi|5@;zF@w>bmHoI*8fkFOglq zhmvU#k!ISS@EdDoe*#1N8F^d`@MNrskRaJAd<)wK`Y5pObl7YB&}+sK&xrXA(PF%Z z(}vZy880<7E9-2jA`e@x9K9=ShLxzzWA9T(0TRsB``ha?X$od%n@K0Fk%c`r-L=<>UB~b*Q0~ zb7zM=-aPq|W$@G}ddpO5wBhyZ=r+@B^cy-Uw-C5LKCfLV;NaY;T?~m&@xj!*lwC?? z8*U;NOn>rC@q{Bf5r);fLcSqhUndmL{uRTK$o2S_tv~S|i@nPO$*_6;E|Ug0WJYchMqDsEP zfAHe)`&4Q+%r&x7zoxe>7M|7(i&b<%=OKxyQ$orZxqpMQ?8i=1=#$B@r4Q{y3p&I@jfo&x^pV~g2Q#n;Y~B3|7x(-QAb;Yg`mnqQVV-V?20 z)RRz+vz(Q|-NZQ&-J@rzQ5hM3_heueFLN!k(*3S`S{8skJbUa=VYJ`*w`XtK{BA{L6Iaz@W2? z4XSeLo3>pIvCG(G{j}w^r%2<86ho8J85Y>=vSP%eg}9Gty?Bz!+L@auaob%alpoW}p=XTzq9*yP3OVDCX zb`*C>;x`Rv#gSu8oAF0HqxB)K#3o9g<%gXNh*nuSq`G$9_!!=! z+o#X?0j4&s-XC1wor!UT^jV*zmSSY*l(Bm>2+g^zb*D`9x*Al$`Kg*ql72B;t|` zFFbxfI$w*j8a1ReVcZ7|NA4VVptuK{hc908{2fMu>^wJ@mGSB=nhlo1Ztk3dMFrG%@%`KttxF8Am}ZQnSLj+oz_! zL1(p6MrpqgC~s5AS8x>z{X%H7+>!BlMCV@L&M^O(kNK+R2GNA2LM&~k1 z@SzbIe^>A({zlu)bp7|Q5>tp-7aXNyiq)a*#oVc44$Nkpe1=EUK=ewX-f(Fn&pVE@5`;x#M2s-bnZHU$ zrv@>ywX*Nb`JI@nsi?ZZoE@{Dq9bwqfH3u181qT*fbaeFP0tu4V_@?M_N2)-)$pRY zVyVXf(HR;4{`#ZuYj{e{d^RDU5x!r+R>xh)u?cN^_2+-CLm#h1*Tj#HMVqB8qvvL6^x@FLA-{T zd)=>^;Yhh|+i*J(ubAuxJ|l!+{^eSw?=Cxx^HA{PRCE_czV7l~h#zap%$fS!z&1Bb zN`4@1(O?+%%qS;_lcD!c)W$?cL_yy1wCSK9e6#Sv>%5;-t0<9&<{t6-VMKj>aN5{` zWE;M?!&WX*3fX%qa6#eeE#&jmso|Y5zmef7xvlx_sKG7+;ja&OA%8kJ8{Zuuey`lR ztdjcKpbV9gHyiOh3d+~dFCQ0)7e-KwDR)K_U;2MpW0~E2x&ud^nL5n24B$O8f;s?7USFWp`{IVi7#^ zU9LnI(J2z>Zfs{lygGc!B<4GD3MS zzj)C$Co_e)9wrJV5Wi)gzT2LnPV)+BBeb3N(D(Z=jQyfrYdZ@}J*++twE`CeIw1>oI=>qXf=)Tj} zx=s(c!k$}uJ|6Kws~j?FN!l-hTpg~dzu{#05?=J5FR_!w6r+o8&s(x#aTzE18E3jf z1il|orZ*a3hZ?6pyw)u|2z~gDZ$0!uHBH}gHc@|!G@enJno<_-fT~X=AKW|q5js0& zHr-iCf%5(|*KXDMBNmBfMc3m|GK=qe*o1AN*ixN9)Kvw@vRLwB-?dAlcu!i6qvZbQ zI4f*!MRsW$#_>F(tz7Sik!>{EwNj5`E`zY^datrEyGlo)u?`Y@>h`{~R-(dfU!D+vi$^H(r5C1Ir@b?Bk!nEsN z-NTDGGDg4DU3V1OJI>-h$+Zj3YU0Yt@4+ibiAjkvlC6Fu+m!Nssz+6XVtOk_8gfx_ zGuEE0+z}j5)$_Zk4584QGW!CaK&`P@RW`<7-NTa<63I&M;e8@L(n!tp0m{kOxAIT>7@tX zL!;ZCj8{VqvWXKa7SdQIy4}L2&hEj&k;;DXc}FFd66+@44QFWBdD~>~t4qj@cBX@_ z`$w?iw|pxm3HN=6OFrk~pVD^IxCms~HmBXMfy=_Wo5Pk@`f=nObvxZJGQ7vFvnu!9 z1)TN0aUiKA9pjwjQZzpzijidlTEwh6F_+^nNt#}L%$}oCF~Scu`iIK}vqA!3;l7ja zY=51D%~4-M&ah*Oks>n%N);R#Z8}S{`g#?fPIfo-m6XBvO%Kn{bM8>%pnyc`5qapt zu}J18$q2=+M}4udf~hmAcOBGI(e!4`w*M#IYfIEc{%_7bH=%s|qar$f8pNXXkjaKF z3V*S!_QF_565=&yy--_&YFcc%5UKg#8{U&UA-%Tv6lVp7Y_Ut}kY)_=b#C+#Tx+V0?@n(-U;@|K>WBEeP*JzO& zqd}|OJxLGYsde(ivn!+UU0pShtX>nVgX<|T{txBQ=eF14<59TWOS^U_U>CBbt6wAg zGdl{cBd&J;(p^NSP5+X~QS(A5&lTP-5tfT0TBXoiaD;|6FNZj8)T!V$d4JAQEZuz^ zvAwSI+hz>!neOhF9YMUXbV+f)yd znbE=?-ZL#+$+%7!H2CJJ)*l6%@2Z~s>c#~x3`Dm*h^2#Hug(lEmk@;(ySMaJM=yLY z{T6y`30_Qeb-c*m9gV=P6yLjb(g})DMsW@Q2U%uVocEb4N|MtvS0rqvp-g>%98eoQ#pQ z^&P7I4J)v)+K05&f6E_7x<-q5i2su``)b~v3qV=5=C=q75D!Cc?-AkhO!LObe{)U^ z@YrK6<`eWBpPpfMalhSmQb?o$eUd+oU9zyy=h-5Y7Ot^}d%iv6{TT``=2>(#1Zd#b zriPZA<7;@jFZe-r)eiX1+$`TiVFNWTlS^;A`vdyi5Wc}0L|hgsSqR_l#|6xce)bhB zW;Lj4JIxjQ7|m+V_c&W^2JQ>yWAC3@S*b!S@_iYP@ChS2mDTjuY~LeZJtX_|zgz_D zeO=V%hqx~a(yL!p567FI#gWBvwBYnO&dfa~_fE zqsYuA&LM|xS-`ep*Xcy0`Y5+Jr;O(gn3jP?w?N1~z7b zh8~<%Cnz$mc9UYdaEXOrTy;4d$dDe@BS)@v$kZY9{XVA zp|G8PUzN{e6ii;q0O+PtDvH7Fz;wdQkw>nc(8uJ<0G)m%6cdSz ztdQA+hSZiCCOmwA0eNQTBhR688s}IGNh*}L+?`-Qfu67^KK_`q4*D$7eqldL&i44_7cj63b^iPLnylwjEAqV z5p|KB!mr5&o7+o<{N34tQ9D(1i#cqOsHRq_n>}ptI8v=B>s*NFw3lhAToCERS^0)K zef8fkPO8PzSNC^g)a1mVp;sT0P-31MrLEiQ4mStPT?_hGRD(e!};YCQpuOqx(%UTFifE-3NUR+dZ#d z42NR0PCp~_S)n1dyS1rh0p#7|Lvl+9EoFYEzsvC*jR*lr=!7q;BB5-kPC_jcBFD(qxHrEL@;q-Qi`5!V6bFS8YH8R8bOT*?IFeLmzKX(WR?E-Q`KhtNF7Y|hR; zU2ccV#;RnY+fg7RPu?(B(8WL&wJO2IvxvoD%6j1i<3U0?twm?zK{vfG8_skWTS4BZ zyz1@^eg9yRRy-gjbM-dcmb8=P*WJ2@tVPRWCJG4Ot|K&JlM6mgI%h^>o zM5mQov5VdT9kTt#pew%xOznK2b6NFiKSnm18+Ul@K+I+QHctH{W)Ju!TB(4JG?-$) zJR_}7Sm-3q$!Us9>({H;v1KvKx+x+&CToG$}1UfV1%5leLBw^i2F4MdwT0^#Aj37I}^T z@3UFrvb`(qW8(22tjwa6seD)bD=U52>rNH=X=2*$XGOiplB5(ad8P#xYku%aAx>M13OaV(l_rD`-r5)bc~eqA%y8sn_pQZ(E>6e&nEj4Ukhc1%e-EHjX#!vmQ%7%=zuuTcs;`hn-0<@0Os0im zHc9MnT>gWGIpsCdT7;^Ty^rnk$|%QQ$fS7JvjYEhk>h6;CBogm z@-O0#R^OD(3jCA-r%h>|8zF0Bl)bVzB+n|qYuW+3Cc{JL&e4( zvAk$S{AD~=GM|R>BjOcny|@yy8zG2IXKmw_3k38(%&Uq(#l}%Y`72MOP~!x8O&7UP zMRA67_Z)4q01+}gw3R;q4X+O-??+w5{r$Rj^#R0;=OpQBB)%GnSG)R7^BWbw+dTLO zzdWEU$I{IivcxaWE8`ypaFFRNf7ao%fIC`8M741#lVBhIry~Q)OhQ&y|C2Q&K*BIh z>4qp`k+|=6gpVikFi{vk66{23B|iJ2R(OI25WOiRp^y4Zv@kltyB#T!cxPbkRQNH# z+1qV2+zuFL^F!uV{TQH=!(%~59FP(5sP}I&z=cw@I&d#Q#g?&<6PcNCY5sRyWitTZ zk@~4$g3Y7#xpr&NC%#y~+km)*j`vH~Q)YPp&^;=Cqr40t?8$st2`wp}{?8`!GJ+Df zq~E*GeG*XFSFp*9B#DcZ^qVV2ImDTj81~3PpSas>5|wk$0jI)F=R2Wy$8PN17JHNj z&}KE@lOF;|oqcaBwE=jk9H2L$22iSfe&6;J;3EHR@q?!TKDv>Qj7R_r!Aq`Ri3mzo z3nv|W2S{1-t1?4hjwe4eI@N%=;vXq+lb^l~sBm#%v1Ns)%=w)+J9Yvb#?R`qj{t@a zShzcXfQJ8x)}-{qjKnZwhFWejz^@qnSx;GHuVV9uJ>#5!LJ}%A0c2D%?Rj;{w{SVx zSS?$B?lYj-dzWP!oOzaySK7d9t#g$%qAWCob39bBtH`X%jWmQf`Z_{5A!_s=7YfLp0wSPWou z!j#d5*ZMPn^QFt*oeu$yuRT?wh8po7N|MXM^Z?5Cqo>Z&B06hgN8EEu0siFjOM;I8 z7wWy`YoSJLgKt&wJ+y*Yy?fi3Ml=As4)ePh?nY6^4VcWX=K^x3))E_`Sj@*2Ypn!W z7}Kz^#b}LQ7t_D%W$#@-z$^_};Mp3$`caR&o7*rB*;y*e_A3C|i+(k!NQqcx0Xi24 zltb(V@emOOh#4z*_vl{^7#}P3Ao~4wqDv@0I%V4926*x;`9fAXz}%W`-enEo?2?tN zavt!;_tHbI$AIY2qT}RIfXoE`zQHa)X;xWK0pV9|v8dc@aX@!PCAFpKLgme*uGh;3Sj(ozt}ebz_GpG{80mdeU6Fh2yBjHZ`Dj_CMu5o zef)neD3Cbz=*bR#egfQB4VLH#cxtdC?i&FLnc?^A2>5TAn=+Ar)0gT#I}$))uKhv< z!1n#!A9g+hdjGr`IY7W?!E*tGVQeW5pKn3TI5yueN?TYWwIuA5!+ZImP-7K zAU2yj-i`Pbc&uOMj>j=X12H-KP+ayLKt8(SG$}e{EDyt}?RioF`nOJ_+5^aqUle1X zXpI29uGgX}HUTADadu`tfC$U&s%MS^>{hg|cIE&y^uI8?5(S7%F#qDk5*c$+?Pb^W zL%@!%(Zb+W`~Y$Cu4sUS3!tht*5vm=K#CZtY-$p~wgiycWnW{rOgkMlXvg&zVWv5B_+MNFf5ZYink)y8{v<`=DaRst9+ zuJIn|1@N{tT8AP`(K2I7<;cU$_8GL$go7U`2r5B=Zz~T0JxqCv5dS2NZQpJK0*4$JoLVovos2@jTruE z{u25sM^W98&)t0uOwq9;t&AdrUqJ9kC%NHi&4Xl7WQ!DS95xA)P4J703{s7Hf>V4f30KK4N0(~gPD66e`ow1jI*H22k z=XL_ZT0DwY+yLqPO2fY?0HtxPd$u8IqZ;Wzi_DMA2bnqy1eMkcUc)F`g%o zTCwyz8dvq303FhdvxSUkgKu>PLP_xE_`9~@61R|t3Dg0~>j6+Bp&{dN(h!Ml zO(4hO_84b~J^P6;oy)M!-yl|pThq5ryJaJ$`X>(s(t08f50*4i3O8f(3B`+gi}yY7 zOYxwjT}{u#@YD0|qw(3V{sKz#EI$j`5kE%1CEfOO6W}s(av~F%5Tf^DeU11oJmd~l z`yC*6oc%$vMpxAc6h#vEuKd?Etl4MOYZwDYXsNO`+1>nOO zoE($hVFCTX*6 zXI=yHCT`ITQUHQ&m`(+F0c>ejd2UkyG*fDB?(hbP3r8xIQUlI)ndqDN0;r8{+OqA# zZ~a$y9`hdc19XS2M28{-fs^m|`>yE{FLhpUlt~nc`kA+X#YMVEouV*6Ni_q z!509>t#r}a!+@)g>n@#r29R)`J?UHpP>kJuVDuP3=OfDwA+((!lYa5l@LGW5FP-t< zXpBL=2c0_Ja*#Xo?!)``70ghcM=G*82uqpJgDqjFxFMP5J9))>&p zrJ!*GLWR@s6FIjZ4J2HBk?ngQR1HtAKlmX*53roN?bXvKNKyV*^LMX20yyb(9oha6 zP=D>yIFbC3bSKg?gVX`4TR|3@s({3Sm!fPB081r?dw(b))%ijcJ3cA^9AAh=xyu7; z?{VnL$)X%~hj=HK!yK)SNPr>|!LVe+*uHFp3WckW9xAl$*05|flIQGh36CM_LR07ao~0X=AF z!QxkZlm_Vmm${D}8x;ba;!GWxL~w%-u<|9$qE3TJPgd$K79z2i8Ra=v1_8r|MwZtg zW>C|Crv-mF0cDhbEw>QUpbYZYg4>}+Q26%4q`MLU9)A-n_Yy^9xz6oD{S)wH<+C&` zBS3LMhUSnsKzwST^C+?|=<=w3=pkG+7~VY_~QcK|58P&lnZoUS7mFITA# zcz%B6<@sTNEroD)2r;|cwD%Bc9l*b0fYLx15FNvIwW9!#ZfmJ0bPn)NsVX=*8c@MQ zS4BeyXr|HrVU0oy>RZe`IY9!Lsw0t=Mu7xhngwh*g=d{0VzF0!cEF<@Kre7* z@9HhU$wROBRX+kaS390maRc~UP6k~_1&C)n{~F2!kahpKy4@e3_K1p^xML006IGGv zgiyiejH&iQs{qH}>lrC%alvmKPPELO0mSXR^^<)Rkmq5_U=#(YppFnE{u3D7?q8|3 zatJVScegCxwaF)~m5I^D`!khO&zy^&d5|XAkz<)ChP%=QK z=7ClaQm=LXW&J5Y|J3~?$`QbVfn(jBNWkCa)Ft9Wxlk(e_5*Fmu~5c8fB9&!{Dq!( zJfouE3lJu|;}9$bka=wp-wRnnpRVxp(xR`1`rQ9hb0-v#`SHU84{bo>8M}qA?0^L? zVf(Egs5pvMQX2jb0Cwds@n)_7$xkjk*+~8{eHQWRIZPVn;Y0r5^e`ahN6msp3ZO>i zwS5@T^rpYu-rtF03@1NF6~CV-{IkBzJd$WK;o@t4)vkzF_#-trbrGSba8Gt+Ml^eu1>S5Yi{WbB6fd9fBr1lW<(s!P0eC7$| zSE#FGI+1>%^%8n*0_OmUPE_||(8od@N^`n*RRffN-cvRV0$e`7-rx2JK&>0De&sS! zVkS;#Bx(oXOaIuz9ms@`$b+tvda(fCFMmRxL+}u%=g*`q5Xum9#tBmU7l0@AN`9V?hK&P;(accvhdZG19&qqK>n0W7S4j}VpiT4x& z7aaZbg6LuZ;B8#|&mXWb*!Aw=)D3j(V2e#}_3a2#uzvdX1A9~eYDzDv_M--a?~=`1 zA3~u8U(44#&jbsDc{Dr5&YT3WQr`;VpaY2QSMug@0;nH&;v)17U`6-BMnVk`K<{cI zi-s1QeKXi57@PDt7v>~#m zm$?0F0fuKc*&_J>Zs#bOlM(=N>_-phAU8tFINA0U6PFM@7x;I5LEJ-Dxo&PYz>837 zZp9y+HGp$GkLHOF&qJ^CnT^fF12p)Zd%q*YL#+kgwrn7gLIVY3Yj!RGa)firssATQ zM9Yf~5h*bx)|AQ2gG3?~A536_m|@3mE`(p+54eyy6w2@rAV%8~wDk_4X!RvvVlO~% zxZrIqL)5h0?@@d>ewS&B3O5uXg@?Nh;f`-3W_cU*k`)Z6hr42+vi8U+BN~!m-5ILq5&2^ zC7gGO0?gkHZ5e~ z6NeR-12q8Wc_uXWw&TyYMP9~@jyG_`S+p;o26-4Hvo01Ae-q;z%gC_5jKmI})js&J z>J}y~I%I$AKEw=lYM4GhD+yC&obJ+D*TaireDD8IN&#jTnTI2xMnqxU>&gqL?ntNm zt7)Gap^presvq&+!e}bZv8%>U0H&ounO6~}=stR>6WeDH0uf`9xAiH2r1}D~0H_g5 zbyn9>?Hs`L<+1rQvw+_IR<)&6+%t;sy0*O210=nFf$H11&z0h9P@MmPTj0~KgRT1`0GsXKq$Q^oOuxTk(d$-`ZZD#&Kci0dLYv&eoepYHE|1hDp};BMGc+?zj)Eu+f#0%(z_ ztShI-{ri94n%r9T0jWXV*T+BN+ev!C5z;j@qCBUGneyF_0pC1VY>$-SA&{z<{_t~B z18VxVuL*18zt%i2De`5D0CB8z&vo`DvLK)^F z$^iTa$vC6TaMq~b^79x7Rq&d!KRJggr{`2_ ztOpd}^qgn?W8FojfW!Bjj=V=Gvp=;Ls}eMSaV}AAKV=L$Tvm zFeRW~+At}|3ov(m=e-~tN!xQZVtXKtq_J{w=L8Z*ZnD&?6QBE~J!X1HNqk0>=1TXc z#Gg2lK=D@BpEy#nosIr2ab)z@=dW*ZBz@aAWs^5Jl74)0<-{A}h|qw0lOJ(Jt^I_p zA92LC_B|W%gZ}jJPcl8e2vd4V(ex`H;z)0%!(|`h$gc$YF>f5nI24xU=Zzy7yl-v@ zc@akx+y)oCh$Ck9=3$=15r1$^Q*Gs7iQf>f=j}jyq)qq{zQ)$F*3+Z3gT<<*-2RLz_+G=qD2><<($^})^ zHXF3${kH>3BSfpZDFF9Q@GdEC)MMkUelu>7h+H%@HZ zSryNY)}8oKwbVdx1CUmb!+m!LAeb^?LmS~vbk+&2Fgpl%_QA*V6%#;}&RISLJt|Sc z(DG>71;FL6hT|Wu0GN+yr8mL%L@F~y(~(=~Ex#M21is4wrcR6O+=*o?q0RPk1FG58!r>Ak2O0rSKS4fvgkuI}cXE&fU`=aO$%#~!2Pva2J3RRC5egAlkP`~ zdEl~iFu>b-dwV37t(-}lRg*=;B3ICE;sCK7WNx5CN3Pd9z~alwy8{krJL1lT%N#cW zk*-O;w)+4--Ga~8_n=wv4N+cJKm_xxFTdm;g5dcdUatv1fj;?LqXySku&@@0@{LGZ zdjp((6z&oiFa?zpbqe19p+oK#5LTf_doPrHQ=l0=5Ad2f_xPL`pjjx&ATJC%ThRp4 z8WY>h70J)Lzo>+CMM0vQws#Q0qMi_wm)*axU&fK8X*VqyfSS0f*IY6nI=uBQ@w4)G zqu*}?Zz^KnjnfjPk*_iVkE1Wf?>)dZy(7zS%5u9IOBZtGtfzjpZ(rh z%<~YCCu8;TV+r6#;?h@%1K5Rx_wU2nRy`c46D;B3t^uey?2idTl;1CZdP4CFBD{AY zEwgwq1vn%|J648zf0yQ>f9H52K(O*tvC~c%-%5Vy^aQ3T(vdLoyHNp`*F7pWQcnV2 z*B^O8fx<6jqI}+FGz!IXrK<&7@Ig?)bzjEROVN))${Yza~8xM7F{$Yzgb%WXb{)UyAwL*OA&D>LzW&H>g8K<3Kst8@1OMUk2hGe`mD zx88dZo6clbukA0qK)lovr#rYpywrQ=^R7JNrO^$B3&bDAGv|_xm3WDlR^@9Q*NK<@ zZeva@Mwqg8W>~e^;H4}orPhuA@KP2%8QZBX;w7eBr`w;1msr&L%wG~OUEF&qN{o2v zO3~|TGU6ps?XhoF#7mO<1P;3sFUghqi%JqNsXUq2r$7;9>ClP2Z6sbY{1jg5OT1)m zxH@r{c*&mOHq`;*rB{^+JjCbNS-!@9)rsHdWrZ-^bGwh1vMERY8(`!B+~I2exsMaz z>->Nl0S$B~cZNR)BxgE% z`uhT$Z%iKW%m%2B+z54U23)mDIbpK|pr_rFY=-Nof|U$|^M>aEjp7Z3IvBYiW%&Gm zszv}8OW%rn-T+OS70KI~0D*M5*6R&`L)QzG{#$?---iwk6aU}HZ?dp>%*hT&+uyTz z1~%urrd>5-GyrIcg>M|hlAZtGz@`%wx_bWM7aDuW>LApQeV=@ZhtTFVr=Fx=+XG0y z<{Y_n4)Cgf@-*?uNuIX(jr1X1fDm=cr7m{>L&~1wCUlbApP~jLRn>sGwACfz8QQrc zbhhE|p8`7V$!>971Jq4>ucgrgKFQK2oWxwYg(cCl;V8!3^wX>)bhwGgjq%RO@W!&0 z8@SH@U>6Z?Pt7mn4)812u}-pJeHie9yIq_3BYmzxr0TxeR{*VD6J^A<+_{R+jr9AW zVXjo~+TnZ1%v@11`^p9^Nx8h4%#Mef_zj2UZd0nyf$iwxs0xn0Q01Lifx%^E4qrYztQ{Dh{Yj=9NG5{JH)cH3T0joJ0k85ag z`jG%l|F>u$xuTX@pHwvgx~kfXX9-h%MRXXJ5y9M|(~oWte?Q9|-2K?p1-&J2$HJ2s znG*oE7Trb$OpzyBp!a*+5MUW%a5mE)5N&HHZ&3`W(ll}ug&KK_*PmumqEYA5u|DhV z;s6Lx8t)>0&z-NeY{IQ?1#s^$)w~=H$bN6;wN(k|iZCy(nFnk-zL@ey$1XUnV@az9 z!3(4$tVCGQDho_FtPNLO0ipYCLQB&D|ug z5#zWhF+cFx1rSdBZc)8~DgLZh1d@IO6p04p5#O=q8~Z+LHtz;-4Dv2yL+891_T77k z=lABtTlTV?!YY%eUX^xTEd_9d&PFRZ0z!3a-?nchlEg2EF@gey|ao(KW5TRL@Pxd8@DNv{1YfU|}c z@duBheSI~5`;ypmBujwi@&{USwA|c`uC-Uo09FaBi+_4?brCgj)Zpo7z(I$y#4%)4 zMnM%*1obN3bC1&DtpNH@=BnI?JF$mX*4t1~dw(dFefZhk;$d%q?tzz=f93(+73Stp zK!hA}r7?EqzW~L*2a>KZ1CpY}nC}odwsy@lLdO6OiccmSvi1W6x_7+wD+Ek(kWnS| zL!ZmdJnm&A02hnw z4gs{q-KUgx0CH!FwVtvfrlg*;11{14*$YSRhnWCk+9ca@f&t6sHquoeupEmVPOljl z#gWOB_huzY0M9FxCqiIq;l-NPh8GtxV3*R6rTkBT&CL91E@Dz<%k`Fh_AsM*&rX{~ zq6r&@(y-L;1{^rY*q+h|yew^}Wv-vQQ#Tq@MU@Sa1aSN0iUt~5UKc@<&{)HKD2sAO+Un)ZTAb~W-W zAh3!|ICBr6@WFe}5A%SIZwB2BO@JRR$4?F9Vp6)pZO;}00Nk0brGG5|^4IuxQ=-MC zn{+MQJBA!fe`^pO!+~~@p0`bI;lCY#*1%m?q^1BX=ZalaYT?fQkA^J<8GwuOCyv?p z0%WG!bv=y%&um{6gew4iX|8Tdho>1?iHo;Oj{usk#)Q;u0v4O)C;CR950wu2#kmRq z$FCCGKL~eb3Tv-x#sI`TrS!{(c!q6Ol*gf9nw!v>QuRo)7Ex%oez3`xC;{BO_H~d>oJ@oOBaz(+Fjre#Q?$k*Y3qGX>=2qSvuN)vSBklscb`h_gJF4hPS#Xu zQ^CSUL8GzRBEW4WS`uRo_*HN6{^)Pye2%2I+7(BHJG(YzyTWb+H(TOMvN5s2dX`^l z7~@4{=(4nx9OY#od-;I4@I)0B*$WTO9A#dgIp2j;zkFXBeMi*$B|p&T!!a zF01Dn+t;1(A(%yC>VLaB@%fl}->Nx1I%MQ5v#pxc55PWyAl3tc_?(TWF#CHi(k5J$ zq_!-&2Or8g-5Pg)Y>kiUQv5gNIZ?17O+$wmSIh8G-cQzxGbQSPqb8F1FG=umpXdi2 z`Psu0<%N+`L#ZG^H6ZNUunEfs3`AIX@8tm)GVnrhK z4a)mTS1{};qQG-A%&0DP*rH&4c(TKU-6HXv6DP68PCEGYx~Yd$KkgKsBXrb|XB+YN zDmUG2x`7{3@g$9cfC%g>i}`nKN3jP?C8>?OP3-&Z`}FqR(Km~5NA0pKqhSRcynReo`6dGQ zR$!-W;7KE>ac)7jWm6RT9PW^#>^TI*$Un$QE`k!=VJ7!i z4E->qLiPW&cjr+xweQ2gOM|46GB&7$sAy0mqBNjFXd)V=fs&F)Aw)$aku+y&q9~<= zW{qS@(yUH(rhQJ6lIp##Arsll8$X_Do@h}7*y~FTAc$d(A8MOR|$Xh^+Z{#D;!wj84sBq z<@W(go&1aCV8ABS?QK*GgmIDJW2TmZ6EPAbjKb%v`3$dKvY{{drai!Qwf+bJ&4K&V zCc2uQgd@z8`SmT07gKAVr3qgIfLG{gkt1YOYzB|;%GarYU1P3^k{Zy!QKDSbuNaU> zkqP>23NUK%A5?9EATT~Q9t?-U7!y^k_NAWp%+jyqt9HoJTn*G5ykj-O7GAfhZprz?_OjNr@wfgi|NcuXQ8D> z8;`xt)Prc`&5_#THVwv3?uMzX(~t=%&zyhEzgGUA%x3v-@6AFm`)so(9>V0F|BWDw z)+kWE!@(2xg%cX!6icwUd*&7xA?%8-ra*NdSg)dZu4Dt)l860|QxK;%vu!02UnN|( zm;nYf*zKy95RuiE3tu8wN%i<9k{H}^QlGvQk-4+*iw$5%P(kfAWC=0kGMW0Q0#VT@ z6EO}LmVbQPUlkD)EbIiSKn%C@BEZo&H7hJZ08;@Me$+!7}rd;KCp$m(JuR0rZ`z!i_I z7{Cbqjpdtri2Z|`-a~E?!}SWw+u^K04C~m;Cg7w(3_ZT~YpM<*UD`bhiZ3x3K2k5! z0T`H6%u|7(OAL70CT@Z%Ot9M`Lv~Ifc2pj-hdd@&uYc|Hf>RN}Qdy;cP8vWYIx0s% z^(2T{x9287eJ7Y_t0%MJ96&ILiQZ!D6o$Q8LxUNB-r!u&fx)Ef#-y+?P_*p&t6WDB za@@Y6rvTI&nzpjoKq|M}7FBFu|FL*g-8+bA{*X;u5Jg8jbnpoKJHKgdfLcIMr*DqH zHGv4Kaz>vvH)8i(U9D_{-OXePVJ0^w9}QW`Em z6O7Qs4U%y3CKyjL*NEY3F+OX?31lJ0-f&N?0x)H^x(sI`3_jG;l@RVWtGm(>*SllB zVec@XdrZBDcT@zk`KNOwHizIkS6hUkBbEuO<-nybf-t!motBI^CN+VdPYB}nLnmBR zgCL4js>39KI9)R-%!HRcXX+9TG z%D!oQ6Vc~CrY8nqi}2d5y@6OAUV&fm5bVA4m6l-W*k_V%ND3kDulPCp3L^Mm9TyXE zC^3n(8?mL?xBWR{#a!d}_@Zn>YYV?*$jETJ+3D zAr`(U{bu>?lbl`tIkH@(xWyqkEtIosYsBi=9=Jv2Oi>JH*E|oS#kh+$Ve>v6chS0y z_p0MATCG;sOH3QuGle7ioLw<?%{jXc4GOeyc5UyBJjmIl(ICm-?2>%_ zz}FKWJniwS3EV~Pk9~1zA)+B{vk2~@=AW#+&DoXUVj;uX6>QP=h_ma6fvZY6K4y

M4Sn`@o*+0KIgEZJff(@pj#W!!nf!G zjtlDe%F)G~f+k`4p(F^l(AJ&Ju#Z7lr(T?Titzhjo4_e?w|{BjBr$@PCqKIdQ=Rt3 zgccx*7&=}dsQ{ph$0rnSLa0}s+-rr{MeG{IR?v?vP>ROnr$^}Vt-&fm&vI+SU)n*5 zi+pjs9>I7SDc%EM2z4KR_8XxrY`A+4!eq@Nwlc!qb~HjC5qkNf3Z5Mpp&2c&uoaBZ z_q`@~)?Wxx>K*{$VU=j4=Dzcn2rIMXA}99HcHw{K7$<+Dgu=hq2bb z9KtPE<`^0CaG+(@grkyiWxvg##T=Y-#_Pqf<-XT>T@I>Fi{fdEafKe7V28LW8ljAB zXI#~|TNucJ^@FY990X)F#&M8Tcrb&5^6~-B_`2Hn{#*@i2^VS>`Nn~2cU&I_=7b_f zo)La>tN@dr;VGh2n2XqP??90QLPhKPYZ1h(`V##9gP^k<2TNQK4WmoTFqrhG;a26G zkrKH$w35?1-uW-9ByrasQ${s*H+^-NOsy|sj^D0&Y%*XJi=-u@T&?!>q=G=W+rW+Y-cle3Uw?`yk!ld*~GB*2codU zuRQ`0>zmfO0TCehy$eeUb$^m@H=Z4+I+~j(*zZ*7nq$2z#1fgeW_Z%1D*8&j;5@Tx zjkjzgZdohyHdG#=<15AHj1>JE@4ZF1#aQNTsQ|*vS6a~mVOisiH>@P62V~yzG?T84q5i3r_F2qh*Kg2D5 z`~v%CAl$uX+jt-jG)wGnKq1Wb zVWz&(PlqiGUCBot*gK3H*RDr9A%1ZCmEbUEY8@yuk-J5ToPGoemxu!Zf&J{FFN#Lhjq zHy^uTfbXkbrw_u)SaGHq!XSR~5mtZ!HTEY1Z0UfE;iJxN2%(sOzySbzjIm4NE}~a& zX+9^sqf*HuQ)JE zF2XpoOqH5Ua5S+@r;B{YY+;#n8M;ez@apuK1_yVZg_v^S(?9+p;U~ym>^9nxM zgV@%tr^D&xZH0by7dTsPJ+X=9z^j{@!-1*fsag&cb_@LAU|L8tjxW~M);9`xa$#+K zF`TE!L6oxLW)6-ze%=Ql5|_^U;tOC7Hy1TxYB5bC+`e*h^`6nZCQN?%*m&7D9Qbsz zr(VtY%(O=?(k&Q6>ZDXnD|Q^!{G(r+F`z#qWJ$X-Vw%o}j%lXfm^QsxX-Zws z5&X7IKd?AZ4$KMerUH6$YSbwh@E*al?Y**qZU@b)ecFH@`LuQY76_3f3e^kHML=u$jva%BxMizA zPHaA+wrqjH3PAhLzyLaixBb)F&RfEO4$A=}9c)EMb7p!E8_;R%GV7WnqD$^LUfhA> z`QFrSgy_GmbNL0}hlg2;D!$ea_Due!I6(LDio-!7h|#kRD=@Kp0<<+g;IQhM8jA79 zM50`HGPxMfGQHE?9BXk<_g;~I>r)8m6KPYHQbt7Hi6~D1^v~Np>@f>*S9sstAOKbR z?dw;Ah~)F~hp-E1OE-jw;Aor?y`G{SoT1t0ke)S8>6Uu6(JEmFB>zH zJ|~Dr4$CrK^*|2?60vF z(Jg!EeiEW>&Yrmfh$ewe!PtYe55JX$zaeT_3l8ZaD!O?a(-CjJvbSK;(h6!@^ZgMy zZ)y}Uv1yNU^6r=*9zD7zkc&u;y?#mx5qte~|3$>D3$A-{2-B|nnb%<2&_WOEX}?BX zuv?k23=t@#aJ3t8qCgI>DZ`D94@sKYBizO3sxLw~70(jK0ZTjJAuxOwVI?u`t2M%` z@^`^Jgt5=aT}%L)fh;Q^5}{L1wKYSm_3zfj?xv|Jw8_;W6hAldVA9f-1byhi&ZR9{ zRr{77G5>o-Dpm>F?2tEM=MghC3vkj1LF4Vp*^Y6hPF{bkQiK@Y@Mt!Mh05+v8q`A2 zZr}Ypg6PqY;@o+F+QzzuYk(2druY!f6(Q|cIH`^h7@5BT(~r_ZN#GhsWPe>!fgMM=_%I(|DmW(wD2T`xwJ!mW1G3)6zJZ}|N6Q9X=b*Y*5L-&& z{{1)-6M!;Jm{}aVv9PYueklx!Ig8zpm+MB6wDsSKvUSz*LMK*p(EX zNH}qEP?2h9%E72-FP_mTyh@s0Rva7{AH}Sr@J5sdVUeNmmW9sfzR z@KTxtg15{)ne#ln0Xyz?aJFcw-z-J&Fz0cuT~K%){t6D^Y_V%QgXbm+&w^6F9f)Zk z9(t*95Ekaf3Eeb(Plunlh5PsJea##^U$f^G2c8n9NgOPjG~^sJS9{mSL%48i>c2wqzu0_d44^OnOR+%pwmDe;*DO>Oaf#3XZ(5O!0~bX%gqTE%eY$u{M`JnD}#7OhYqBnp((K+Jl{17X)yXUDOHeV`L$4uz6DPEX`L#FTe z*fC`*#5MJ*v{{HpM`V}2Mf_{6eWG7%DOagBZ0UEpAKHmkqW}34aqedTD*wGWIm{@k zwfs`O<%qPDC-ztvspE^+o=XDIOp}X~Y!C@@+lo0wk4XC1{t>p&^<-_PFtzBBi9Ir$ zlS{XZ=O!G&49$d}w#A6ci_V=1Kr|)HNLUMCDoRJb!t7<9j+bnGf~c3eKj97_7R9ZS zUW)LN%-_&~D2dhG9|vHGOH>C~BM#o(9Vd>+TloFuI{3h;A^q_K4^5{8DwBO<7OGwp8-sS*3KBjz0;oR z_{;;P(e-n&6$72QN~0LDL9T|d@3sgTdjD(e!NE;4wkGQ$Y?nT|sE!ES%8n3kEz478LBL|N&k&V-Yy-=dFIn! z+Fhe@i=}PymV*e@UZ-U^a))MmE#<|+IM_2v`B8@`4oWS>Xbi>*x+mlC9Q3`Zd~P+u zOp^9y(oN6k(};}{&+_HF$M zY|g+EhY1UJ0DF@9Mnenn%_m441I5nyy_t>4#=aKJ+kiRGb`!45I)c!@9dZp5o4rWJ z*aOB6Gklho?-ZmBGh;O2h$6&?S<}VqXt5s9U+ZptwhtaGklU~WZ)8U(B&;#`q7N{- z9H)(6d=k#brl~37@m$>Js$hX{Gw$iXlRi5NC|#)`o{j~-bKcz0ow%j0-o>j+L?)iG)v;3<<2ZXYNz%7!F_;~R8yyD z413&BU3cwl!0j~}UTrx1uTL!ZJI@aYEtfj8^eN!N714{;`vHNwd9Psn3BLuSk)qE4 zUhVWdxd#C54?7aA1po&_+uF~uRO76eiLOzE4EAn~B zXC_~Ze0B18$Ri|=nQR5wF0!R$+sSB<@gXBd#*mCE8D}!`WM7cIL-rThgJhrn>19YE zX!QhjJY5Wn!}yn<=1Z0X#)=0f*K7d%%wVQ&#o97_o4R|S6JRi?=l2mm0Q*Si!!uU^ zgjK6q1da>F=B8h`ZaqO;`T2e>o&x$8H0-Es2K4gOjy5p?-ORTsG)$PTZ)K(vSj;=h zU;mt?2x!YHOqSaSXt|qbvU)e5Ir!v5vhDL|u5+BUavz-OcTgQt@LAJvlL z!=D2hq~f+DRs-Hmk7hq@1=J4Qj4i`qRo!~suo3g^ZB-c2w*XL)b1CMRGT_?f%=BM4 z;;d{wz8n_;$f}Iiji1E<%e$y`9R8g7w8?<^8{P;p{WIOCaK#Fy{<#~X69xbw^L)7p zcy-2?<*n+I%K-5cJo=MR*%*gDtT3N~#AB#D*y%h42Tvb8<$Gd^3Mkng8^Q&7OAnJP ziQ%dT>>ln-=Z3b^C2NFVa>IzAbtkN@nj+6}-7qq9sN2pw&c#DggwAHZ~G zohVN?tTfqBIv~djLzNm6y+EB8&M8#K!yEPappH?MHXStQOM^uyhs492`6U2Fz1b)D zq2v0yd|5IdPQqgZcvY(7psrF^MjrKX*#p=kn5dWa)FwAuP0- zETuOx7Xd7t!>&|sX!6|J46)~Sfc1N+MdwTb_I-+nOtb-)PPmKCQvhW0rWJHTiqbzs z_B&-lEu#OFTOoV`GKwLU<(jY80@$pR>L3IOz;LPU6a1P72;ZlWn{prUtk1>P17gT% zI`K$A>kOdxyyoM*Nq~{?^4o` z4FG5W6g2)))ZVCqru4 zivSH7g)Q@c0=ggEDH#X@j3%CVR4)a8E)-)P{s<3V_NF?lUIWkw7u~!6Ghpi_=BBj} zXXgHMbxUWD0lfUv1?g7-7mr_Ny@h9H-tly9OnwT;bk#4yn?x~-9pzFiF)W{Lrb5)* z0DTs%o_I4!=I@;^_fW1u3`MuatSgZNEYbIqiOm6M=~(ccFansaRi~LX0qoU8KPW+S zh~tXPe7;FQ$dbCajtD^9qV)4Gu`jY0gt+30%S5@@p&edZ0Zl>%8phuNbpFK>OFjNn z_{quw(e4`9I1%nGSB>vIew!<)KY>9x?(|8w8A|S~~B?WcYb>)(O23 zz=&qm48L%)OW%#;QUmT9NrGWoyrk#MEqz@x_Aw05z&>r_m(&GCei6C_Ry*X(6OpEf5R0(+wjFIIZHrmk?PN5__>d7JV@O7oj58T|vMck&+ELB%76AN`hglLQRq>qX)HG>E~<%QFMk z0tUX+oyS#{33kuyvFk8?;exr9*PTuTPh4*b_BbKdWt#!jgJ3RJ`0^aPjG^XX`eq@3 zuKTR4DiuK6Ija3}4S;H=^YnW;px^Vt@*d2w25w&ou?w50bMzK%!Rsu&NW4jNf?CoLhrB3Fmu{At}uE05ztc7UW~K5iSLHD zHC|%2f2%f4+w25r+Rh_gy$R6RP!MS*0r+C>#oq|0JmOQ+veO6POhd_b`sfvvT>tb1+H# z-817Z;zNSz~hBbbkP=o+kxbkL)hKUJZt4<(E#?TbNsfnR8n91~4Fh-nbzg>6+F*ZDB<}-ky*Qosjw$!k9{4om;%*b}~ zSJ!q#dh0h*e$3F4&edZ*CAq z%y|AnM;~mU@inQeLvXSnO!#DeTVooV1juhQ#%wVeTy}dyA;8pq#gA1?MCVFXIXS>~ zYt=w;?8@!$SJ&`idp_ARR1YmJ74Q}-du=z44?*f)z$ED}i zSz@4EeQ8CCt^m(UtBn(w3BL16OFrO8IqkB_-~o>1p!}G(Co=&d0({1%n6%fdz3OE! zMWa$%b`Ij1;_ld%kCphM$=h-)?gjxK-Cfk=DF%2<4YSe1?#|U7ZkwwCD7@le-&X-B z{$BT^2n+ICWvxTF6e&?3aIfF#B;Zs1tRrh@0Gg%!3BkL74iCz)HYEV%^{%0(*iweb zE5Bgui$S~PzpUE;V_8vw%18fHIo4uBi;@ShvBMxN1E&fQ4Tl=I4e%g~L+4uY-v(r7 z?8~geagtF4j4yXg!;9w~=BLk{Swed=77=Nr{8}Ax?DS^$0q|mURip; z4!VO``q(`Omu6r-d!H?UtGO{#*u%E>*8*l0bE#fPEY{?%k;6Xu*|peg;^$ z_y@>r23U41?ia`gSXdmF_wbg?*R6fmd8DD1K6(jm=WCsu*;o$ zA_&{=p#18GI}9JD#abhfi6z;w|o(M+Uf!^Oga;g^LBbWdVLG&QzO1Wo8B#>V!nk0R-9q-qIxkxD@bm z`bx+y=9N3ADo^qNB679hc8P#n?|&K!W0UXF@_4P;fTZci%kIHYWj>H!eVz%`fSJB= zVDq|mK-Rve+!wzAa!(vBeve@(h+K6}0)__j^<%~+=PE#1b;e|NDWIyy{na?+E%P1s zia?`0z$dA`4L2}!OcRe%k@m!+x50gMd&71m*e-+8V6fj%F=x=BvnbQItycRbmW3OMXuC1Vc> zNgU6WjCaQ(5IAvE@^~@evQcTwnH)ew#Ju?-IP_w^x!sD42HacxN<1zMka>@(pVmFF!XxqA(0Gyuh8Usq(8o9nBO@#-DsAHcv1&P$$(pVQ z$oR3XG)E3_+LUq&uM;wj^Dejwb^#=0WwrC5%NYHkuY?^_051mjc8Ht@gzO)AhEv)Y zR+Tq{4{Zi0sH$3v$-}44k(heL*N{u}N*=cfw;zC89{de+%K!%(@1>O718C^#`g&Xh z@T9eFU3d^a>V6XUm#8oRqy)}m99sf7M#(9b<^|}PnMGE014IiL&c`d@V|9mon3gOA zmYNl=AXpa+SltuYR|y^6|3c?;Wibq>{tdU4?iWIR=qqNPKc6cAFf{sR{}@h8z14}T z8y{5zw*R~=l??et`M9(7S1j}k#WGc+>DFPm*Ti?OD|y#V0e041QK8Ck(}^EX*LhzM z2Dr|MFbiP7-6!Y{y{k?@*Z=S;+$iMr1a3nyEEYv|hqH6nDR)}cK`*!;#YEZWhxk=v z=XuGPi`HrYejk?OZa5=!gw<`)H5mZRT9UBH@DtolXW3Z)=vf!IXN5|7bgmC{bgPz) zq1PeEw-&vbgUb6d;66y(ijw+vKoMx(?Xlzj=2fsIby!hcyAmEV707pf^#S;EOWqE) z(l9(>=G7wmB``)hg2GICuSLSEE5BQpC&L%Mh%gc&zs242fits=^|FP=sw2HH( zfN!m=fZrl`w-C8DjgxyJ2&&Gtv>^z1&-b)~dtOitD3=(I_A%fj*sHE~OUx|<(CT}> zx(CL7Ur2#hkfSVg!RiTHt+pe8c0Ci;KB!LAu;6u%zG9zhRx8-;fw0gzh3C$D9|xIm zb=S0aCeS;KwNb3#s?~t5rZydevyg|^q-uwXYax5r*%xH$%m;Mef8?|;fAY-*J@oYkk14~N-)0{$_qg`_2cTGG2&~H!z9!z_@?sKTEXLpo1Q}RAUB9! zwo#{CJfLrHx4eeS=#qscVwu`^u%{QD;Y z|3u)Q2>izgz}&M9N1YvvSN_Lc|85J+Nqa20?WLq0Okg|dpk|75BOl0p+LoUPZ&%W+ z@7$A8h4%@Gky4a}E=4fU@T- z+7Y?*54jEwxhM>|LJGNDNZwy-XOIg?kgFccO)Wj&qLm;hP-hLc^8wvZ|U*Z{Vm9wJ&<=kAa4Ueu8B`B5Kpd_PA(mO z`>%Dh$wj5f6_&~6hRL;e$%SaiRawa;O3C#)$;B)GwUXn%ON@X1`@bs!|LOaO|8frL zKkpy@dHeKF1pbM@e-Q!rP{SHc6*Bq!Kh{9_ocLjEm*V~a_~dB!DruPqd%$$6mUq)2jx;NR2b{+c`X_e83{W)+cBY{>a0e^0*H`mdP?8#6 q*d~9y{P$0J|N32P&*Z;; Date: Thu, 14 Oct 2021 08:11:04 +0200 Subject: [PATCH 054/111] proxmox-rrd: pass time and value to update function --- proxmox-rrd/src/cache.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 5225bd49..35716fc0 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -244,24 +244,23 @@ impl RRDCache { pub fn update_value( &self, rel_path: &str, + time: f64, value: f64, dst: DST, ) -> Result<(), Error> { let mut state = self.state.write().unwrap(); // block other writers - let now = proxmox_time::epoch_f64(); - - if (now - state.last_journal_flush) > self.apply_interval { + if (time - state.last_journal_flush) > self.apply_interval { if let Err(err) = self.apply_journal_locked(&mut state) { log::error!("apply journal failed: {}", err); } } - Self::append_journal_entry(&mut state, now, value, dst, rel_path)?; + Self::append_journal_entry(&mut state, time, value, dst, rel_path)?; if let Some(rrd) = state.rrd_map.get_mut(rel_path) { - rrd.update(now, value); + rrd.update(time, value); } else { let mut path = self.basedir.clone(); path.push(rel_path); @@ -269,7 +268,7 @@ impl RRDCache { let mut rrd = Self::load_rrd(&path, dst); - rrd.update(now, value); + rrd.update(time, value); state.rrd_map.insert(rel_path.into(), rrd); } From f2e1ab2d44c217f6aeb116ad840508334377f2c0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 10:17:07 +0200 Subject: [PATCH 055/111] proxmox-rrd: add regression tests and two minor fixes --- proxmox-rrd/src/rrd.rs | 92 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 96d2c366..432fa4a1 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -181,7 +181,7 @@ impl RRA { let reso = self.resolution; let num_entries = self.data.len() as u64; - let min_time = epoch - num_entries*reso; + let min_time = epoch.saturating_sub(num_entries*reso); let min_time = (min_time/reso + 1)*reso; let mut t = last_update.saturating_sub(num_entries*reso); @@ -258,7 +258,7 @@ impl RRA { let mut index = self.slot(t); for _ in 0..num_entries { if t > end { break; }; - if t < rrd_start || t > rrd_end { + if t < rrd_start || t >= rrd_end { list.push(None); } else { let value = self.data[index]; @@ -402,3 +402,91 @@ impl RRD { } } + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_rra_last_gauge_test() -> Result<(), Error> { + let rra = RRA::new(CF::Last, 60, 5); + let mut rrd = RRD::new(DST::Gauge, vec![rra]); + + for i in 2..10 { + rrd.update((i as f64)*30.0, i as f64); + } + + assert!(rrd.extract_data(CF::Average, 60, Some(0), Some(5*60)).is_err(), "CF::Average should not exist"); + + let (start, reso, data) = rrd.extract_data(CF::Last, 60, Some(0), Some(20*60))?; + assert_eq!(start, 0); + assert_eq!(reso, 60); + assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]); + + Ok(()) + } + + #[test] + fn basic_rra_average_derive_test() -> Result<(), Error> { + let rra = RRA::new(CF::Average, 60, 5); + let mut rrd = RRD::new(DST::Derive, vec![rra]); + + for i in 2..10 { + rrd.update((i as f64)*30.0, (i*60) as f64); + } + + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5*60))?; + assert_eq!(start, 60); + assert_eq!(reso, 60); + assert_eq!(data, [Some(1.0), Some(2.0), Some(2.0), Some(2.0), None]); + + Ok(()) + } + + #[test] + fn basic_rra_average_gauge_test() -> Result<(), Error> { + let rra = RRA::new(CF::Average, 60, 5); + let mut rrd = RRD::new(DST::Gauge, vec![rra]); + + for i in 2..10 { + rrd.update((i as f64)*30.0, i as f64); + } + + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5*60))?; + assert_eq!(start, 60); + assert_eq!(reso, 60); + assert_eq!(data, [Some(2.5), Some(4.5), Some(6.5), Some(8.5), None]); + + for i in 10..14 { + rrd.update((i as f64)*30.0, i as f64); + } + + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5*60))?; + assert_eq!(start, 60); + assert_eq!(reso, 60); + assert_eq!(data, [None, Some(4.5), Some(6.5), Some(8.5), Some(10.5)]); + + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(3*60), Some(8*60))?; + assert_eq!(start, 3*60); + assert_eq!(reso, 60); + assert_eq!(data, [Some(6.5), Some(8.5), Some(10.5), Some(12.5), None]); + + // add much newer vaule (should delete all previous/outdated value) + let i = 100; rrd.update((i as f64)*30.0, i as f64); + println!("TEST {:?}", serde_json::to_string_pretty(&rrd)); + + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(100*30), Some(100*30 + 5*60))?; + assert_eq!(start, 100*30); + assert_eq!(reso, 60); + assert_eq!(data, [Some(100.0), None, None, None, None]); + + // extract with end time smaller than start time + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(100*30), Some(60))?; + assert_eq!(start, 100*30); + assert_eq!(reso, 60); + assert_eq!(data, []); + + Ok(()) + } +} From 52d5f340f2b739faf40a7c7ed9e8a686e1d026ad Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 10:55:12 +0200 Subject: [PATCH 056/111] proxmox-rrd: add more regression tests --- proxmox-rrd/src/rrd.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 432fa4a1..5382ba9d 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -408,6 +408,40 @@ impl RRD { mod tests { use super::*; + #[test] + fn basic_rra_maximum_gauge_test() -> Result<(), Error> { + let rra = RRA::new(CF::Maximum, 60, 5); + let mut rrd = RRD::new(DST::Gauge, vec![rra]); + + for i in 2..10 { + rrd.update((i as f64)*30.0, i as f64); + } + + let (start, reso, data) = rrd.extract_data(CF::Maximum, 60, Some(0), Some(5*60))?; + assert_eq!(start, 0); + assert_eq!(reso, 60); + assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]); + + Ok(()) + } + + #[test] + fn basic_rra_minimum_gauge_test() -> Result<(), Error> { + let rra = RRA::new(CF::Minimum, 60, 5); + let mut rrd = RRD::new(DST::Gauge, vec![rra]); + + for i in 2..10 { + rrd.update((i as f64)*30.0, i as f64); + } + + let (start, reso, data) = rrd.extract_data(CF::Minimum, 60, Some(0), Some(5*60))?; + assert_eq!(start, 0); + assert_eq!(reso, 60); + assert_eq!(data, [None, Some(2.0), Some(4.0), Some(6.0), Some(8.0)]); + + Ok(()) + } + #[test] fn basic_rra_last_gauge_test() -> Result<(), Error> { let rra = RRA::new(CF::Last, 60, 5); From 8619b21e4d55c55c94c0fd1f887a0619e7cf101c Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 11:41:26 +0200 Subject: [PATCH 057/111] proxmox-rrd: make rrd load callback configurable --- proxmox-rrd/src/cache.rs | 48 +++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 35716fc0..0c26ede8 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -25,6 +25,7 @@ pub struct RRDCache { file_options: CreateOptions, dir_options: CreateOptions, state: RwLock, + load_rrd_cb: fn(cache: &RRDCache, path: &Path, rel_path: &str, dst: DST) -> RRD, } // shared state behind RwLock @@ -44,11 +45,25 @@ struct JournalEntry { impl RRDCache { /// Creates a new instance + /// + /// `basedir`: All files are stored relative to this path. + /// + /// `file_options`: Files are created with this options. + /// + /// `dir_options`: Directories are created with this options. + /// + /// `apply_interval`: Commit journal after `apply_interval` seconds. + /// + /// `load_rrd_cb`; The callback function is use to load RRD files, + /// and should return a newly generated RRD if the file does not + /// exists (or is unreadable). This may generate RRDs with + /// different configurations (dependent on `rel_path`). pub fn new>( basedir: P, file_options: Option, dir_options: Option, apply_interval: f64, + load_rrd_cb: fn(cache: &RRDCache, path: &Path, rel_path: &str, dst: DST) -> RRD, ) -> Result { let basedir = basedir.as_ref().to_owned(); @@ -75,11 +90,26 @@ impl RRDCache { file_options, dir_options, apply_interval, + load_rrd_cb, state: RwLock::new(state), }) } - fn create_default_rrd(dst: DST) -> RRD { + /// Create a new RRD as used by the proxmox backup server + /// + /// It contains the following RRAs: + /// + /// * cf=average,r=60,n=1440 => 1day + /// * cf=maximum,r=60,n=1440 => 1day + /// * cf=average,r=30*60,n=1440 => 1month + /// * cf=maximum,r=30*60,n=1440 => 1month + /// * cf=average,r=6*3600,n=1440 => 1year + /// * cf=maximum,r=6*3600,n=1440 => 1year + /// * cf=average,r=7*86400,n=570 => 10years + /// * cf=maximum,r=7*86400,n=570 => 10year + /// + /// The resultion data file size is about 80KB. + pub fn create_proxmox_backup_default_rrd(dst: DST) -> RRD { let mut rra_list = Vec::new(); @@ -194,7 +224,7 @@ impl RRDCache { path.push(&entry.rel_path); create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - let mut rrd = Self::load_rrd(&path, entry.dst); + let mut rrd = (self.load_rrd_cb)(&self, &path, &entry.rel_path, entry.dst); if entry.time > get_last_update(&entry.rel_path, &rrd) { rrd.update(entry.time, entry.value); @@ -228,18 +258,6 @@ impl RRDCache { Ok(()) } - fn load_rrd(path: &Path, dst: DST) -> RRD { - match RRD::load(path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - Self::create_default_rrd(dst) - }, - } - } - /// Update data in RAM and write file back to disk (journal) pub fn update_value( &self, @@ -266,7 +284,7 @@ impl RRDCache { path.push(rel_path); create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - let mut rrd = Self::load_rrd(&path, dst); + let mut rrd = (self.load_rrd_cb)(&self, &path, rel_path, dst); rrd.update(time, value); state.rrd_map.insert(rel_path.into(), rrd); From 26bd6a4f77a0be4230b73fae0f6281c0b57905cc Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 11:53:54 +0200 Subject: [PATCH 058/111] proxmox-rrd: improve dev docs --- proxmox-rrd/src/cache.rs | 4 +++- proxmox-rrd/src/rrd.rs | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 0c26ede8..6cac5849 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -54,7 +54,7 @@ impl RRDCache { /// /// `apply_interval`: Commit journal after `apply_interval` seconds. /// - /// `load_rrd_cb`; The callback function is use to load RRD files, + /// `load_rrd_cb`; The callback function is used to load RRD files, /// and should return a newly generated RRD if the file does not /// exists (or is unreadable). This may generate RRDs with /// different configurations (dependent on `rel_path`). @@ -171,6 +171,7 @@ impl RRDCache { Ok(()) } + /// Apply journal. Should be used at server startup. pub fn apply_journal(&self) -> Result<(), Error> { let mut state = self.state.write().unwrap(); // block writers self.apply_journal_locked(&mut state) @@ -296,6 +297,7 @@ impl RRDCache { /// Extract data from cached RRD /// /// `start`: Start time. If not sepecified, we simply extract 10 data points. + /// /// `end`: End time. Default is to use the current time. pub fn extract_cached_data( &self, diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 5382ba9d..60dfad7c 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -56,6 +56,7 @@ pub enum CF { } #[derive(Serialize, Deserialize)] +/// Data source specification pub struct DataSource { /// Data source type pub dst: DST, @@ -120,12 +121,15 @@ impl DataSource { } #[derive(Serialize, Deserialize)] +/// Round Robin Archive pub struct RRA { + /// Number of seconds spaned by a single data entry. pub resolution: u64, + /// Consolitation function. pub cf: CF, - /// Count values computed inside this update interval + /// Count values computed inside this update interval. pub last_count: u64, - /// The actual data + /// The actual data entries. pub data: Vec, } @@ -277,8 +281,11 @@ impl RRA { } #[derive(Serialize, Deserialize)] +/// Round Robin Database pub struct RRD { + /// The data source definition pub source: DataSource, + /// List of round robin archives pub rra_list: Vec, } From 2b9fb32de174c5b546cb0001d98504a30e6a5786 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 16:10:55 +0200 Subject: [PATCH 059/111] proxmox-rrd: cleanup - use staturating_add instead of if/else --- proxmox-rrd/src/rrd.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 60dfad7c..5dda00aa 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -219,11 +219,7 @@ impl RRA { self.last_count = 0; } - let new_count = if self.last_count < u64::MAX { - self.last_count + 1 - } else { - u64::MAX // should never happen - }; + let new_count = self.last_count.saturating_add(1); if self.last_count == 0 { self.data[index] = value; From ec5d84f4d344bae02fd4c921cd7f984fcda59277 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 16:29:00 +0200 Subject: [PATCH 060/111] proxmox-rrd: cleanup - use slot_end_time() --- proxmox-rrd/src/rrd.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 5dda00aa..3550e30a 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -186,9 +186,9 @@ impl RRA { let num_entries = self.data.len() as u64; let min_time = epoch.saturating_sub(num_entries*reso); - let min_time = (min_time/reso + 1)*reso; - let mut t = last_update.saturating_sub(num_entries*reso); + let min_time = self.slot_end_time(min_time); + let mut t = last_update.saturating_sub(num_entries*reso); let mut index = self.slot(t); for _ in 0..num_entries { From 3275f1ac1615121e50f28e11203577e2b1f74ab1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Oct 2021 16:41:08 +0200 Subject: [PATCH 061/111] proxmox-rrd: log journal apply/flush times, split apply and flush We need to apply the journal only once. --- proxmox-rrd/src/cache.rs | 76 ++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 6cac5849..4c7f05f0 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -5,6 +5,7 @@ use std::sync::RwLock; use std::io::Write; use std::io::{BufRead, BufReader}; use std::os::unix::io::AsRawFd; +use std::time::SystemTime; use anyhow::{format_err, bail, Error}; use nix::fcntl::OFlag; @@ -33,6 +34,7 @@ struct RRDCacheState { rrd_map: HashMap, journal: File, last_journal_flush: f64, + journal_applied: bool, } struct JournalEntry { @@ -83,6 +85,7 @@ impl RRDCache { journal, rrd_map: HashMap::new(), last_journal_flush: 0.0, + journal_applied: false, }; Ok(Self { @@ -171,18 +174,46 @@ impl RRDCache { Ok(()) } - /// Apply journal. Should be used at server startup. + /// Apply and commit the journal. Should be used at server startup. pub fn apply_journal(&self) -> Result<(), Error> { let mut state = self.state.write().unwrap(); // block writers - self.apply_journal_locked(&mut state) + self.apply_and_commit_journal_locked(&mut state) } - fn apply_journal_locked(&self, state: &mut RRDCacheState) -> Result<(), Error> { - - log::info!("applying rrd journal"); + fn apply_and_commit_journal_locked(&self, state: &mut RRDCacheState) -> Result<(), Error> { state.last_journal_flush = proxmox_time::epoch_f64(); + if !state.journal_applied { + let start_time = SystemTime::now(); + log::debug!("applying rrd journal"); + + match self.apply_journal_locked(state) { + Ok(entries) => { + let elapsed = start_time.elapsed()?.as_secs_f64(); + log::info!("applied rrd journal ({} entries in {:.3} seconds)", entries, elapsed); + } + Err(err) => bail!("apply rrd journal failed - {}", err), + } + } + + let start_time = SystemTime::now(); + log::debug!("commit rrd journal"); + + match self.commit_journal_locked(state) { + Ok(rrd_file_count) => { + let elapsed = start_time.elapsed()?.as_secs_f64(); + log::info!("rrd journal successfully committed ({} files in {:.3} seconds)", + rrd_file_count, elapsed); + } + Err(err) => bail!("rrd journal commit failed: {}", err), + } + + Ok(()) + } + + fn apply_journal_locked(&self, state: &mut RRDCacheState) -> Result { + let mut journal_path = self.basedir.clone(); journal_path.push(RRD_JOURNAL_NAME); @@ -234,10 +265,22 @@ impl RRDCache { } } + + // We need to apply the journal only once, because further updates + // are always directly applied. + state.journal_applied = true; + + Ok(linenr) + } + + fn commit_journal_locked(&self, state: &mut RRDCacheState) -> Result { + // save all RRDs + let mut rrd_file_count = 0; let mut errors = 0; for (rel_path, rrd) in state.rrd_map.iter() { + rrd_file_count += 1; let mut path = self.basedir.clone(); path.push(&rel_path); if let Err(err) = rrd.save(&path, self.file_options.clone()) { @@ -246,17 +289,16 @@ impl RRDCache { } } - // if everything went ok, commit the journal - - if errors == 0 { - nix::unistd::ftruncate(state.journal.as_raw_fd(), 0) - .map_err(|err| format_err!("unable to truncate journal - {}", err))?; - log::info!("rrd journal successfully committed"); - } else { - log::error!("errors during rrd flush - unable to commit rrd journal"); + if errors != 0 { + bail!("errors during rrd flush - unable to commit rrd journal"); } - Ok(()) + // if everything went ok, commit the journal + + nix::unistd::ftruncate(state.journal.as_raw_fd(), 0) + .map_err(|err| format_err!("unable to truncate journal - {}", err))?; + + Ok(rrd_file_count) } /// Update data in RAM and write file back to disk (journal) @@ -270,10 +312,8 @@ impl RRDCache { let mut state = self.state.write().unwrap(); // block other writers - if (time - state.last_journal_flush) > self.apply_interval { - if let Err(err) = self.apply_journal_locked(&mut state) { - log::error!("apply journal failed: {}", err); - } + if !state.journal_applied || (time - state.last_journal_flush) > self.apply_interval { + self.apply_and_commit_journal_locked(&mut state)?; } Self::append_journal_entry(&mut state, time, value, dst, rel_path)?; From 2be07c2250474b647fa8d8575fbbfd0d49f14753 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Oct 2021 09:22:07 +0200 Subject: [PATCH 062/111] proxmox-rrd: avoild blocking readers while applying the journal By using and extra RwLock on the rrd data. --- proxmox-rrd/src/cache.rs | 167 ++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 72 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 4c7f05f0..54fbe378 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -24,14 +24,88 @@ pub struct RRDCache { apply_interval: f64, basedir: PathBuf, file_options: CreateOptions, - dir_options: CreateOptions, state: RwLock, - load_rrd_cb: fn(cache: &RRDCache, path: &Path, rel_path: &str, dst: DST) -> RRD, + rrd_map: RwLock, +} + +struct RRDMap { + basedir: PathBuf, + file_options: CreateOptions, + dir_options: CreateOptions, + map: HashMap, + load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, +} + +impl RRDMap { + + fn update( + &mut self, + rel_path: &str, + time: f64, + value: f64, + dst: DST, + new_only: bool, + ) -> Result<(), Error> { + if let Some(rrd) = self.map.get_mut(rel_path) { + if !new_only || time > rrd.last_update() { + rrd.update(time, value); + } + } else { + let mut path = self.basedir.clone(); + path.push(rel_path); + create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; + + let mut rrd = (self.load_rrd_cb)(&path, rel_path, dst); + + if !new_only || time > rrd.last_update() { + rrd.update(time, value); + } + self.map.insert(rel_path.to_string(), rrd); + } + Ok(()) + } + + fn flush_rrd_files(&self) -> Result { + let mut rrd_file_count = 0; + + let mut errors = 0; + for (rel_path, rrd) in self.map.iter() { + rrd_file_count += 1; + + let mut path = self.basedir.clone(); + path.push(&rel_path); + + if let Err(err) = rrd.save(&path, self.file_options.clone()) { + errors += 1; + log::error!("unable to save {:?}: {}", path, err); + } + } + + if errors != 0 { + bail!("errors during rrd flush - unable to commit rrd journal"); + } + + Ok(rrd_file_count) + } + + fn extract_cached_data( + &self, + base: &str, + name: &str, + cf: CF, + resolution: u64, + start: Option, + end: Option, + ) -> Result>)>, Error> { + match self.map.get(&format!("{}/{}", base, name)) { + Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)), + None => Ok(None), + } + } } // shared state behind RwLock struct RRDCacheState { - rrd_map: HashMap, journal: File, last_journal_flush: f64, journal_applied: bool, @@ -65,7 +139,7 @@ impl RRDCache { file_options: Option, dir_options: Option, apply_interval: f64, - load_rrd_cb: fn(cache: &RRDCache, path: &Path, rel_path: &str, dst: DST) -> RRD, + load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, ) -> Result { let basedir = basedir.as_ref().to_owned(); @@ -83,19 +157,25 @@ impl RRDCache { let state = RRDCacheState { journal, - rrd_map: HashMap::new(), last_journal_flush: 0.0, journal_applied: false, }; + let rrd_map = RRDMap { + basedir: basedir.clone(), + file_options: file_options.clone(), + dir_options: dir_options, + map: HashMap::new(), + load_rrd_cb, + }; + Ok(Self { basedir, file_options, - dir_options, apply_interval, - load_rrd_cb, state: RwLock::new(state), - }) + rrd_map: RwLock::new(rrd_map), + }) } /// Create a new RRD as used by the proxmox backup server @@ -221,17 +301,7 @@ impl RRDCache { let journal = atomic_open_or_create_file(&journal_path, flags, &[], self.file_options.clone())?; let mut journal = BufReader::new(journal); - let mut last_update_map = HashMap::new(); - - let mut get_last_update = |rel_path: &str, rrd: &RRD| { - if let Some(time) = last_update_map.get(rel_path) { - return *time; - } - let last_update = rrd.last_update(); - last_update_map.insert(rel_path.to_string(), last_update); - last_update - }; - + // fixme: apply blocked to avoid too many calls to self.rrd_map.write() ?? let mut linenr = 0; loop { linenr += 1; @@ -247,25 +317,9 @@ impl RRDCache { } }; - if let Some(rrd) = state.rrd_map.get_mut(&entry.rel_path) { - if entry.time > get_last_update(&entry.rel_path, &rrd) { - rrd.update(entry.time, entry.value); - } - } else { - let mut path = self.basedir.clone(); - path.push(&entry.rel_path); - create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - - let mut rrd = (self.load_rrd_cb)(&self, &path, &entry.rel_path, entry.dst); - - if entry.time > get_last_update(&entry.rel_path, &rrd) { - rrd.update(entry.time, entry.value); - } - state.rrd_map.insert(entry.rel_path.clone(), rrd); - } + self.rrd_map.write().unwrap().update(&entry.rel_path, entry.time, entry.value, entry.dst, true)?; } - // We need to apply the journal only once, because further updates // are always directly applied. state.journal_applied = true; @@ -275,23 +329,8 @@ impl RRDCache { fn commit_journal_locked(&self, state: &mut RRDCacheState) -> Result { - // save all RRDs - let mut rrd_file_count = 0; - - let mut errors = 0; - for (rel_path, rrd) in state.rrd_map.iter() { - rrd_file_count += 1; - let mut path = self.basedir.clone(); - path.push(&rel_path); - if let Err(err) = rrd.save(&path, self.file_options.clone()) { - errors += 1; - log::error!("unable to save {:?}: {}", path, err); - } - } - - if errors != 0 { - bail!("errors during rrd flush - unable to commit rrd journal"); - } + // save all RRDs - we only need a read lock here + let rrd_file_count = self.rrd_map.read().unwrap().flush_rrd_files()?; // if everything went ok, commit the journal @@ -318,18 +357,7 @@ impl RRDCache { Self::append_journal_entry(&mut state, time, value, dst, rel_path)?; - if let Some(rrd) = state.rrd_map.get_mut(rel_path) { - rrd.update(time, value); - } else { - let mut path = self.basedir.clone(); - path.push(rel_path); - create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - - let mut rrd = (self.load_rrd_cb)(&self, &path, rel_path, dst); - - rrd.update(time, value); - state.rrd_map.insert(rel_path.into(), rrd); - } + self.rrd_map.write().unwrap().update(rel_path, time, value, dst, false)?; Ok(()) } @@ -348,12 +376,7 @@ impl RRDCache { start: Option, end: Option, ) -> Result>)>, Error> { - - let state = self.state.read().unwrap(); - - match state.rrd_map.get(&format!("{}/{}", base, name)) { - Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)), - None => Ok(None), - } + self.rrd_map.read().unwrap() + .extract_cached_data(base, name, cf, resolution, start, end) } } From 2c24c1dd22a03711ecff45468a0e6019783ad030 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Oct 2021 09:35:44 +0200 Subject: [PATCH 063/111] proxmox-rrd: rename RRDCacheState to JournalState --- proxmox-rrd/src/cache.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 54fbe378..bf8486a0 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -24,7 +24,7 @@ pub struct RRDCache { apply_interval: f64, basedir: PathBuf, file_options: CreateOptions, - state: RwLock, + state: RwLock, rrd_map: RwLock, } @@ -105,7 +105,7 @@ impl RRDMap { } // shared state behind RwLock -struct RRDCacheState { +struct JournalState { journal: File, last_journal_flush: f64, journal_applied: bool, @@ -155,7 +155,7 @@ impl RRDCache { let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; let journal = atomic_open_or_create_file(&journal_path, flags, &[], file_options.clone())?; - let state = RRDCacheState { + let state = JournalState { journal, last_journal_flush: 0.0, journal_applied: false, @@ -243,7 +243,7 @@ impl RRDCache { } fn append_journal_entry( - state: &mut RRDCacheState, + state: &mut JournalState, time: f64, value: f64, dst: DST, @@ -260,7 +260,7 @@ impl RRDCache { self.apply_and_commit_journal_locked(&mut state) } - fn apply_and_commit_journal_locked(&self, state: &mut RRDCacheState) -> Result<(), Error> { + fn apply_and_commit_journal_locked(&self, state: &mut JournalState) -> Result<(), Error> { state.last_journal_flush = proxmox_time::epoch_f64(); @@ -292,7 +292,7 @@ impl RRDCache { Ok(()) } - fn apply_journal_locked(&self, state: &mut RRDCacheState) -> Result { + fn apply_journal_locked(&self, state: &mut JournalState) -> Result { let mut journal_path = self.basedir.clone(); journal_path.push(RRD_JOURNAL_NAME); @@ -327,7 +327,7 @@ impl RRDCache { Ok(linenr) } - fn commit_journal_locked(&self, state: &mut RRDCacheState) -> Result { + fn commit_journal_locked(&self, state: &mut JournalState) -> Result { // save all RRDs - we only need a read lock here let rrd_file_count = self.rrd_map.read().unwrap().flush_rrd_files()?; From 30b4800f4f6bc227087ab8371c15a66cc0953c84 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Oct 2021 12:26:33 +0200 Subject: [PATCH 064/111] proxmox-rrd: implement non blocking journal Do not block while applying the journal. --- proxmox-rrd/Cargo.toml | 1 + proxmox-rrd/src/cache.rs | 434 ++++++++++++++++++++++++++++----------- 2 files changed, 320 insertions(+), 115 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 31473962..900b8fef 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -11,6 +11,7 @@ proxmox-router = "1.1" [dependencies] anyhow = "1.0" bitflags = "1.2.1" +crossbeam-channel = "0.5" log = "0.4" nix = "0.19.1" serde = { version = "1.0", features = ["derive"] } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index bf8486a0..7366281a 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -1,12 +1,13 @@ use std::fs::File; use std::path::{Path, PathBuf}; use std::collections::HashMap; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use std::io::Write; use std::io::{BufRead, BufReader}; -use std::os::unix::io::AsRawFd; use std::time::SystemTime; - +use std::ffi::OsStr; +use std::thread::spawn; +use crossbeam_channel::{bounded, Receiver, TryRecvError}; use anyhow::{format_err, bail, Error}; use nix::fcntl::OFlag; @@ -21,23 +22,37 @@ const RRD_JOURNAL_NAME: &str = "rrd.journal"; /// This cache is designed to run as single instance (no concurrent /// access from other processes). pub struct RRDCache { + config: Arc, + state: Arc>, + rrd_map: Arc>, +} + +struct CacheConfig { apply_interval: f64, basedir: PathBuf, file_options: CreateOptions, - state: RwLock, - rrd_map: RwLock, + dir_options: CreateOptions, } struct RRDMap { - basedir: PathBuf, - file_options: CreateOptions, - dir_options: CreateOptions, + config: Arc, map: HashMap, load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, } impl RRDMap { + fn new( + config: Arc, + load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, + ) -> Self { + Self { + config, + map: HashMap::new(), + load_rrd_cb, + } + } + fn update( &mut self, rel_path: &str, @@ -51,9 +66,13 @@ impl RRDMap { rrd.update(time, value); } } else { - let mut path = self.basedir.clone(); + let mut path = self.config.basedir.clone(); path.push(rel_path); - create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; + create_path( + path.parent().unwrap(), + Some(self.config.dir_options.clone()), + Some(self.config.dir_options.clone()), + )?; let mut rrd = (self.load_rrd_cb)(&path, rel_path, dst); @@ -72,10 +91,10 @@ impl RRDMap { for (rel_path, rrd) in self.map.iter() { rrd_file_count += 1; - let mut path = self.basedir.clone(); + let mut path = self.config.basedir.clone(); path.push(&rel_path); - if let Err(err) = rrd.save(&path, self.file_options.clone()) { + if let Err(err) = rrd.save(&path, self.config.file_options.clone()) { errors += 1; log::error!("unable to save {:?}: {}", path, err); } @@ -106,9 +125,11 @@ impl RRDMap { // shared state behind RwLock struct JournalState { + config: Arc, journal: File, last_journal_flush: f64, journal_applied: bool, + apply_thread_result: Option>>, } struct JournalEntry { @@ -118,6 +139,98 @@ struct JournalEntry { rel_path: String, } +impl JournalState { + + fn new(config: Arc) -> Result { + let journal = JournalState::open_journal_writer(&config)?; + Ok(Self { + config, + journal, + last_journal_flush: 0.0, + journal_applied: false, + apply_thread_result: None, + }) + } + + fn open_journal_reader(&self) -> Result, Error> { + + // fixme : dup self.journal instead?? + let mut journal_path = self.config.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let flags = OFlag::O_CLOEXEC|OFlag::O_RDONLY; + let journal = atomic_open_or_create_file( + &journal_path, + flags, + &[], + self.config.file_options.clone(), + )?; + Ok(BufReader::new(journal)) + } + + fn open_journal_writer(config: &CacheConfig) -> Result { + let mut journal_path = config.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; + let journal = atomic_open_or_create_file( + &journal_path, + flags, + &[], + config.file_options.clone(), + )?; + Ok(journal) + } + + fn rotate_journal(&mut self) -> Result<(), Error> { + let mut journal_path = self.config.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let mut new_name = journal_path.clone(); + let now = proxmox_time::epoch_i64(); + new_name.set_extension(format!("journal-{:08x}", now)); + std::fs::rename(journal_path, new_name)?; + + self.journal = Self::open_journal_writer(&self.config)?; + Ok(()) + } + + fn remove_old_journals(&self) -> Result<(), Error> { + + let journal_list = self.list_old_journals()?; + + for (_time, _filename, path) in journal_list { + std::fs::remove_file(path)?; + } + + Ok(()) + } + + fn list_old_journals(&self) -> Result, Error> { + let mut list = Vec::new(); + for entry in std::fs::read_dir(&self.config.basedir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + if let Some(stem) = path.file_stem() { + if stem != OsStr::new("rrd") { continue; } + if let Some(extension) = path.extension() { + if let Some(extension) = extension.to_str() { + if let Some(rest) = extension.strip_prefix("journal-") { + if let Ok(time) = u64::from_str_radix(rest, 16) { + list.push((time, format!("rrd.{}", extension), path.to_owned())); + } + } + } + } + } + } + } + list.sort_unstable_by_key(|t| t.0); + Ok(list) + } +} + impl RRDCache { /// Creates a new instance @@ -149,33 +262,21 @@ impl RRDCache { create_path(&basedir, Some(dir_options.clone()), Some(dir_options.clone())) .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; - let mut journal_path = basedir.clone(); - journal_path.push(RRD_JOURNAL_NAME); - - let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; - let journal = atomic_open_or_create_file(&journal_path, flags, &[], file_options.clone())?; - - let state = JournalState { - journal, - last_journal_flush: 0.0, - journal_applied: false, - }; - - let rrd_map = RRDMap { + let config = Arc::new(CacheConfig { basedir: basedir.clone(), file_options: file_options.clone(), dir_options: dir_options, - map: HashMap::new(), - load_rrd_cb, - }; + apply_interval, + }); + + let state = JournalState::new(Arc::clone(&config))?; + let rrd_map = RRDMap::new(Arc::clone(&config), load_rrd_cb); Ok(Self { - basedir, - file_options, - apply_interval, - state: RwLock::new(state), - rrd_map: RwLock::new(rrd_map), - }) + config: Arc::clone(&config), + state: Arc::new(RwLock::new(state)), + rrd_map: Arc::new(RwLock::new(rrd_map)), + }) } /// Create a new RRD as used by the proxmox backup server @@ -243,102 +344,64 @@ impl RRDCache { } fn append_journal_entry( - state: &mut JournalState, + &self, time: f64, value: f64, dst: DST, rel_path: &str, ) -> Result<(), Error> { + let mut state = self.state.write().unwrap(); // block other writers let journal_entry = format!("{}:{}:{}:{}\n", time, value, dst as u8, rel_path); state.journal.write_all(journal_entry.as_bytes())?; Ok(()) } /// Apply and commit the journal. Should be used at server startup. - pub fn apply_journal(&self) -> Result<(), Error> { - let mut state = self.state.write().unwrap(); // block writers - self.apply_and_commit_journal_locked(&mut state) - } + pub fn apply_journal(&self) -> Result { + let state = Arc::clone(&self.state); + let rrd_map = Arc::clone(&self.rrd_map); - fn apply_and_commit_journal_locked(&self, state: &mut JournalState) -> Result<(), Error> { + let mut state_guard = self.state.write().unwrap(); + let journal_applied = state_guard.journal_applied; + let now = proxmox_time::epoch_f64(); + let wants_commit = (now - state_guard.last_journal_flush) > self.config.apply_interval; - state.last_journal_flush = proxmox_time::epoch_f64(); + if journal_applied && !wants_commit { return Ok(journal_applied); } - if !state.journal_applied { - let start_time = SystemTime::now(); - log::debug!("applying rrd journal"); - - match self.apply_journal_locked(state) { - Ok(entries) => { - let elapsed = start_time.elapsed()?.as_secs_f64(); - log::info!("applied rrd journal ({} entries in {:.3} seconds)", entries, elapsed); + if let Some(ref recv) = state_guard.apply_thread_result { + match recv.try_recv() { + Ok(Ok(())) => { + // finished without errors, OK + } + Ok(Err(err)) => { + // finished with errors, log them + log::error!("{}", err); + } + Err(TryRecvError::Empty) => { + // still running + return Ok(journal_applied); + } + Err(TryRecvError::Disconnected) => { + // crashed, start again + log::error!("apply journal thread crashed - try again"); } - Err(err) => bail!("apply rrd journal failed - {}", err), } } - let start_time = SystemTime::now(); - log::debug!("commit rrd journal"); + state_guard.last_journal_flush = proxmox_time::epoch_f64(); - match self.commit_journal_locked(state) { - Ok(rrd_file_count) => { - let elapsed = start_time.elapsed()?.as_secs_f64(); - log::info!("rrd journal successfully committed ({} files in {:.3} seconds)", - rrd_file_count, elapsed); - } - Err(err) => bail!("rrd journal commit failed: {}", err), - } + let (sender, receiver) = bounded(1); + state_guard.apply_thread_result = Some(receiver); - Ok(()) + spawn(move || { + let result = apply_and_commit_journal_thread(state, rrd_map, journal_applied) + .map_err(|err| err.to_string()); + sender.send(result).unwrap(); + }); + + Ok(journal_applied) } - fn apply_journal_locked(&self, state: &mut JournalState) -> Result { - - let mut journal_path = self.basedir.clone(); - journal_path.push(RRD_JOURNAL_NAME); - - let flags = OFlag::O_CLOEXEC|OFlag::O_RDONLY; - let journal = atomic_open_or_create_file(&journal_path, flags, &[], self.file_options.clone())?; - let mut journal = BufReader::new(journal); - - // fixme: apply blocked to avoid too many calls to self.rrd_map.write() ?? - let mut linenr = 0; - loop { - linenr += 1; - let mut line = String::new(); - let len = journal.read_line(&mut line)?; - if len == 0 { break; } - - let entry = match Self::parse_journal_line(&line) { - Ok(entry) => entry, - Err(err) => { - log::warn!("unable to parse rrd journal line {} (skip) - {}", linenr, err); - continue; // skip unparsable lines - } - }; - - self.rrd_map.write().unwrap().update(&entry.rel_path, entry.time, entry.value, entry.dst, true)?; - } - - // We need to apply the journal only once, because further updates - // are always directly applied. - state.journal_applied = true; - - Ok(linenr) - } - - fn commit_journal_locked(&self, state: &mut JournalState) -> Result { - - // save all RRDs - we only need a read lock here - let rrd_file_count = self.rrd_map.read().unwrap().flush_rrd_files()?; - - // if everything went ok, commit the journal - - nix::unistd::ftruncate(state.journal.as_raw_fd(), 0) - .map_err(|err| format_err!("unable to truncate journal - {}", err))?; - - Ok(rrd_file_count) - } /// Update data in RAM and write file back to disk (journal) pub fn update_value( @@ -349,16 +412,14 @@ impl RRDCache { dst: DST, ) -> Result<(), Error> { - let mut state = self.state.write().unwrap(); // block other writers + let journal_applied = self.apply_journal()?; - if !state.journal_applied || (time - state.last_journal_flush) > self.apply_interval { - self.apply_and_commit_journal_locked(&mut state)?; + self.append_journal_entry(time, value, dst, rel_path)?; + + if journal_applied { + self.rrd_map.write().unwrap().update(rel_path, time, value, dst, false)?; } - Self::append_journal_entry(&mut state, time, value, dst, rel_path)?; - - self.rrd_map.write().unwrap().update(rel_path, time, value, dst, false)?; - Ok(()) } @@ -380,3 +441,146 @@ impl RRDCache { .extract_cached_data(base, name, cf, resolution, start, end) } } + + +fn apply_and_commit_journal_thread( + state: Arc>, + rrd_map: Arc>, + commit_only: bool, +) -> Result<(), Error> { + + if commit_only { + state.write().unwrap().rotate_journal()?; // start new journal, keep old one + } else { + let start_time = SystemTime::now(); + log::debug!("applying rrd journal"); + + match apply_journal_impl(Arc::clone(&state), Arc::clone(&rrd_map)) { + Ok(entries) => { + let elapsed = start_time.elapsed().unwrap().as_secs_f64(); + log::info!("applied rrd journal ({} entries in {:.3} seconds)", entries, elapsed); + } + Err(err) => bail!("apply rrd journal failed - {}", err), + } + } + + let start_time = SystemTime::now(); + log::debug!("commit rrd journal"); + + match commit_journal_impl(state, rrd_map) { + Ok(rrd_file_count) => { + let elapsed = start_time.elapsed().unwrap().as_secs_f64(); + log::info!("rrd journal successfully committed ({} files in {:.3} seconds)", + rrd_file_count, elapsed); + } + Err(err) => bail!("rrd journal commit failed: {}", err), + } + Ok(()) +} + +fn apply_journal_lines( + state: Arc>, + rrd_map: Arc>, + journal_name: &str, // used for logging + reader: &mut BufReader, + lock_read_line: bool, +) -> Result { + + let mut linenr = 0; + + loop { + linenr += 1; + let mut line = String::new(); + let len = if lock_read_line { + let _lock = state.read().unwrap(); // make sure we read entire lines + reader.read_line(&mut line)? + } else { + reader.read_line(&mut line)? + }; + + if len == 0 { break; } + + let entry = match RRDCache::parse_journal_line(&line) { + Ok(entry) => entry, + Err(err) => { + log::warn!( + "unable to parse rrd journal '{}' line {} (skip) - {}", + journal_name, linenr, err, + ); + continue; // skip unparsable lines + } + }; + + rrd_map.write().unwrap().update(&entry.rel_path, entry.time, entry.value, entry.dst, true)?; + } + Ok(linenr) +} + +fn apply_journal_impl( + state: Arc>, + rrd_map: Arc>, +) -> Result { + + let mut lines = 0; + + // Apply old journals first + let journal_list = state.read().unwrap().list_old_journals()?; + + for (_time, filename, path) in journal_list { + log::info!("apply old journal log {}", filename); + let file = std::fs::OpenOptions::new().read(true).open(path)?; + let mut reader = BufReader::new(file); + lines += apply_journal_lines( + Arc::clone(&state), + Arc::clone(&rrd_map), + &filename, + &mut reader, + false, + )?; + } + + let mut journal = state.read().unwrap().open_journal_reader()?; + + lines += apply_journal_lines( + Arc::clone(&state), + Arc::clone(&rrd_map), + "rrd.journal", + &mut journal, + true, + )?; + + { + let mut state_guard = state.write().unwrap(); // block other writers + + lines += apply_journal_lines( + Arc::clone(&state), + Arc::clone(&rrd_map), + "rrd.journal", + &mut journal, + false, + )?; + + state_guard.rotate_journal()?; // start new journal, keep old one + + // We need to apply the journal only once, because further updates + // are always directly applied. + state_guard.journal_applied = true; + } + + + Ok(lines) +} + +fn commit_journal_impl( + state: Arc>, + rrd_map: Arc>, +) -> Result { + + // save all RRDs - we only need a read lock here + let rrd_file_count = rrd_map.read().unwrap().flush_rrd_files()?; + + // if everything went ok, remove the old journal files + state.write().unwrap().remove_old_journals()?; + + Ok(rrd_file_count) +} From 9dcc64b71a207f0c6e58a9c98e1398812b83a882 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 16 Oct 2021 12:00:25 +0200 Subject: [PATCH 065/111] proxmox-rrd: move JournalState into extra file --- proxmox-rrd/src/cache.rs | 135 ++---------------------------- proxmox-rrd/src/cache/journal.rs | 137 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 128 deletions(-) create mode 100644 proxmox-rrd/src/cache/journal.rs diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 7366281a..0b7aa0de 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -2,20 +2,18 @@ use std::fs::File; use std::path::{Path, PathBuf}; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use std::io::Write; use std::io::{BufRead, BufReader}; use std::time::SystemTime; -use std::ffi::OsStr; use std::thread::spawn; -use crossbeam_channel::{bounded, Receiver, TryRecvError}; +use crossbeam_channel::{bounded, TryRecvError}; use anyhow::{format_err, bail, Error}; -use nix::fcntl::OFlag; -use proxmox::tools::fs::{atomic_open_or_create_file, create_path, CreateOptions}; +use proxmox::tools::fs::{create_path, CreateOptions}; use crate::rrd::{DST, CF, RRD, RRA}; -const RRD_JOURNAL_NAME: &str = "rrd.journal"; +mod journal; +use journal::*; /// RRD cache - keep RRD data in RAM, but write updates to disk /// @@ -27,7 +25,7 @@ pub struct RRDCache { rrd_map: Arc>, } -struct CacheConfig { +pub(crate) struct CacheConfig { apply_interval: f64, basedir: PathBuf, file_options: CreateOptions, @@ -123,113 +121,6 @@ impl RRDMap { } } -// shared state behind RwLock -struct JournalState { - config: Arc, - journal: File, - last_journal_flush: f64, - journal_applied: bool, - apply_thread_result: Option>>, -} - -struct JournalEntry { - time: f64, - value: f64, - dst: DST, - rel_path: String, -} - -impl JournalState { - - fn new(config: Arc) -> Result { - let journal = JournalState::open_journal_writer(&config)?; - Ok(Self { - config, - journal, - last_journal_flush: 0.0, - journal_applied: false, - apply_thread_result: None, - }) - } - - fn open_journal_reader(&self) -> Result, Error> { - - // fixme : dup self.journal instead?? - let mut journal_path = self.config.basedir.clone(); - journal_path.push(RRD_JOURNAL_NAME); - - let flags = OFlag::O_CLOEXEC|OFlag::O_RDONLY; - let journal = atomic_open_or_create_file( - &journal_path, - flags, - &[], - self.config.file_options.clone(), - )?; - Ok(BufReader::new(journal)) - } - - fn open_journal_writer(config: &CacheConfig) -> Result { - let mut journal_path = config.basedir.clone(); - journal_path.push(RRD_JOURNAL_NAME); - - let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; - let journal = atomic_open_or_create_file( - &journal_path, - flags, - &[], - config.file_options.clone(), - )?; - Ok(journal) - } - - fn rotate_journal(&mut self) -> Result<(), Error> { - let mut journal_path = self.config.basedir.clone(); - journal_path.push(RRD_JOURNAL_NAME); - - let mut new_name = journal_path.clone(); - let now = proxmox_time::epoch_i64(); - new_name.set_extension(format!("journal-{:08x}", now)); - std::fs::rename(journal_path, new_name)?; - - self.journal = Self::open_journal_writer(&self.config)?; - Ok(()) - } - - fn remove_old_journals(&self) -> Result<(), Error> { - - let journal_list = self.list_old_journals()?; - - for (_time, _filename, path) in journal_list { - std::fs::remove_file(path)?; - } - - Ok(()) - } - - fn list_old_journals(&self) -> Result, Error> { - let mut list = Vec::new(); - for entry in std::fs::read_dir(&self.config.basedir)? { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - if let Some(stem) = path.file_stem() { - if stem != OsStr::new("rrd") { continue; } - if let Some(extension) = path.extension() { - if let Some(extension) = extension.to_str() { - if let Some(rest) = extension.strip_prefix("journal-") { - if let Ok(time) = u64::from_str_radix(rest, 16) { - list.push((time, format!("rrd.{}", extension), path.to_owned())); - } - } - } - } - } - } - } - list.sort_unstable_by_key(|t| t.0); - Ok(list) - } -} impl RRDCache { @@ -343,19 +234,6 @@ impl RRDCache { Ok(JournalEntry { time, value, dst, rel_path }) } - fn append_journal_entry( - &self, - time: f64, - value: f64, - dst: DST, - rel_path: &str, - ) -> Result<(), Error> { - let mut state = self.state.write().unwrap(); // block other writers - let journal_entry = format!("{}:{}:{}:{}\n", time, value, dst as u8, rel_path); - state.journal.write_all(journal_entry.as_bytes())?; - Ok(()) - } - /// Apply and commit the journal. Should be used at server startup. pub fn apply_journal(&self) -> Result { let state = Arc::clone(&self.state); @@ -414,7 +292,8 @@ impl RRDCache { let journal_applied = self.apply_journal()?; - self.append_journal_entry(time, value, dst, rel_path)?; + self.state.write().unwrap() + .append_journal_entry(time, value, dst, rel_path)?; if journal_applied { self.rrd_map.write().unwrap().update(rel_path, time, value, dst, false)?; diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs new file mode 100644 index 00000000..e0f7a88b --- /dev/null +++ b/proxmox-rrd/src/cache/journal.rs @@ -0,0 +1,137 @@ +use std::fs::File; +use std::path::PathBuf; +use std::sync::Arc; +use std::io::{Write, BufReader}; +use std::ffi::OsStr; + +use anyhow::Error; +use nix::fcntl::OFlag; +use crossbeam_channel::Receiver; + +use proxmox::tools::fs::atomic_open_or_create_file; + +const RRD_JOURNAL_NAME: &str = "rrd.journal"; + +use crate::rrd::DST; +use crate::cache::CacheConfig; + +// shared state behind RwLock +pub struct JournalState { + config: Arc, + journal: File, + pub last_journal_flush: f64, + pub journal_applied: bool, + pub apply_thread_result: Option>>, +} + +pub struct JournalEntry { + pub time: f64, + pub value: f64, + pub dst: DST, + pub rel_path: String, +} + +impl JournalState { + + pub(crate) fn new(config: Arc) -> Result { + let journal = JournalState::open_journal_writer(&config)?; + Ok(Self { + config, + journal, + last_journal_flush: 0.0, + journal_applied: false, + apply_thread_result: None, + }) + } + + pub fn append_journal_entry( + &mut self, + time: f64, + value: f64, + dst: DST, + rel_path: &str, + ) -> Result<(), Error> { + let journal_entry = format!( + "{}:{}:{}:{}\n", time, value, dst as u8, rel_path); + self.journal.write_all(journal_entry.as_bytes())?; + Ok(()) + } + + pub fn open_journal_reader(&self) -> Result, Error> { + + // fixme : dup self.journal instead?? + let mut journal_path = self.config.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let flags = OFlag::O_CLOEXEC|OFlag::O_RDONLY; + let journal = atomic_open_or_create_file( + &journal_path, + flags, + &[], + self.config.file_options.clone(), + )?; + Ok(BufReader::new(journal)) + } + + fn open_journal_writer(config: &CacheConfig) -> Result { + let mut journal_path = config.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; + let journal = atomic_open_or_create_file( + &journal_path, + flags, + &[], + config.file_options.clone(), + )?; + Ok(journal) + } + + pub fn rotate_journal(&mut self) -> Result<(), Error> { + let mut journal_path = self.config.basedir.clone(); + journal_path.push(RRD_JOURNAL_NAME); + + let mut new_name = journal_path.clone(); + let now = proxmox_time::epoch_i64(); + new_name.set_extension(format!("journal-{:08x}", now)); + std::fs::rename(journal_path, new_name)?; + + self.journal = Self::open_journal_writer(&self.config)?; + Ok(()) + } + + pub fn remove_old_journals(&self) -> Result<(), Error> { + + let journal_list = self.list_old_journals()?; + + for (_time, _filename, path) in journal_list { + std::fs::remove_file(path)?; + } + + Ok(()) + } + + pub fn list_old_journals(&self) -> Result, Error> { + let mut list = Vec::new(); + for entry in std::fs::read_dir(&self.config.basedir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + if let Some(stem) = path.file_stem() { + if stem != OsStr::new("rrd") { continue; } + if let Some(extension) = path.extension() { + if let Some(extension) = extension.to_str() { + if let Some(rest) = extension.strip_prefix("journal-") { + if let Ok(time) = u64::from_str_radix(rest, 16) { + list.push((time, format!("rrd.{}", extension), path.to_owned())); + } + } + } + } + } + } + } + list.sort_unstable_by_key(|t| t.0); + Ok(list) + } +} From 4393b93a8bd63791bddfbb643936d8b942b1edfd Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 16 Oct 2021 12:22:40 +0200 Subject: [PATCH 066/111] proxmox-rrd: move RRDMap into extra file --- proxmox-rrd/src/cache.rs | 93 +--------------------------- proxmox-rrd/src/cache/rrd_map.rs | 100 +++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 90 deletions(-) create mode 100644 proxmox-rrd/src/cache/rrd_map.rs diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 0b7aa0de..afe7a33c 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -1,6 +1,5 @@ use std::fs::File; use std::path::{Path, PathBuf}; -use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::io::{BufRead, BufReader}; use std::time::SystemTime; @@ -15,6 +14,9 @@ use crate::rrd::{DST, CF, RRD, RRA}; mod journal; use journal::*; +mod rrd_map; +use rrd_map::*; + /// RRD cache - keep RRD data in RAM, but write updates to disk /// /// This cache is designed to run as single instance (no concurrent @@ -32,95 +34,6 @@ pub(crate) struct CacheConfig { dir_options: CreateOptions, } -struct RRDMap { - config: Arc, - map: HashMap, - load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, -} - -impl RRDMap { - - fn new( - config: Arc, - load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, - ) -> Self { - Self { - config, - map: HashMap::new(), - load_rrd_cb, - } - } - - fn update( - &mut self, - rel_path: &str, - time: f64, - value: f64, - dst: DST, - new_only: bool, - ) -> Result<(), Error> { - if let Some(rrd) = self.map.get_mut(rel_path) { - if !new_only || time > rrd.last_update() { - rrd.update(time, value); - } - } else { - let mut path = self.config.basedir.clone(); - path.push(rel_path); - create_path( - path.parent().unwrap(), - Some(self.config.dir_options.clone()), - Some(self.config.dir_options.clone()), - )?; - - let mut rrd = (self.load_rrd_cb)(&path, rel_path, dst); - - if !new_only || time > rrd.last_update() { - rrd.update(time, value); - } - self.map.insert(rel_path.to_string(), rrd); - } - Ok(()) - } - - fn flush_rrd_files(&self) -> Result { - let mut rrd_file_count = 0; - - let mut errors = 0; - for (rel_path, rrd) in self.map.iter() { - rrd_file_count += 1; - - let mut path = self.config.basedir.clone(); - path.push(&rel_path); - - if let Err(err) = rrd.save(&path, self.config.file_options.clone()) { - errors += 1; - log::error!("unable to save {:?}: {}", path, err); - } - } - - if errors != 0 { - bail!("errors during rrd flush - unable to commit rrd journal"); - } - - Ok(rrd_file_count) - } - - fn extract_cached_data( - &self, - base: &str, - name: &str, - cf: CF, - resolution: u64, - start: Option, - end: Option, - ) -> Result>)>, Error> { - match self.map.get(&format!("{}/{}", base, name)) { - Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)), - None => Ok(None), - } - } -} - impl RRDCache { diff --git a/proxmox-rrd/src/cache/rrd_map.rs b/proxmox-rrd/src/cache/rrd_map.rs new file mode 100644 index 00000000..27c5ae1c --- /dev/null +++ b/proxmox-rrd/src/cache/rrd_map.rs @@ -0,0 +1,100 @@ +use std::path::Path; +use std::sync::Arc; +use std::collections::HashMap; + +use anyhow::{bail, Error}; + +use proxmox::tools::fs::create_path; + +use crate::rrd::{CF, DST, RRD}; + +use super::CacheConfig; + +pub struct RRDMap { + config: Arc, + map: HashMap, + load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, +} + +impl RRDMap { + + pub(crate) fn new( + config: Arc, + load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, + ) -> Self { + Self { + config, + map: HashMap::new(), + load_rrd_cb, + } + } + + pub fn update( + &mut self, + rel_path: &str, + time: f64, + value: f64, + dst: DST, + new_only: bool, + ) -> Result<(), Error> { + if let Some(rrd) = self.map.get_mut(rel_path) { + if !new_only || time > rrd.last_update() { + rrd.update(time, value); + } + } else { + let mut path = self.config.basedir.clone(); + path.push(rel_path); + create_path( + path.parent().unwrap(), + Some(self.config.dir_options.clone()), + Some(self.config.dir_options.clone()), + )?; + + let mut rrd = (self.load_rrd_cb)(&path, rel_path, dst); + + if !new_only || time > rrd.last_update() { + rrd.update(time, value); + } + self.map.insert(rel_path.to_string(), rrd); + } + Ok(()) + } + + pub fn flush_rrd_files(&self) -> Result { + let mut rrd_file_count = 0; + + let mut errors = 0; + for (rel_path, rrd) in self.map.iter() { + rrd_file_count += 1; + + let mut path = self.config.basedir.clone(); + path.push(&rel_path); + + if let Err(err) = rrd.save(&path, self.config.file_options.clone()) { + errors += 1; + log::error!("unable to save {:?}: {}", path, err); + } + } + + if errors != 0 { + bail!("errors during rrd flush - unable to commit rrd journal"); + } + + Ok(rrd_file_count) + } + + pub fn extract_cached_data( + &self, + base: &str, + name: &str, + cf: CF, + resolution: u64, + start: Option, + end: Option, + ) -> Result>)>, Error> { + match self.map.get(&format!("{}/{}", base, name)) { + Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)), + None => Ok(None), + } + } +} From a74384f725181268aee3623fc3d9fde00b802a83 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 16 Oct 2021 12:38:34 +0200 Subject: [PATCH 067/111] proxmox-rrd: cleanup - use struct instead of tuple --- proxmox-rrd/src/cache.rs | 8 ++++---- proxmox-rrd/src/cache/journal.rs | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index afe7a33c..c3969686 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -318,14 +318,14 @@ fn apply_journal_impl( // Apply old journals first let journal_list = state.read().unwrap().list_old_journals()?; - for (_time, filename, path) in journal_list { - log::info!("apply old journal log {}", filename); - let file = std::fs::OpenOptions::new().read(true).open(path)?; + for entry in journal_list { + log::info!("apply old journal log {}", entry.name); + let file = std::fs::OpenOptions::new().read(true).open(&entry.path)?; let mut reader = BufReader::new(file); lines += apply_journal_lines( Arc::clone(&state), Arc::clone(&rrd_map), - &filename, + &entry.name, &mut reader, false, )?; diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index e0f7a88b..cd84cc7f 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -31,6 +31,12 @@ pub struct JournalEntry { pub rel_path: String, } +pub struct JournalFileInfo { + pub time: u64, + pub name: String, + pub path: PathBuf, +} + impl JournalState { pub(crate) fn new(config: Arc) -> Result { @@ -104,14 +110,14 @@ impl JournalState { let journal_list = self.list_old_journals()?; - for (_time, _filename, path) in journal_list { - std::fs::remove_file(path)?; + for entry in journal_list { + std::fs::remove_file(entry.path)?; } Ok(()) } - pub fn list_old_journals(&self) -> Result, Error> { + pub fn list_old_journals(&self) -> Result, Error> { let mut list = Vec::new(); for entry in std::fs::read_dir(&self.config.basedir)? { let entry = entry?; @@ -123,7 +129,11 @@ impl JournalState { if let Some(extension) = extension.to_str() { if let Some(rest) = extension.strip_prefix("journal-") { if let Ok(time) = u64::from_str_radix(rest, 16) { - list.push((time, format!("rrd.{}", extension), path.to_owned())); + list.push(JournalFileInfo { + time, + name: format!("rrd.{}", extension), + path: path.to_owned(), + }); } } } @@ -131,7 +141,7 @@ impl JournalState { } } } - list.sort_unstable_by_key(|t| t.0); + list.sort_unstable_by_key(|entry| entry.time); Ok(list) } } From e23f3ec77477b309e5169629bde20bd95a1fb9ee Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 18 Oct 2021 10:00:16 +0200 Subject: [PATCH 068/111] proxmox-rrd: cleanup list_old_journals --- proxmox-rrd/src/cache/journal.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index cd84cc7f..592e18ec 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -122,20 +122,24 @@ impl JournalState { for entry in std::fs::read_dir(&self.config.basedir)? { let entry = entry?; let path = entry.path(); - if path.is_file() { - if let Some(stem) = path.file_stem() { - if stem != OsStr::new("rrd") { continue; } - if let Some(extension) = path.extension() { - if let Some(extension) = extension.to_str() { - if let Some(rest) = extension.strip_prefix("journal-") { - if let Ok(time) = u64::from_str_radix(rest, 16) { - list.push(JournalFileInfo { - time, - name: format!("rrd.{}", extension), - path: path.to_owned(), - }); - } - } + + if !path.is_file() { continue; } + + match path.file_stem() { + None => continue, + Some(stem) if stem != OsStr::new("rrd") => continue, + Some(_) => (), + } + + if let Some(extension) = path.extension() { + if let Some(extension) = extension.to_str() { + if let Some(rest) = extension.strip_prefix("journal-") { + if let Ok(time) = u64::from_str_radix(rest, 16) { + list.push(JournalFileInfo { + time, + name: format!("rrd.{}", extension), + path: path.to_owned(), + }); } } } From cc0bb59788a1d152d5b1c3c4dc2ca7309702d340 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 18 Oct 2021 11:57:19 +0200 Subject: [PATCH 069/111] proxmox-rrd: log all errors from apply_and_commit_journal_thread (but only once) --- proxmox-rrd/src/cache.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index c3969686..0fd49d72 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -154,19 +154,17 @@ impl RRDCache { let mut state_guard = self.state.write().unwrap(); let journal_applied = state_guard.journal_applied; - let now = proxmox_time::epoch_f64(); - let wants_commit = (now - state_guard.last_journal_flush) > self.config.apply_interval; - - if journal_applied && !wants_commit { return Ok(journal_applied); } if let Some(ref recv) = state_guard.apply_thread_result { match recv.try_recv() { Ok(Ok(())) => { // finished without errors, OK + state_guard.apply_thread_result = None; } Ok(Err(err)) => { // finished with errors, log them log::error!("{}", err); + state_guard.apply_thread_result = None; } Err(TryRecvError::Empty) => { // still running @@ -175,10 +173,16 @@ impl RRDCache { Err(TryRecvError::Disconnected) => { // crashed, start again log::error!("apply journal thread crashed - try again"); + state_guard.apply_thread_result = None; } } } + let now = proxmox_time::epoch_f64(); + let wants_commit = (now - state_guard.last_journal_flush) > self.config.apply_interval; + + if journal_applied && !wants_commit { return Ok(journal_applied); } + state_guard.last_journal_flush = proxmox_time::epoch_f64(); let (sender, receiver) = bounded(1); From 77c2e4668b9aefcb817d2164d3a4bd4df7a2bd39 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 18 Oct 2021 14:52:27 +0200 Subject: [PATCH 070/111] proxmox-rrd: use fine grained locking in commit_journal_impl Aquire the rrd_map lock for each file (else we block access for a long time) --- proxmox-rrd/src/cache.rs | 18 +++++++++++++++++- proxmox-rrd/src/cache/rrd_map.rs | 31 ++++++++++++++----------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 0fd49d72..faeae89f 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -152,6 +152,7 @@ impl RRDCache { let state = Arc::clone(&self.state); let rrd_map = Arc::clone(&self.rrd_map); + let mut state_guard = self.state.write().unwrap(); let journal_applied = state_guard.journal_applied; @@ -372,8 +373,23 @@ fn commit_journal_impl( rrd_map: Arc>, ) -> Result { + let files = rrd_map.read().unwrap().file_list(); + + let mut rrd_file_count = 0; + let mut errors = 0; + // save all RRDs - we only need a read lock here - let rrd_file_count = rrd_map.read().unwrap().flush_rrd_files()?; + for rel_path in files.iter() { + rrd_file_count += 1; + if let Err(err) = rrd_map.read().unwrap().flush_rrd_file(&rel_path) { + errors += 1; + log::error!("unable to save rrd {}: {}", rel_path, err); + } + } + + if errors != 0 { + bail!("errors during rrd flush - unable to commit rrd journal"); + } // if everything went ok, remove the old journal files state.write().unwrap().remove_old_journals()?; diff --git a/proxmox-rrd/src/cache/rrd_map.rs b/proxmox-rrd/src/cache/rrd_map.rs index 27c5ae1c..c271c075 100644 --- a/proxmox-rrd/src/cache/rrd_map.rs +++ b/proxmox-rrd/src/cache/rrd_map.rs @@ -60,27 +60,24 @@ impl RRDMap { Ok(()) } - pub fn flush_rrd_files(&self) -> Result { - let mut rrd_file_count = 0; + pub fn file_list(&self) -> Vec { + let mut list = Vec::new(); - let mut errors = 0; - for (rel_path, rrd) in self.map.iter() { - rrd_file_count += 1; + for rel_path in self.map.keys() { + list.push(rel_path.clone()); + } + list + } + + pub fn flush_rrd_file(&self, rel_path: &str) -> Result<(), Error> { + if let Some(rrd) = self.map.get(rel_path) { let mut path = self.config.basedir.clone(); - path.push(&rel_path); - - if let Err(err) = rrd.save(&path, self.config.file_options.clone()) { - errors += 1; - log::error!("unable to save {:?}: {}", path, err); - } + path.push(rel_path); + rrd.save(&path, self.config.file_options.clone()) + } else { + bail!("rrd file {} not loaded", rel_path); } - - if errors != 0 { - bail!("errors during rrd flush - unable to commit rrd journal"); - } - - Ok(rrd_file_count) } pub fn extract_cached_data( From 336e8f3e7f49730802a5c549d373b3fc8a4e6e65 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 18 Oct 2021 13:45:30 +0200 Subject: [PATCH 071/111] proxmox-rrd: use syncfs after writing rrd files Signed-off-by: Dietmar Maurer --- proxmox-rrd/Cargo.toml | 1 + proxmox-rrd/src/cache.rs | 6 ++++++ proxmox-rrd/src/cache/journal.rs | 11 +++++++++++ proxmox-rrd/src/rrd.rs | 1 - 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 900b8fef..67095689 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -12,6 +12,7 @@ proxmox-router = "1.1" anyhow = "1.0" bitflags = "1.2.1" crossbeam-channel = "0.5" +libc = "0.2" log = "0.4" nix = "0.19.1" serde = { version = "1.0", features = ["derive"] } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index faeae89f..547f6603 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -147,6 +147,10 @@ impl RRDCache { Ok(JournalEntry { time, value, dst, rel_path }) } + pub fn sync_journal(&self) -> Result<(), Error> { + self.state.read().unwrap().sync_journal() + } + /// Apply and commit the journal. Should be used at server startup. pub fn apply_journal(&self) -> Result { let state = Arc::clone(&self.state); @@ -387,6 +391,8 @@ fn commit_journal_impl( } } + state.read().unwrap().syncfs()?; + if errors != 0 { bail!("errors during rrd flush - unable to commit rrd journal"); } diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index 592e18ec..3514e9e7 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::io::{Write, BufReader}; use std::ffi::OsStr; +use std::os::unix::io::AsRawFd; use anyhow::Error; use nix::fcntl::OFlag; @@ -50,6 +51,16 @@ impl JournalState { }) } + pub fn sync_journal(&self) -> Result<(), Error> { + nix::unistd::fdatasync(self.journal.as_raw_fd())?; + Ok(()) + } + + pub fn syncfs(&self) -> Result<(), nix::Error> { + let res = unsafe { libc::syncfs(self.journal.as_raw_fd()) }; + nix::errno::Errno::result(res).map(drop) + } + pub fn append_journal_entry( &mut self, time: f64, diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 3550e30a..15d73856 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -14,7 +14,6 @@ use std::path::Path; use anyhow::{bail, format_err, Error}; - use serde::{Serialize, Deserialize}; use proxmox::tools::fs::{replace_file, CreateOptions}; From 75bb60e7b31e8cb8e48ea3b5cf160d5c6941d6ae Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Oct 2021 09:46:05 +0200 Subject: [PATCH 072/111] proxmox-rrd: add option to avoid page cache for load/save use fadvice(.., POSIX_FADV_DONTNEED) for RRD files. We read those files only once, and always rewrite them. Signed-off-by: Dietmar Maurer --- proxmox-rrd/src/cache/rrd_map.rs | 2 +- proxmox-rrd/src/rrd.rs | 68 ++++++++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/proxmox-rrd/src/cache/rrd_map.rs b/proxmox-rrd/src/cache/rrd_map.rs index c271c075..6f7686d6 100644 --- a/proxmox-rrd/src/cache/rrd_map.rs +++ b/proxmox-rrd/src/cache/rrd_map.rs @@ -74,7 +74,7 @@ impl RRDMap { if let Some(rrd) = self.map.get(rel_path) { let mut path = self.config.basedir.clone(); path.push(rel_path); - rrd.save(&path, self.config.file_options.clone()) + rrd.save(&path, self.config.file_options.clone(), true) } else { bail!("rrd file {} not loaded", rel_path); } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 15d73856..b985d83b 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -12,11 +12,13 @@ //! * Arbitrary number of RRAs (dynamically changeable) use std::path::Path; +use std::io::{Read, Write}; +use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; use anyhow::{bail, format_err, Error}; use serde::{Serialize, Deserialize}; -use proxmox::tools::fs::{replace_file, CreateOptions}; +use proxmox::tools::fs::{make_tmp_file, CreateOptions}; use proxmox_schema::api; use crate::rrd_v1; @@ -321,8 +323,21 @@ impl RRD { } /// Load data from a file - pub fn load(path: &Path) -> Result { - let raw = std::fs::read(path)?; + pub fn load(path: &Path, avoid_page_cache: bool) -> Result { + + let mut file = std::fs::File::open(path)?; + let buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0); + let mut raw = Vec::with_capacity(buffer_size); + file.read_to_end(&mut raw)?; + + if avoid_page_cache { + nix::fcntl::posix_fadvise( + file.as_raw_fd(), + 0, + buffer_size as i64, + nix::fcntl::PosixFadviseAdvice::POSIX_FADV_DONTNEED, + ).map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))?; + } match Self::from_raw(&raw) { Ok(rrd) => Ok(rrd), @@ -331,11 +346,48 @@ impl RRD { } /// Store data into a file (atomic replace file) - pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> { - let mut data: Vec = Vec::new(); - data.extend(&PROXMOX_RRD_MAGIC_2_0); - serde_cbor::to_writer(&mut data, self)?; - replace_file(filename, &data, options) + pub fn save( + &self, + path: &Path, + options: CreateOptions, + avoid_page_cache: bool, + ) -> Result<(), Error> { + + let (fd, tmp_path) = make_tmp_file(&path, options)?; + let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) }; + + let mut try_block = || -> Result<(), Error> { + let mut data: Vec = Vec::new(); + data.extend(&PROXMOX_RRD_MAGIC_2_0); + serde_cbor::to_writer(&mut data, self)?; + file.write_all(&data)?; + + if avoid_page_cache { + nix::fcntl::posix_fadvise( + file.as_raw_fd(), + 0, + data.len() as i64, + nix::fcntl::PosixFadviseAdvice::POSIX_FADV_DONTNEED, + )?; + } + + Ok(()) + }; + + match try_block() { + Ok(()) => (), + error => { + let _ = nix::unistd::unlink(&tmp_path); + return error; + } + } + + if let Err(err) = std::fs::rename(&tmp_path, &path) { + let _ = nix::unistd::unlink(&tmp_path); + bail!("Atomic rename failed - {}", err); + } + + Ok(()) } pub fn last_update(&self) -> f64 { From ed6a7f52e51df7140426e76de25a978cb4541f5e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Oct 2021 10:51:22 +0200 Subject: [PATCH 073/111] proxmox-rrd: cleanup - impl FromStr for JournalEntry Signed-off-by: Dietmar Maurer --- proxmox-rrd/src/cache.rs | 29 +-------------------------- proxmox-rrd/src/cache/journal.rs | 34 +++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 547f6603..137e376e 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -120,33 +120,6 @@ impl RRDCache { RRD::new(dst, rra_list) } - fn parse_journal_line(line: &str) -> Result { - - let line = line.trim(); - - let parts: Vec<&str> = line.splitn(4, ':').collect(); - if parts.len() != 4 { - bail!("wrong numper of components"); - } - - let time: f64 = parts[0].parse() - .map_err(|_| format_err!("unable to parse time"))?; - let value: f64 = parts[1].parse() - .map_err(|_| format_err!("unable to parse value"))?; - let dst: u8 = parts[2].parse() - .map_err(|_| format_err!("unable to parse data source type"))?; - - let dst = match dst { - 0 => DST::Gauge, - 1 => DST::Derive, - _ => bail!("got strange value for data source type '{}'", dst), - }; - - let rel_path = parts[3].to_string(); - - Ok(JournalEntry { time, value, dst, rel_path }) - } - pub fn sync_journal(&self) -> Result<(), Error> { self.state.read().unwrap().sync_journal() } @@ -301,7 +274,7 @@ fn apply_journal_lines( if len == 0 { break; } - let entry = match RRDCache::parse_journal_line(&line) { + let entry: JournalEntry = match line.parse() { Ok(entry) => entry, Err(err) => { log::warn!( diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index 3514e9e7..1f9c67b9 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -4,8 +4,9 @@ use std::sync::Arc; use std::io::{Write, BufReader}; use std::ffi::OsStr; use std::os::unix::io::AsRawFd; +use std::str::FromStr; -use anyhow::Error; +use anyhow::{bail, format_err, Error}; use nix::fcntl::OFlag; use crossbeam_channel::Receiver; @@ -32,6 +33,37 @@ pub struct JournalEntry { pub rel_path: String, } +impl FromStr for JournalEntry { + type Err = Error; + + fn from_str(line: &str) -> Result { + + let line = line.trim(); + + let parts: Vec<&str> = line.splitn(4, ':').collect(); + if parts.len() != 4 { + bail!("wrong numper of components"); + } + + let time: f64 = parts[0].parse() + .map_err(|_| format_err!("unable to parse time"))?; + let value: f64 = parts[1].parse() + .map_err(|_| format_err!("unable to parse value"))?; + let dst: u8 = parts[2].parse() + .map_err(|_| format_err!("unable to parse data source type"))?; + + let dst = match dst { + 0 => DST::Gauge, + 1 => DST::Derive, + _ => bail!("got strange value for data source type '{}'", dst), + }; + + let rel_path = parts[3].to_string(); + + Ok(JournalEntry { time, value, dst, rel_path }) + } +} + pub struct JournalFileInfo { pub time: u64, pub name: String, From 86b50e18ed0e95cbddc31046b2b6f5ec0d975856 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Oct 2021 11:14:57 +0200 Subject: [PATCH 074/111] proxmox-rrd: improve dev docs Signed-off-by: Dietmar Maurer --- proxmox-rrd/src/cache.rs | 1 + proxmox-rrd/src/rrd.rs | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 137e376e..e4dd2489 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -120,6 +120,7 @@ impl RRDCache { RRD::new(dst, rra_list) } + /// Sync the journal data to disk (using `fdatasync` syscall) pub fn sync_journal(&self) -> Result<(), Error> { self.state.read().unwrap().sync_journal() } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index b985d83b..20bf6ae3 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -70,6 +70,7 @@ pub struct DataSource { impl DataSource { + /// Create a new Instance pub fn new(dst: DST) -> Self { Self { dst, @@ -136,6 +137,7 @@ pub struct RRA { impl RRA { + /// Creates a new instance pub fn new(cf: CF, resolution: u64, points: usize) -> Self { Self { cf, @@ -145,20 +147,24 @@ impl RRA { } } + /// Data slot end time pub fn slot_end_time(&self, time: u64) -> u64 { self.resolution * (time / self.resolution + 1) } + /// Data slot start time pub fn slot_start_time(&self, time: u64) -> u64 { self.resolution * (time / self.resolution) } + /// Data slot index pub fn slot(&self, time: u64) -> usize { ((time / self.resolution) as usize) % self.data.len() } - // directly overwrite data slots - // the caller need to set last_update value on the DataSource manually. + /// Directly overwrite data slots. + /// + /// The caller need to set `last_update` value on the [DataSource] manually. pub fn insert_data( &mut self, start: u64, @@ -240,6 +246,11 @@ impl RRA { } } + /// Extract data + /// + /// Extract data from `start` to `end`. The RRA itself does not + /// store the `last_update` time, so you need to pass this a + /// parameter (see [DataSource]). pub fn extract_data( &self, start: u64, @@ -288,6 +299,7 @@ pub struct RRD { impl RRD { + /// Creates a new Instance pub fn new(dst: DST, rra_list: Vec) -> RRD { let source = DataSource::new(dst); @@ -323,6 +335,10 @@ impl RRD { } /// Load data from a file + /// + /// Setting `avoid_page_cache` uses + /// `fadvise(..,POSIX_FADV_DONTNEED)` to avoid keeping the data in + /// the linux page cache. pub fn load(path: &Path, avoid_page_cache: bool) -> Result { let mut file = std::fs::File::open(path)?; @@ -346,7 +362,11 @@ impl RRD { } /// Store data into a file (atomic replace file) - pub fn save( + /// + /// Setting `avoid_page_cache` uses + /// `fadvise(..,POSIX_FADV_DONTNEED)` to avoid keeping the data in + /// the linux page cache. + pub fn save( &self, path: &Path, options: CreateOptions, @@ -390,6 +410,7 @@ impl RRD { Ok(()) } + /// Returns the last update time. pub fn last_update(&self) -> f64 { self.source.last_update } From a7ee3455dacdb699be8b8503971f2a071ec2926d Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Oct 2021 18:41:03 +0200 Subject: [PATCH 075/111] proxmox-rrd: fix regression tests --- proxmox-rrd/examples/prrd.rs | 22 +++++++++++----------- proxmox-rrd/tests/file_format_test.rs | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index 59e75d3c..e00e5a0e 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -51,7 +51,7 @@ pub struct RRAConfig { /// Dump the RRD file in JSON format pub fn dump_rrd(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path))?; + let rrd = RRD::load(&PathBuf::from(path), false)?; serde_json::to_writer_pretty(std::io::stdout(), &rrd)?; println!(""); Ok(()) @@ -69,7 +69,7 @@ pub fn dump_rrd(path: String) -> Result<(), Error> { /// RRD file information pub fn rrd_info(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path))?; + let rrd = RRD::load(&PathBuf::from(path), false)?; println!("DST: {:?}", rrd.source.dst); @@ -109,10 +109,10 @@ pub fn update_rrd( let time = time.map(|v| v as f64) .unwrap_or_else(proxmox_time::epoch_f64); - let mut rrd = RRD::load(&path)?; + let mut rrd = RRD::load(&path, false)?; rrd.update(time, value); - rrd.save(&path, CreateOptions::new())?; + rrd.save(&path, CreateOptions::new(), false)?; Ok(()) } @@ -149,7 +149,7 @@ pub fn fetch_rrd( end: Option, ) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path))?; + let rrd = RRD::load(&PathBuf::from(path), false)?; let data = rrd.extract_data(cf, resolution, start, end)?; @@ -177,7 +177,7 @@ pub fn first_update_time( rra_index: usize, ) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path))?; + let rrd = RRD::load(&PathBuf::from(path), false)?; if rra_index >= rrd.rra_list.len() { bail!("rra-index is out of range"); @@ -202,7 +202,7 @@ pub fn first_update_time( /// Return the Unix timestamp of the last update pub fn last_update_time(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path))?; + let rrd = RRD::load(&PathBuf::from(path), false)?; println!("{}", rrd.source.last_update); Ok(()) @@ -220,7 +220,7 @@ pub fn last_update_time(path: String) -> Result<(), Error> { /// Return the time and value from the last update pub fn last_update(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path))?; + let rrd = RRD::load(&PathBuf::from(path), false)?; let result = json!({ "time": rrd.source.last_update, @@ -272,7 +272,7 @@ pub fn create_rrd( let rrd = RRD::new(dst, rra_list); - rrd.save(&path, CreateOptions::new())?; + rrd.save(&path, CreateOptions::new(), false)?; Ok(()) } @@ -302,7 +302,7 @@ pub fn resize_rrd( let path = PathBuf::from(&path); - let mut rrd = RRD::load(&path)?; + let mut rrd = RRD::load(&path, false)?; if rra_index >= rrd.rra_list.len() { bail!("rra-index is out of range"); @@ -331,7 +331,7 @@ pub fn resize_rrd( rrd.rra_list[rra_index] = new_rra; - rrd.save(&path, CreateOptions::new())?; + rrd.save(&path, CreateOptions::new(), false)?; Ok(()) } diff --git a/proxmox-rrd/tests/file_format_test.rs b/proxmox-rrd/tests/file_format_test.rs index cecb242d..b0e0e894 100644 --- a/proxmox-rrd/tests/file_format_test.rs +++ b/proxmox-rrd/tests/file_format_test.rs @@ -28,11 +28,11 @@ const RRD_V2_FN: &str = "./tests/testdata/cpu.rrd_v2"; #[test] fn upgrade_from_rrd_v1() -> Result<(), Error> { - let rrd = RRD::load(Path::new(RRD_V1_FN))?; + let rrd = RRD::load(Path::new(RRD_V1_FN), true)?; const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.upgraded"; let new_path = Path::new(RRD_V2_NEW_FN); - rrd.save(new_path, CreateOptions::new())?; + rrd.save(new_path, CreateOptions::new(), true)?; let result = compare_file(RRD_V2_FN, RRD_V2_NEW_FN); let _ = std::fs::remove_file(RRD_V2_NEW_FN); @@ -45,11 +45,11 @@ fn upgrade_from_rrd_v1() -> Result<(), Error> { #[test] fn load_and_save_rrd_v2() -> Result<(), Error> { - let rrd = RRD::load(Path::new(RRD_V2_FN))?; + let rrd = RRD::load(Path::new(RRD_V2_FN), true)?; const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.saved"; let new_path = Path::new(RRD_V2_NEW_FN); - rrd.save(new_path, CreateOptions::new())?; + rrd.save(new_path, CreateOptions::new(), true)?; let result = compare_file(RRD_V2_FN, RRD_V2_NEW_FN); let _ = std::fs::remove_file(RRD_V2_NEW_FN); From 412712029cf155d664dcd655c29e899f722dc62f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Oct 2021 18:33:19 +0200 Subject: [PATCH 076/111] proxmox-rrd: use fsync instead of syncfs syncfs can sync unrelated data, and we do not want that. Signed-off-by: Dietmar Maurer --- proxmox-rrd/src/cache.rs | 62 +++++++++++++++++++++++++++++--- proxmox-rrd/src/cache/journal.rs | 11 +++--- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index e4dd2489..9bbd89de 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -4,6 +4,9 @@ use std::sync::{Arc, RwLock}; use std::io::{BufRead, BufReader}; use std::time::SystemTime; use std::thread::spawn; +use std::os::unix::io::AsRawFd; +use std::collections::BTreeSet; + use crossbeam_channel::{bounded, TryRecvError}; use anyhow::{format_err, bail, Error}; @@ -127,6 +130,7 @@ impl RRDCache { /// Apply and commit the journal. Should be used at server startup. pub fn apply_journal(&self) -> Result { + let config = Arc::clone(&self.config); let state = Arc::clone(&self.state); let rrd_map = Arc::clone(&self.rrd_map); @@ -168,7 +172,7 @@ impl RRDCache { state_guard.apply_thread_result = Some(receiver); spawn(move || { - let result = apply_and_commit_journal_thread(state, rrd_map, journal_applied) + let result = apply_and_commit_journal_thread(config, state, rrd_map, journal_applied) .map_err(|err| err.to_string()); sender.send(result).unwrap(); }); @@ -219,6 +223,7 @@ impl RRDCache { fn apply_and_commit_journal_thread( + config: Arc, state: Arc>, rrd_map: Arc>, commit_only: bool, @@ -242,7 +247,7 @@ fn apply_and_commit_journal_thread( let start_time = SystemTime::now(); log::debug!("commit rrd journal"); - match commit_journal_impl(state, rrd_map) { + match commit_journal_impl(config, state, rrd_map) { Ok(rrd_file_count) => { let elapsed = start_time.elapsed().unwrap().as_secs_f64(); log::info!("rrd journal successfully committed ({} files in {:.3} seconds)", @@ -346,7 +351,32 @@ fn apply_journal_impl( Ok(lines) } +fn fsync_file_or_dir(path: &Path) -> Result<(), Error> { + let file = std::fs::File::open(path)?; + nix::unistd::fsync(file.as_raw_fd())?; + Ok(()) +} + +pub(crate)fn fsync_file_and_parent(path: &Path) -> Result<(), Error> { + let file = std::fs::File::open(path)?; + nix::unistd::fsync(file.as_raw_fd())?; + if let Some(parent) = path.parent() { + fsync_file_or_dir(parent)?; + } + Ok(()) +} + +fn rrd_parent_dir(basedir: &Path, rel_path: &str) -> PathBuf { + let mut path = basedir.to_owned(); + let rel_path = Path::new(rel_path); + if let Some(parent) = rel_path.parent() { + path.push(parent); + } + path +} + fn commit_journal_impl( + config: Arc, state: Arc>, rrd_map: Arc>, ) -> Result { @@ -356,8 +386,15 @@ fn commit_journal_impl( let mut rrd_file_count = 0; let mut errors = 0; + let mut dir_set = BTreeSet::new(); + + log::info!("write rrd data back to disk"); + // save all RRDs - we only need a read lock here + // Note: no fsync here (we do it afterwards) for rel_path in files.iter() { + let parent_dir = rrd_parent_dir(&config.basedir, &rel_path); + dir_set.insert(parent_dir); rrd_file_count += 1; if let Err(err) = rrd_map.read().unwrap().flush_rrd_file(&rel_path) { errors += 1; @@ -365,12 +402,29 @@ fn commit_journal_impl( } } - state.read().unwrap().syncfs()?; - if errors != 0 { bail!("errors during rrd flush - unable to commit rrd journal"); } + // Important: We fsync files after writing all data! This increase + // the likelihood that files are already synced, so this is + // much faster (although we need to re-open the files). + + log::info!("starting rrd data sync"); + + for rel_path in files.iter() { + let mut path = config.basedir.clone(); + path.push(&rel_path); + fsync_file_or_dir(&path) + .map_err(|err| format_err!("fsync rrd file {} failed - {}", rel_path, err))?; + } + + // also fsync directories + for dir_path in dir_set { + fsync_file_or_dir(&dir_path) + .map_err(|err| format_err!("fsync rrd dir {:?} failed - {}", dir_path, err))?; + } + // if everything went ok, remove the old journal files state.write().unwrap().remove_old_journals()?; diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index 1f9c67b9..85e3906a 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -88,11 +88,6 @@ impl JournalState { Ok(()) } - pub fn syncfs(&self) -> Result<(), nix::Error> { - let res = unsafe { libc::syncfs(self.journal.as_raw_fd()) }; - nix::errno::Errno::result(res).map(drop) - } - pub fn append_journal_entry( &mut self, time: f64, @@ -143,9 +138,13 @@ impl JournalState { let mut new_name = journal_path.clone(); let now = proxmox_time::epoch_i64(); new_name.set_extension(format!("journal-{:08x}", now)); - std::fs::rename(journal_path, new_name)?; + std::fs::rename(journal_path, &new_name)?; self.journal = Self::open_journal_writer(&self.config)?; + + // make sure the old journal data landed on the disk + super::fsync_file_and_parent(&new_name)?; + Ok(()) } From 75ca726c295c244bc69007bea7039fd9ece810b3 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Oct 2021 14:56:15 +0200 Subject: [PATCH 077/111] use new fsync parameter to replace_file and atomic_open_or_create Depend on proxmox 0.15.0 and proxmox-openid 0.8.1 Signed-off-by: Dietmar Maurer --- proxmox-rrd/Cargo.toml | 2 +- proxmox-rrd/src/cache/journal.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 67095689..7e1b2cb6 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -19,6 +19,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_cbor = "0.11.1" -proxmox = { version = "0.14.0" } +proxmox = { version = "0.15.0" } proxmox-time = "1" proxmox-schema = { version = "1", features = [ "api-macro" ] } diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index 85e3906a..a85154a4 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -113,6 +113,7 @@ impl JournalState { flags, &[], self.config.file_options.clone(), + false, )?; Ok(BufReader::new(journal)) } @@ -127,6 +128,7 @@ impl JournalState { flags, &[], config.file_options.clone(), + false, )?; Ok(journal) } From 8e344d3d677ecca335edc030274195ce094c9675 Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Thu, 28 Oct 2021 11:40:44 +0200 Subject: [PATCH 078/111] rrd: use saturating_sub to avoid underflow Without this, the tests fail in debug mode. Also having start (u64) underflow to a value greater than end does not really make sense Signed-off-by: Dominik Csapak Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/src/rrd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 20bf6ae3..2fab9df3 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -469,7 +469,7 @@ impl RRD { match rra { Some(rra) => { let end = end.unwrap_or_else(|| proxmox_time::epoch_f64() as u64); - let start = start.unwrap_or(end - 10*rra.resolution); + let start = start.unwrap_or(end.saturating_sub(10*rra.resolution)); Ok(rra.extract_data(start, end, self.source.last_update)) } None => bail!("unable to find RRA suitable ({:?}:{})", cf, resolution), From 0fdc15a3f8802c900d39d48d30f379a72c5d4699 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 18 Nov 2021 13:43:41 +0100 Subject: [PATCH 079/111] use proxmox::tools::fd::fd_change_cloexec from proxmox 0.15.3 Depend on proxmox 0.15.3 Signed-off-by: Dietmar Maurer --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 7e1b2cb6..8e9d1b77 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -19,6 +19,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_cbor = "0.11.1" -proxmox = { version = "0.15.0" } +proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1", features = [ "api-macro" ] } From a092ef9c32baef05ab144073d2a9af3efe202863 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 23 Nov 2021 17:57:00 +0100 Subject: [PATCH 080/111] update to proxmox-sys 0.2 crate - imported pbs-api-types/src/common_regex.rs from old proxmox crate - use hex crate to generate/parse hex digest - remove all reference to proxmox crate (use proxmox-sys and proxmox-serde instead) Signed-off-by: Dietmar Maurer --- proxmox-rrd/Cargo.toml | 3 ++- proxmox-rrd/examples/prrd.rs | 2 +- proxmox-rrd/src/cache.rs | 2 +- proxmox-rrd/src/cache/journal.rs | 2 +- proxmox-rrd/src/cache/rrd_map.rs | 2 +- proxmox-rrd/src/rrd.rs | 2 +- proxmox-rrd/tests/file_format_test.rs | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 8e9d1b77..3e47ea00 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_cbor = "0.11.1" -proxmox = { version = "0.15.3" } +#proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1", features = [ "api-macro" ] } +proxmox-sys = "0.2" \ No newline at end of file diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index e00e5a0e..6b6f59ce 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -11,7 +11,7 @@ use proxmox_router::cli::{run_cli_command, complete_file_name, CliCommand, CliCo use proxmox_schema::{api, parse_property_string}; use proxmox_schema::{ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema}; -use proxmox::tools::fs::CreateOptions; +use proxmox_sys::fs::CreateOptions; use proxmox_rrd::rrd::{CF, DST, RRA, RRD}; diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 9bbd89de..b786f14f 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -10,7 +10,7 @@ use std::collections::BTreeSet; use crossbeam_channel::{bounded, TryRecvError}; use anyhow::{format_err, bail, Error}; -use proxmox::tools::fs::{create_path, CreateOptions}; +use proxmox_sys::fs::{create_path, CreateOptions}; use crate::rrd::{DST, CF, RRD, RRA}; diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index a85154a4..fbc8773c 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -10,7 +10,7 @@ use anyhow::{bail, format_err, Error}; use nix::fcntl::OFlag; use crossbeam_channel::Receiver; -use proxmox::tools::fs::atomic_open_or_create_file; +use proxmox_sys::fs::atomic_open_or_create_file; const RRD_JOURNAL_NAME: &str = "rrd.journal"; diff --git a/proxmox-rrd/src/cache/rrd_map.rs b/proxmox-rrd/src/cache/rrd_map.rs index 6f7686d6..6577fb2e 100644 --- a/proxmox-rrd/src/cache/rrd_map.rs +++ b/proxmox-rrd/src/cache/rrd_map.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use anyhow::{bail, Error}; -use proxmox::tools::fs::create_path; +use proxmox_sys::fs::create_path; use crate::rrd::{CF, DST, RRD}; diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 2fab9df3..2aebe1ae 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -18,7 +18,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; use anyhow::{bail, format_err, Error}; use serde::{Serialize, Deserialize}; -use proxmox::tools::fs::{make_tmp_file, CreateOptions}; +use proxmox_sys::fs::{make_tmp_file, CreateOptions}; use proxmox_schema::api; use crate::rrd_v1; diff --git a/proxmox-rrd/tests/file_format_test.rs b/proxmox-rrd/tests/file_format_test.rs index b0e0e894..81e49ca3 100644 --- a/proxmox-rrd/tests/file_format_test.rs +++ b/proxmox-rrd/tests/file_format_test.rs @@ -4,7 +4,7 @@ use std::process::Command; use anyhow::{bail, Error}; use proxmox_rrd::rrd::RRD; -use proxmox::tools::fs::CreateOptions; +use proxmox_sys::fs::CreateOptions; fn compare_file(fn1: &str, fn2: &str) -> Result<(), Error> { From e207a84d932f34fa66448373d62fbcaa83496388 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 16 Dec 2021 11:08:44 +0100 Subject: [PATCH 081/111] bump proxmox-schema to 1.1 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 3e47ea00..12641ff6 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -21,5 +21,5 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" -proxmox-schema = { version = "1", features = [ "api-macro" ] } +proxmox-schema = { version = "1.1", features = [ "api-macro" ] } proxmox-sys = "0.2" \ No newline at end of file From cb32acc7034963486c8fd59088888bc5f9a635c2 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 16 Dec 2021 11:02:53 +0100 Subject: [PATCH 082/111] cleanup schema function calls Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/examples/prrd.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index 6b6f59ce..8d2f00ea 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -8,8 +8,7 @@ use serde_json::json; use proxmox_router::RpcEnvironment; use proxmox_router::cli::{run_cli_command, complete_file_name, CliCommand, CliCommandMap, CliEnvironment}; -use proxmox_schema::{api, parse_property_string}; -use proxmox_schema::{ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema}; +use proxmox_schema::{api, ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema}; use proxmox_sys::fs::CreateOptions; @@ -262,7 +261,7 @@ pub fn create_rrd( for item in rra.iter() { let rra: RRAConfig = serde_json::from_value( - parse_property_string(item, &RRAConfig::API_SCHEMA)? + RRAConfig::API_SCHEMA.parse_property_string(item)? )?; println!("GOT {:?}", rra); rra_list.push(RRA::new(rra.cf, rra.r, rra.n as usize)); From 5b193680007f39f78e7fd4243bb4db0e874292b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 30 Dec 2021 12:57:37 +0100 Subject: [PATCH 083/111] tree-wide: fix needless borrows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit found and fixed via clippy Signed-off-by: Fabian Grünbichler --- proxmox-rrd/src/cache.rs | 4 ++-- proxmox-rrd/src/rrd.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index b786f14f..622a0589 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -393,10 +393,10 @@ fn commit_journal_impl( // save all RRDs - we only need a read lock here // Note: no fsync here (we do it afterwards) for rel_path in files.iter() { - let parent_dir = rrd_parent_dir(&config.basedir, &rel_path); + let parent_dir = rrd_parent_dir(&config.basedir, rel_path); dir_set.insert(parent_dir); rrd_file_count += 1; - if let Err(err) = rrd_map.read().unwrap().flush_rrd_file(&rel_path) { + if let Err(err) = rrd_map.read().unwrap().flush_rrd_file(rel_path) { errors += 1; log::error!("unable to save rrd {}: {}", rel_path, err); } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 2aebe1ae..4b48d0cf 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -317,7 +317,7 @@ impl RRD { } let rrd = if raw[0..8] == rrd_v1::PROXMOX_RRD_MAGIC_1_0 { - let v1 = rrd_v1::RRDv1::from_raw(&raw)?; + let v1 = rrd_v1::RRDv1::from_raw(raw)?; v1.to_rrd_v2() .map_err(|err| format_err!("unable to convert from old V1 format - {}", err))? } else if raw[0..8] == PROXMOX_RRD_MAGIC_2_0 { From 872b5f41cd361ea086d6546dbf139bccb20c7adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 30 Dec 2021 13:20:03 +0100 Subject: [PATCH 084/111] tree-wide: drop redundant clones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- proxmox-rrd/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 622a0589..ff97c3e5 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -70,8 +70,8 @@ impl RRDCache { .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; let config = Arc::new(CacheConfig { - basedir: basedir.clone(), - file_options: file_options.clone(), + basedir: basedir, + file_options: file_options, dir_options: dir_options, apply_interval, }); From c8e73a225a8083925a77fef2c6a4fa59af6fb8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 30 Dec 2021 14:18:43 +0100 Subject: [PATCH 085/111] rrd: drop redundant field names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- proxmox-rrd/src/cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index ff97c3e5..6b2a84af 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -70,9 +70,9 @@ impl RRDCache { .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; let config = Arc::new(CacheConfig { - basedir: basedir, - file_options: file_options, - dir_options: dir_options, + basedir, + file_options, + dir_options, apply_interval, }); From d80d195c26b5234880207cb4bcf4dadbe08f1c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Tue, 8 Feb 2022 14:57:16 +0100 Subject: [PATCH 086/111] misc clippy fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the trivial ones ;) Signed-off-by: Fabian Grünbichler --- proxmox-rrd/examples/prrd.rs | 2 +- proxmox-rrd/src/cache.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index 8d2f00ea..5fa120e4 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -52,7 +52,7 @@ pub fn dump_rrd(path: String) -> Result<(), Error> { let rrd = RRD::load(&PathBuf::from(path), false)?; serde_json::to_writer_pretty(std::io::stdout(), &rrd)?; - println!(""); + println!(); Ok(()) } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 6b2a84af..edd46ed3 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -63,8 +63,8 @@ impl RRDCache { ) -> Result { let basedir = basedir.as_ref().to_owned(); - let file_options = file_options.unwrap_or_else(|| CreateOptions::new()); - let dir_options = dir_options.unwrap_or_else(|| CreateOptions::new()); + let file_options = file_options.unwrap_or_else(CreateOptions::new); + let dir_options = dir_options.unwrap_or_else(CreateOptions::new); create_path(&basedir, Some(dir_options.clone()), Some(dir_options.clone())) .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; From 6149c171cabc11686192219f5ae9d8566bd55ec2 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 15 Feb 2022 07:53:29 +0100 Subject: [PATCH 087/111] rrd cache: code style, avoid useless intermediate mutable Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/cache.rs | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index edd46ed3..ec90cd85 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -101,24 +101,20 @@ impl RRDCache { /// /// The resultion data file size is about 80KB. pub fn create_proxmox_backup_default_rrd(dst: DST) -> RRD { - - let mut rra_list = Vec::new(); - - // 1min * 1440 => 1day - rra_list.push(RRA::new(CF::Average, 60, 1440)); - rra_list.push(RRA::new(CF::Maximum, 60, 1440)); - - // 30min * 1440 => 30days = 1month - rra_list.push(RRA::new(CF::Average, 30*60, 1440)); - rra_list.push(RRA::new(CF::Maximum, 30*60, 1440)); - - // 6h * 1440 => 360days = 1year - rra_list.push(RRA::new(CF::Average, 6*3600, 1440)); - rra_list.push(RRA::new(CF::Maximum, 6*3600, 1440)); - - // 1week * 570 => 10years - rra_list.push(RRA::new(CF::Average, 7*86400, 570)); - rra_list.push(RRA::new(CF::Maximum, 7*86400, 570)); + let rra_list = vec![ + // 1 min * 1440 => 1 day + RRA::new(CF::Average, 60, 1440), + RRA::new(CF::Maximum, 60, 1440), + // 30 min * 1440 => 30 days ~ 1 month + RRA::new(CF::Average, 30 * 60, 1440), + RRA::new(CF::Maximum, 30 * 60, 1440), + // 6 h * 1440 => 360 days ~ 1 year + RRA::new(CF::Average, 6 * 3600, 1440), + RRA::new(CF::Maximum, 6 * 3600, 1440), + // 1 week * 570 => 10 years + RRA::new(CF::Average, 7 * 86400, 570), + RRA::new(CF::Maximum, 7 * 86400, 570), + ]; RRD::new(dst, rra_list) } From 5740f36ef8c02496ca487f8da63de401118964e9 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 15 Feb 2022 07:55:08 +0100 Subject: [PATCH 088/111] rrd: avoid intermediate index, directly loop over data Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 4b48d0cf..5fd7d0e2 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -177,8 +177,8 @@ impl RRA { let mut index = self.slot(start); - for i in 0..data.len() { - if let Some(v) = data[i] { + for item in data { + if let Some(v) = item { self.data[index] = v; } index += 1; if index >= self.data.len() { index = 0; } From 43b602248d4a044c7d84b8bd268f8eba13c80633 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 15 Feb 2022 07:58:18 +0100 Subject: [PATCH 089/111] rrd: extract data: avoid always calculating start-time fallback Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 5fd7d0e2..1d5d665f 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -469,7 +469,7 @@ impl RRD { match rra { Some(rra) => { let end = end.unwrap_or_else(|| proxmox_time::epoch_f64() as u64); - let start = start.unwrap_or(end.saturating_sub(10*rra.resolution)); + let start = start.unwrap_or_else(|| end.saturating_sub(10*rra.resolution)); Ok(rra.extract_data(start, end, self.source.last_update)) } None => bail!("unable to find RRA suitable ({:?}:{})", cf, resolution), From 42cafaba32b3f64783c6c29123335d28bd6e6031 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Fri, 4 Mar 2022 09:50:21 +0100 Subject: [PATCH 090/111] bump proxmox-schema dep to 1.3 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 12641ff6..9f6510e3 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -21,5 +21,5 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" -proxmox-schema = { version = "1.1", features = [ "api-macro" ] } +proxmox-schema = { version = "1.3", features = [ "api-macro" ] } proxmox-sys = "0.2" \ No newline at end of file From d5b9d1f48223ac66ff1a20f18f85f7e42a361d87 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Wed, 6 Apr 2022 16:56:33 +0200 Subject: [PATCH 091/111] rrd: rust fmt Signed-off-by: Thomas Lamprecht --- proxmox-rrd/examples/prrd.rs | 95 +++++++----------- proxmox-rrd/src/cache.rs | 79 +++++++++------ proxmox-rrd/src/cache/journal.rs | 49 ++++----- proxmox-rrd/src/cache/rrd_map.rs | 5 +- proxmox-rrd/src/rrd.rs | 137 +++++++++++++++----------- proxmox-rrd/src/rrd_v1.rs | 67 +++++++------ proxmox-rrd/tests/file_format_test.rs | 3 - 7 files changed, 226 insertions(+), 209 deletions(-) diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index 5fa120e4..081cef98 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -3,24 +3,22 @@ use std::path::PathBuf; use anyhow::{bail, Error}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use serde_json::json; +use proxmox_router::cli::{ + complete_file_name, run_cli_command, CliCommand, CliCommandMap, CliEnvironment, +}; use proxmox_router::RpcEnvironment; -use proxmox_router::cli::{run_cli_command, complete_file_name, CliCommand, CliCommandMap, CliEnvironment}; use proxmox_schema::{api, ApiStringFormat, ApiType, IntegerSchema, Schema, StringSchema}; use proxmox_sys::fs::CreateOptions; use proxmox_rrd::rrd::{CF, DST, RRA, RRD}; -pub const RRA_INDEX_SCHEMA: Schema = IntegerSchema::new( - "Index of the RRA.") - .minimum(0) - .schema(); +pub const RRA_INDEX_SCHEMA: Schema = IntegerSchema::new("Index of the RRA.").minimum(0).schema(); -pub const RRA_CONFIG_STRING_SCHEMA: Schema = StringSchema::new( - "RRA configuration") +pub const RRA_CONFIG_STRING_SCHEMA: Schema = StringSchema::new("RRA configuration") .format(&ApiStringFormat::PropertyString(&RRAConfig::API_SCHEMA)) .schema(); @@ -49,7 +47,6 @@ pub struct RRAConfig { )] /// Dump the RRD file in JSON format pub fn dump_rrd(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path), false)?; serde_json::to_writer_pretty(std::io::stdout(), &rrd)?; println!(); @@ -67,14 +64,19 @@ pub fn dump_rrd(path: String) -> Result<(), Error> { )] /// RRD file information pub fn rrd_info(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path), false)?; println!("DST: {:?}", rrd.source.dst); for (i, rra) in rrd.rra_list.iter().enumerate() { // use RRAConfig property string format - println!("RRA[{}]: {:?},r={},n={}", i, rra.cf, rra.resolution, rra.data.len()); + println!( + "RRA[{}]: {:?},r={},n={}", + i, + rra.cf, + rra.resolution, + rra.data.len() + ); } Ok(()) @@ -97,15 +99,11 @@ pub fn rrd_info(path: String) -> Result<(), Error> { }, )] /// Update the RRD database -pub fn update_rrd( - path: String, - time: Option, - value: f64, -) -> Result<(), Error> { - +pub fn update_rrd(path: String, time: Option, value: f64) -> Result<(), Error> { let path = PathBuf::from(path); - let time = time.map(|v| v as f64) + let time = time + .map(|v| v as f64) .unwrap_or_else(proxmox_time::epoch_f64); let mut rrd = RRD::load(&path, false)?; @@ -147,7 +145,6 @@ pub fn fetch_rrd( start: Option, end: Option, ) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path), false)?; let data = rrd.extract_data(cf, resolution, start, end)?; @@ -171,18 +168,14 @@ pub fn fetch_rrd( )] /// Return the Unix timestamp of the first time slot inside the /// specified RRA (slot start time) -pub fn first_update_time( - path: String, - rra_index: usize, -) -> Result<(), Error> { - +pub fn first_update_time(path: String, rra_index: usize) -> Result<(), Error> { let rrd = RRD::load(&PathBuf::from(path), false)?; if rra_index >= rrd.rra_list.len() { bail!("rra-index is out of range"); } let rra = &rrd.rra_list[rra_index]; - let duration = (rra.data.len() as u64)*rra.resolution; + let duration = (rra.data.len() as u64) * rra.resolution; let first = rra.slot_start_time((rrd.source.last_update as u64).saturating_sub(duration)); println!("{}", first); @@ -200,7 +193,6 @@ pub fn first_update_time( )] /// Return the Unix timestamp of the last update pub fn last_update_time(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path), false)?; println!("{}", rrd.source.last_update); @@ -218,7 +210,6 @@ pub fn last_update_time(path: String) -> Result<(), Error> { )] /// Return the time and value from the last update pub fn last_update(path: String) -> Result<(), Error> { - let rrd = RRD::load(&PathBuf::from(path), false)?; let result = json!({ @@ -251,18 +242,12 @@ pub fn last_update(path: String) -> Result<(), Error> { }, )] /// Create a new RRD file -pub fn create_rrd( - dst: DST, - path: String, - rra: Vec, -) -> Result<(), Error> { - +pub fn create_rrd(dst: DST, path: String, rra: Vec) -> Result<(), Error> { let mut rra_list = Vec::new(); for item in rra.iter() { - let rra: RRAConfig = serde_json::from_value( - RRAConfig::API_SCHEMA.parse_property_string(item)? - )?; + let rra: RRAConfig = + serde_json::from_value(RRAConfig::API_SCHEMA.parse_property_string(item)?)?; println!("GOT {:?}", rra); rra_list.push(RRA::new(rra.cf, rra.r, rra.n as usize)); } @@ -293,12 +278,7 @@ pub fn create_rrd( }, )] /// Resize. Change the number of data slots for the specified RRA. -pub fn resize_rrd( - path: String, - rra_index: usize, - slots: i64, -) -> Result<(), Error> { - +pub fn resize_rrd(path: String, rra_index: usize, slots: i64) -> Result<(), Error> { let path = PathBuf::from(&path); let mut rrd = RRD::load(&path, false)?; @@ -315,12 +295,12 @@ pub fn resize_rrd( bail!("numer of new slots is too small ('{}' < 1)", new_slots); } - if new_slots > 1024*1024 { + if new_slots > 1024 * 1024 { bail!("numer of new slots is too big ('{}' > 1M)", new_slots); } let rra_end = rra.slot_end_time(rrd.source.last_update as u64); - let rra_start = rra_end - rra.resolution*(rra.data.len() as u64); + let rra_start = rra_end - rra.resolution * (rra.data.len() as u64); let (start, reso, data) = rra.extract_data(rra_start, rra_end, rrd.source.last_update); let mut new_rra = RRA::new(rra.cf, rra.resolution, new_slots as usize); @@ -336,7 +316,6 @@ pub fn resize_rrd( } fn main() -> Result<(), Error> { - let uid = nix::unistd::Uid::current(); let username = match nix::unistd::User::from_uid(uid)? { @@ -349,57 +328,56 @@ fn main() -> Result<(), Error> { "create", CliCommand::new(&API_METHOD_CREATE_RRD) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) + .completion_cb("path", complete_file_name), ) .insert( "dump", CliCommand::new(&API_METHOD_DUMP_RRD) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) - ) + .completion_cb("path", complete_file_name), + ) .insert( "fetch", CliCommand::new(&API_METHOD_FETCH_RRD) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) - ) + .completion_cb("path", complete_file_name), + ) .insert( "first", CliCommand::new(&API_METHOD_FIRST_UPDATE_TIME) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) + .completion_cb("path", complete_file_name), ) .insert( "info", CliCommand::new(&API_METHOD_RRD_INFO) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) + .completion_cb("path", complete_file_name), ) .insert( "last", CliCommand::new(&API_METHOD_LAST_UPDATE_TIME) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) + .completion_cb("path", complete_file_name), ) .insert( "lastupdate", CliCommand::new(&API_METHOD_LAST_UPDATE) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) + .completion_cb("path", complete_file_name), ) .insert( "resize", CliCommand::new(&API_METHOD_RESIZE_RRD) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) + .completion_cb("path", complete_file_name), ) .insert( "update", CliCommand::new(&API_METHOD_UPDATE_RRD) .arg_param(&["path"]) - .completion_cb("path", complete_file_name) - ) - ; + .completion_cb("path", complete_file_name), + ); let mut rpcenv = CliEnvironment::new(); rpcenv.set_auth_id(Some(format!("{}@pam", username))); @@ -407,5 +385,4 @@ fn main() -> Result<(), Error> { run_cli_command(cmd_def, rpcenv, None); Ok(()) - } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index ec90cd85..90e4e470 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -1,18 +1,18 @@ +use std::collections::BTreeSet; use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use std::io::{BufRead, BufReader}; -use std::time::SystemTime; use std::thread::spawn; -use std::os::unix::io::AsRawFd; -use std::collections::BTreeSet; +use std::time::SystemTime; +use anyhow::{bail, format_err, Error}; use crossbeam_channel::{bounded, TryRecvError}; -use anyhow::{format_err, bail, Error}; use proxmox_sys::fs::{create_path, CreateOptions}; -use crate::rrd::{DST, CF, RRD, RRA}; +use crate::rrd::{CF, DST, RRA, RRD}; mod journal; use journal::*; @@ -37,9 +37,7 @@ pub(crate) struct CacheConfig { dir_options: CreateOptions, } - impl RRDCache { - /// Creates a new instance /// /// `basedir`: All files are stored relative to this path. @@ -66,8 +64,12 @@ impl RRDCache { let file_options = file_options.unwrap_or_else(CreateOptions::new); let dir_options = dir_options.unwrap_or_else(CreateOptions::new); - create_path(&basedir, Some(dir_options.clone()), Some(dir_options.clone())) - .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; + create_path( + &basedir, + Some(dir_options.clone()), + Some(dir_options.clone()), + ) + .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?; let config = Arc::new(CacheConfig { basedir, @@ -130,7 +132,6 @@ impl RRDCache { let state = Arc::clone(&self.state); let rrd_map = Arc::clone(&self.rrd_map); - let mut state_guard = self.state.write().unwrap(); let journal_applied = state_guard.journal_applied; @@ -160,7 +161,9 @@ impl RRDCache { let now = proxmox_time::epoch_f64(); let wants_commit = (now - state_guard.last_journal_flush) > self.config.apply_interval; - if journal_applied && !wants_commit { return Ok(journal_applied); } + if journal_applied && !wants_commit { + return Ok(journal_applied); + } state_guard.last_journal_flush = proxmox_time::epoch_f64(); @@ -176,7 +179,6 @@ impl RRDCache { Ok(journal_applied) } - /// Update data in RAM and write file back to disk (journal) pub fn update_value( &self, @@ -185,14 +187,18 @@ impl RRDCache { value: f64, dst: DST, ) -> Result<(), Error> { - let journal_applied = self.apply_journal()?; - self.state.write().unwrap() + self.state + .write() + .unwrap() .append_journal_entry(time, value, dst, rel_path)?; if journal_applied { - self.rrd_map.write().unwrap().update(rel_path, time, value, dst, false)?; + self.rrd_map + .write() + .unwrap() + .update(rel_path, time, value, dst, false)?; } Ok(()) @@ -212,19 +218,19 @@ impl RRDCache { start: Option, end: Option, ) -> Result>)>, Error> { - self.rrd_map.read().unwrap() + self.rrd_map + .read() + .unwrap() .extract_cached_data(base, name, cf, resolution, start, end) } } - fn apply_and_commit_journal_thread( config: Arc, state: Arc>, rrd_map: Arc>, commit_only: bool, ) -> Result<(), Error> { - if commit_only { state.write().unwrap().rotate_journal()?; // start new journal, keep old one } else { @@ -234,7 +240,11 @@ fn apply_and_commit_journal_thread( match apply_journal_impl(Arc::clone(&state), Arc::clone(&rrd_map)) { Ok(entries) => { let elapsed = start_time.elapsed().unwrap().as_secs_f64(); - log::info!("applied rrd journal ({} entries in {:.3} seconds)", entries, elapsed); + log::info!( + "applied rrd journal ({} entries in {:.3} seconds)", + entries, + elapsed + ); } Err(err) => bail!("apply rrd journal failed - {}", err), } @@ -246,8 +256,11 @@ fn apply_and_commit_journal_thread( match commit_journal_impl(config, state, rrd_map) { Ok(rrd_file_count) => { let elapsed = start_time.elapsed().unwrap().as_secs_f64(); - log::info!("rrd journal successfully committed ({} files in {:.3} seconds)", - rrd_file_count, elapsed); + log::info!( + "rrd journal successfully committed ({} files in {:.3} seconds)", + rrd_file_count, + elapsed + ); } Err(err) => bail!("rrd journal commit failed: {}", err), } @@ -261,7 +274,6 @@ fn apply_journal_lines( reader: &mut BufReader, lock_read_line: bool, ) -> Result { - let mut linenr = 0; loop { @@ -274,20 +286,30 @@ fn apply_journal_lines( reader.read_line(&mut line)? }; - if len == 0 { break; } + if len == 0 { + break; + } let entry: JournalEntry = match line.parse() { Ok(entry) => entry, Err(err) => { log::warn!( "unable to parse rrd journal '{}' line {} (skip) - {}", - journal_name, linenr, err, + journal_name, + linenr, + err, ); continue; // skip unparsable lines } }; - rrd_map.write().unwrap().update(&entry.rel_path, entry.time, entry.value, entry.dst, true)?; + rrd_map.write().unwrap().update( + &entry.rel_path, + entry.time, + entry.value, + entry.dst, + true, + )?; } Ok(linenr) } @@ -296,7 +318,6 @@ fn apply_journal_impl( state: Arc>, rrd_map: Arc>, ) -> Result { - let mut lines = 0; // Apply old journals first @@ -343,7 +364,6 @@ fn apply_journal_impl( state_guard.journal_applied = true; } - Ok(lines) } @@ -353,7 +373,7 @@ fn fsync_file_or_dir(path: &Path) -> Result<(), Error> { Ok(()) } -pub(crate)fn fsync_file_and_parent(path: &Path) -> Result<(), Error> { +pub(crate) fn fsync_file_and_parent(path: &Path) -> Result<(), Error> { let file = std::fs::File::open(path)?; nix::unistd::fsync(file.as_raw_fd())?; if let Some(parent) = path.parent() { @@ -376,7 +396,6 @@ fn commit_journal_impl( state: Arc>, rrd_map: Arc>, ) -> Result { - let files = rrd_map.read().unwrap().file_list(); let mut rrd_file_count = 0; diff --git a/proxmox-rrd/src/cache/journal.rs b/proxmox-rrd/src/cache/journal.rs index fbc8773c..7c260e1e 100644 --- a/proxmox-rrd/src/cache/journal.rs +++ b/proxmox-rrd/src/cache/journal.rs @@ -1,21 +1,21 @@ -use std::fs::File; -use std::path::PathBuf; -use std::sync::Arc; -use std::io::{Write, BufReader}; use std::ffi::OsStr; +use std::fs::File; +use std::io::{BufReader, Write}; use std::os::unix::io::AsRawFd; +use std::path::PathBuf; use std::str::FromStr; +use std::sync::Arc; use anyhow::{bail, format_err, Error}; -use nix::fcntl::OFlag; use crossbeam_channel::Receiver; +use nix::fcntl::OFlag; use proxmox_sys::fs::atomic_open_or_create_file; const RRD_JOURNAL_NAME: &str = "rrd.journal"; -use crate::rrd::DST; use crate::cache::CacheConfig; +use crate::rrd::DST; // shared state behind RwLock pub struct JournalState { @@ -36,20 +36,22 @@ pub struct JournalEntry { impl FromStr for JournalEntry { type Err = Error; - fn from_str(line: &str) -> Result { - - let line = line.trim(); + fn from_str(line: &str) -> Result { + let line = line.trim(); let parts: Vec<&str> = line.splitn(4, ':').collect(); if parts.len() != 4 { bail!("wrong numper of components"); } - let time: f64 = parts[0].parse() + let time: f64 = parts[0] + .parse() .map_err(|_| format_err!("unable to parse time"))?; - let value: f64 = parts[1].parse() + let value: f64 = parts[1] + .parse() .map_err(|_| format_err!("unable to parse value"))?; - let dst: u8 = parts[2].parse() + let dst: u8 = parts[2] + .parse() .map_err(|_| format_err!("unable to parse data source type"))?; let dst = match dst { @@ -60,8 +62,13 @@ impl FromStr for JournalEntry { let rel_path = parts[3].to_string(); - Ok(JournalEntry { time, value, dst, rel_path }) - } + Ok(JournalEntry { + time, + value, + dst, + rel_path, + }) + } } pub struct JournalFileInfo { @@ -71,7 +78,6 @@ pub struct JournalFileInfo { } impl JournalState { - pub(crate) fn new(config: Arc) -> Result { let journal = JournalState::open_journal_writer(&config)?; Ok(Self { @@ -95,19 +101,17 @@ impl JournalState { dst: DST, rel_path: &str, ) -> Result<(), Error> { - let journal_entry = format!( - "{}:{}:{}:{}\n", time, value, dst as u8, rel_path); + let journal_entry = format!("{}:{}:{}:{}\n", time, value, dst as u8, rel_path); self.journal.write_all(journal_entry.as_bytes())?; Ok(()) } pub fn open_journal_reader(&self) -> Result, Error> { - // fixme : dup self.journal instead?? let mut journal_path = self.config.basedir.clone(); journal_path.push(RRD_JOURNAL_NAME); - let flags = OFlag::O_CLOEXEC|OFlag::O_RDONLY; + let flags = OFlag::O_CLOEXEC | OFlag::O_RDONLY; let journal = atomic_open_or_create_file( &journal_path, flags, @@ -122,7 +126,7 @@ impl JournalState { let mut journal_path = config.basedir.clone(); journal_path.push(RRD_JOURNAL_NAME); - let flags = OFlag::O_CLOEXEC|OFlag::O_WRONLY|OFlag::O_APPEND; + let flags = OFlag::O_CLOEXEC | OFlag::O_WRONLY | OFlag::O_APPEND; let journal = atomic_open_or_create_file( &journal_path, flags, @@ -151,7 +155,6 @@ impl JournalState { } pub fn remove_old_journals(&self) -> Result<(), Error> { - let journal_list = self.list_old_journals()?; for entry in journal_list { @@ -167,7 +170,9 @@ impl JournalState { let entry = entry?; let path = entry.path(); - if !path.is_file() { continue; } + if !path.is_file() { + continue; + } match path.file_stem() { None => continue, diff --git a/proxmox-rrd/src/cache/rrd_map.rs b/proxmox-rrd/src/cache/rrd_map.rs index 6577fb2e..56dde2e6 100644 --- a/proxmox-rrd/src/cache/rrd_map.rs +++ b/proxmox-rrd/src/cache/rrd_map.rs @@ -1,6 +1,6 @@ +use std::collections::HashMap; use std::path::Path; use std::sync::Arc; -use std::collections::HashMap; use anyhow::{bail, Error}; @@ -17,7 +17,6 @@ pub struct RRDMap { } impl RRDMap { - pub(crate) fn new( config: Arc, load_rrd_cb: fn(path: &Path, rel_path: &str, dst: DST) -> RRD, @@ -71,7 +70,7 @@ impl RRDMap { } pub fn flush_rrd_file(&self, rel_path: &str) -> Result<(), Error> { - if let Some(rrd) = self.map.get(rel_path) { + if let Some(rrd) = self.map.get(rel_path) { let mut path = self.config.basedir.clone(); path.push(rel_path); rrd.save(&path, self.config.file_options.clone(), true) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 1d5d665f..41af6242 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -11,15 +11,15 @@ //! * Plattform independent (big endian f64, hopefully a standard format?) //! * Arbitrary number of RRAs (dynamically changeable) -use std::path::Path; use std::io::{Read, Write}; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; +use std::path::Path; use anyhow::{bail, format_err, Error}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; -use proxmox_sys::fs::{make_tmp_file, CreateOptions}; use proxmox_schema::api; +use proxmox_sys::fs::{make_tmp_file, CreateOptions}; use crate::rrd_v1; @@ -69,7 +69,6 @@ pub struct DataSource { } impl DataSource { - /// Create a new Instance pub fn new(dst: DST) -> Self { Self { @@ -111,15 +110,13 @@ impl DataSource { value - self.last_value }; self.last_value = value; - value = diff/time_diff; + value = diff / time_diff; } else { self.last_value = value; } Ok(value) } - - } #[derive(Serialize, Deserialize)] @@ -136,7 +133,6 @@ pub struct RRA { } impl RRA { - /// Creates a new instance pub fn new(cf: CF, resolution: u64, points: usize) -> Self { Self { @@ -181,7 +177,10 @@ impl RRA { if let Some(v) = item { self.data[index] = v; } - index += 1; if index >= self.data.len() { index = 0; } + index += 1; + if index >= self.data.len() { + index = 0; + } } Ok(()) } @@ -192,15 +191,18 @@ impl RRA { let reso = self.resolution; let num_entries = self.data.len() as u64; - let min_time = epoch.saturating_sub(num_entries*reso); + let min_time = epoch.saturating_sub(num_entries * reso); let min_time = self.slot_end_time(min_time); - let mut t = last_update.saturating_sub(num_entries*reso); + let mut t = last_update.saturating_sub(num_entries * reso); let mut index = self.slot(t); for _ in 0..num_entries { t += reso; - index += 1; if index >= self.data.len() { index = 0; } + index += 1; + if index >= self.data.len() { + index = 0; + } if t < min_time { self.data[index] = f64::NAN; } else { @@ -233,12 +235,24 @@ impl RRA { self.last_count = 1; } else { let new_value = match self.cf { - CF::Maximum => if last_value > value { last_value } else { value }, - CF::Minimum => if last_value < value { last_value } else { value }, + CF::Maximum => { + if last_value > value { + last_value + } else { + value + } + } + CF::Minimum => { + if last_value < value { + last_value + } else { + value + } + } CF::Last => value, CF::Average => { - (last_value*(self.last_count as f64))/(new_count as f64) - + value/(new_count as f64) + (last_value * (self.last_count as f64)) / (new_count as f64) + + value / (new_count as f64) } }; self.data[index] = new_value; @@ -264,12 +278,14 @@ impl RRA { let mut list = Vec::new(); let rrd_end = self.slot_end_time(last_update); - let rrd_start = rrd_end.saturating_sub(reso*num_entries); + let rrd_start = rrd_end.saturating_sub(reso * num_entries); let mut t = start; let mut index = self.slot(t); for _ in 0..num_entries { - if t > end { break; }; + if t > end { + break; + }; if t < rrd_start || t >= rrd_end { list.push(None); } else { @@ -281,7 +297,10 @@ impl RRA { } } t += reso; - index += 1; if index >= self.data.len() { index = 0; } + index += 1; + if index >= self.data.len() { + index = 0; + } } (start, reso, list) @@ -298,17 +317,11 @@ pub struct RRD { } impl RRD { - /// Creates a new Instance pub fn new(dst: DST, rra_list: Vec) -> RRD { - let source = DataSource::new(dst); - RRD { - source, - rra_list, - } - + RRD { source, rra_list } } fn from_raw(raw: &[u8]) -> Result { @@ -340,7 +353,6 @@ impl RRD { /// `fadvise(..,POSIX_FADV_DONTNEED)` to avoid keeping the data in /// the linux page cache. pub fn load(path: &Path, avoid_page_cache: bool) -> Result { - let mut file = std::fs::File::open(path)?; let buffer_size = file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0); let mut raw = Vec::with_capacity(buffer_size); @@ -352,12 +364,16 @@ impl RRD { 0, buffer_size as i64, nix::fcntl::PosixFadviseAdvice::POSIX_FADV_DONTNEED, - ).map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))?; + ) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))?; } match Self::from_raw(&raw) { Ok(rrd) => Ok(rrd), - Err(err) => Err(std::io::Error::new(std::io::ErrorKind::Other, err.to_string())), + Err(err) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + err.to_string(), + )), } } @@ -366,13 +382,12 @@ impl RRD { /// Setting `avoid_page_cache` uses /// `fadvise(..,POSIX_FADV_DONTNEED)` to avoid keeping the data in /// the linux page cache. - pub fn save( + pub fn save( &self, path: &Path, options: CreateOptions, avoid_page_cache: bool, ) -> Result<(), Error> { - let (fd, tmp_path) = make_tmp_file(&path, options)?; let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) }; @@ -419,7 +434,6 @@ impl RRD { /// /// Note: This does not call [Self::save]. pub fn update(&mut self, time: f64, value: f64) { - let value = match self.source.compute_new_value(time, value) { Ok(value) => value, Err(err) => { @@ -451,11 +465,14 @@ impl RRD { start: Option, end: Option, ) -> Result<(u64, u64, Vec>), Error> { - let mut rra: Option<&RRA> = None; for item in self.rra_list.iter() { - if item.cf != cf { continue; } - if item.resolution > resolution { continue; } + if item.cf != cf { + continue; + } + if item.resolution > resolution { + continue; + } if let Some(current) = rra { if item.resolution > current.resolution { @@ -469,16 +486,14 @@ impl RRD { match rra { Some(rra) => { let end = end.unwrap_or_else(|| proxmox_time::epoch_f64() as u64); - let start = start.unwrap_or_else(|| end.saturating_sub(10*rra.resolution)); + let start = start.unwrap_or_else(|| end.saturating_sub(10 * rra.resolution)); Ok(rra.extract_data(start, end, self.source.last_update)) } None => bail!("unable to find RRA suitable ({:?}:{})", cf, resolution), } } - } - #[cfg(test)] mod tests { use super::*; @@ -489,10 +504,10 @@ mod tests { let mut rrd = RRD::new(DST::Gauge, vec![rra]); for i in 2..10 { - rrd.update((i as f64)*30.0, i as f64); + rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Maximum, 60, Some(0), Some(5*60))?; + let (start, reso, data) = rrd.extract_data(CF::Maximum, 60, Some(0), Some(5 * 60))?; assert_eq!(start, 0); assert_eq!(reso, 60); assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]); @@ -506,10 +521,10 @@ mod tests { let mut rrd = RRD::new(DST::Gauge, vec![rra]); for i in 2..10 { - rrd.update((i as f64)*30.0, i as f64); + rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Minimum, 60, Some(0), Some(5*60))?; + let (start, reso, data) = rrd.extract_data(CF::Minimum, 60, Some(0), Some(5 * 60))?; assert_eq!(start, 0); assert_eq!(reso, 60); assert_eq!(data, [None, Some(2.0), Some(4.0), Some(6.0), Some(8.0)]); @@ -523,12 +538,16 @@ mod tests { let mut rrd = RRD::new(DST::Gauge, vec![rra]); for i in 2..10 { - rrd.update((i as f64)*30.0, i as f64); + rrd.update((i as f64) * 30.0, i as f64); } - assert!(rrd.extract_data(CF::Average, 60, Some(0), Some(5*60)).is_err(), "CF::Average should not exist"); + assert!( + rrd.extract_data(CF::Average, 60, Some(0), Some(5 * 60)) + .is_err(), + "CF::Average should not exist" + ); - let (start, reso, data) = rrd.extract_data(CF::Last, 60, Some(0), Some(20*60))?; + let (start, reso, data) = rrd.extract_data(CF::Last, 60, Some(0), Some(20 * 60))?; assert_eq!(start, 0); assert_eq!(reso, 60); assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]); @@ -542,10 +561,10 @@ mod tests { let mut rrd = RRD::new(DST::Derive, vec![rra]); for i in 2..10 { - rrd.update((i as f64)*30.0, (i*60) as f64); + rrd.update((i as f64) * 30.0, (i * 60) as f64); } - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5*60))?; + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; assert_eq!(start, 60); assert_eq!(reso, 60); assert_eq!(data, [Some(1.0), Some(2.0), Some(2.0), Some(2.0), None]); @@ -559,40 +578,42 @@ mod tests { let mut rrd = RRD::new(DST::Gauge, vec![rra]); for i in 2..10 { - rrd.update((i as f64)*30.0, i as f64); + rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5*60))?; + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; assert_eq!(start, 60); assert_eq!(reso, 60); assert_eq!(data, [Some(2.5), Some(4.5), Some(6.5), Some(8.5), None]); for i in 10..14 { - rrd.update((i as f64)*30.0, i as f64); + rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5*60))?; + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; assert_eq!(start, 60); assert_eq!(reso, 60); assert_eq!(data, [None, Some(4.5), Some(6.5), Some(8.5), Some(10.5)]); - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(3*60), Some(8*60))?; - assert_eq!(start, 3*60); + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(3 * 60), Some(8 * 60))?; + assert_eq!(start, 3 * 60); assert_eq!(reso, 60); assert_eq!(data, [Some(6.5), Some(8.5), Some(10.5), Some(12.5), None]); // add much newer vaule (should delete all previous/outdated value) - let i = 100; rrd.update((i as f64)*30.0, i as f64); + let i = 100; + rrd.update((i as f64) * 30.0, i as f64); println!("TEST {:?}", serde_json::to_string_pretty(&rrd)); - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(100*30), Some(100*30 + 5*60))?; - assert_eq!(start, 100*30); + let (start, reso, data) = + rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(100 * 30 + 5 * 60))?; + assert_eq!(start, 100 * 30); assert_eq!(reso, 60); assert_eq!(data, [Some(100.0), None, None, None, None]); // extract with end time smaller than start time - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(100*30), Some(60))?; - assert_eq!(start, 100*30); + let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(60))?; + assert_eq!(start, 100 * 30); assert_eq!(reso, 60); assert_eq!(data, []); diff --git a/proxmox-rrd/src/rrd_v1.rs b/proxmox-rrd/src/rrd_v1.rs index 7e4b97c2..a1e7bf8e 100644 --- a/proxmox-rrd/src/rrd_v1.rs +++ b/proxmox-rrd/src/rrd_v1.rs @@ -8,11 +8,11 @@ pub const RRD_DATA_ENTRIES: usize = 70; /// Proxmox RRD file magic number // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; -pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; +pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; -use crate::rrd::{RRD, RRA, CF, DST, DataSource}; +use crate::rrd::{DataSource, CF, DST, RRA, RRD}; -bitflags!{ +bitflags! { /// Flags to specify the data soure type and consolidation function pub struct RRAFlags: u64 { // Data Source Types @@ -49,19 +49,16 @@ pub struct RRAv1 { } impl RRAv1 { - - fn extract_data( - &self, - ) -> (u64, u64, Vec>) { + fn extract_data(&self) -> (u64, u64, Vec>) { let reso = self.resolution; let mut list = Vec::new(); - let rra_end = reso*((self.last_update as u64)/reso); - let rra_start = rra_end - reso*(RRD_DATA_ENTRIES as u64); + let rra_end = reso * ((self.last_update as u64) / reso); + let rra_start = rra_end - reso * (RRD_DATA_ENTRIES as u64); let mut t = rra_start; - let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize; + let mut index = ((t / reso) % (RRD_DATA_ENTRIES as u64)) as usize; for _ in 0..RRD_DATA_ENTRIES { let value = self.data[index]; if value.is_nan() { @@ -70,7 +67,8 @@ impl RRAv1 { list.push(Some(value)); } - t += reso; index = (index + 1) % RRD_DATA_ENTRIES; + t += reso; + index = (index + 1) % RRD_DATA_ENTRIES; } (rra_start, reso, list) @@ -106,9 +104,7 @@ pub struct RRDv1 { } impl RRDv1 { - pub fn from_raw(mut raw: &[u8]) -> Result { - let expected_len = std::mem::size_of::(); if raw.len() != expected_len { @@ -118,7 +114,8 @@ impl RRDv1 { let mut rrd: RRDv1 = unsafe { std::mem::zeroed() }; unsafe { - let rrd_slice = std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len); + let rrd_slice = + std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len); raw.read_exact(rrd_slice)?; } @@ -131,7 +128,6 @@ impl RRDv1 { } pub fn to_rrd_v2(&self) -> Result { - let mut rra_list = Vec::new(); // old format v1: @@ -150,30 +146,36 @@ impl RRDv1 { // decade 1 week, 570 points // Linear extrapolation - fn extrapolate_data(start: u64, reso: u64, factor: u64, data: Vec>) -> (u64, u64, Vec>) { - + fn extrapolate_data( + start: u64, + reso: u64, + factor: u64, + data: Vec>, + ) -> (u64, u64, Vec>) { let mut new = Vec::new(); for i in 0..data.len() { let mut next = i + 1; - if next >= data.len() { next = 0 }; + if next >= data.len() { + next = 0 + }; let v = data[i]; let v1 = data[next]; match (v, v1) { (Some(v), Some(v1)) => { - let diff = (v1 - v)/(factor as f64); + let diff = (v1 - v) / (factor as f64); for j in 0..factor { - new.push(Some(v + diff*(j as f64))); + new.push(Some(v + diff * (j as f64))); } } (Some(v), None) => { new.push(Some(v)); - for _ in 0..factor-1 { + for _ in 0..factor - 1 { new.push(None); } } (None, Some(v1)) => { - for _ in 0..factor-1 { + for _ in 0..factor - 1 { new.push(None); } new.push(Some(v1)); @@ -186,7 +188,7 @@ impl RRDv1 { } } - (start, reso/factor, new) + (start, reso / factor, new) } // Try to convert to new, higher capacity format @@ -213,7 +215,7 @@ impl RRDv1 { // compute montly average (merge old self.month_avg, // self.week_avg and self.day_avg) - let mut month_avg = RRA::new(CF::Average, 30*60, 1440); + let mut month_avg = RRA::new(CF::Average, 30 * 60, 1440); let (start, reso, data) = self.month_avg.extract_data(); let (start, reso, data) = extrapolate_data(start, reso, 24, data); @@ -228,7 +230,7 @@ impl RRDv1 { // compute montly maximum (merge old self.month_max, // self.week_max and self.day_max) - let mut month_max = RRA::new(CF::Maximum, 30*60, 1440); + let mut month_max = RRA::new(CF::Maximum, 30 * 60, 1440); let (start, reso, data) = self.month_max.extract_data(); let (start, reso, data) = extrapolate_data(start, reso, 24, data); @@ -242,26 +244,26 @@ impl RRDv1 { month_max.insert_data(start, reso, data)?; // compute yearly average (merge old self.year_avg) - let mut year_avg = RRA::new(CF::Average, 6*3600, 1440); + let mut year_avg = RRA::new(CF::Average, 6 * 3600, 1440); let (start, reso, data) = self.year_avg.extract_data(); let (start, reso, data) = extrapolate_data(start, reso, 28, data); year_avg.insert_data(start, reso, data)?; // compute yearly maximum (merge old self.year_avg) - let mut year_max = RRA::new(CF::Maximum, 6*3600, 1440); + let mut year_max = RRA::new(CF::Maximum, 6 * 3600, 1440); let (start, reso, data) = self.year_max.extract_data(); let (start, reso, data) = extrapolate_data(start, reso, 28, data); year_max.insert_data(start, reso, data)?; // compute decade average (merge old self.year_avg) - let mut decade_avg = RRA::new(CF::Average, 7*86400, 570); + let mut decade_avg = RRA::new(CF::Average, 7 * 86400, 570); let (start, reso, data) = self.year_avg.extract_data(); decade_avg.insert_data(start, reso, data)?; // compute decade maximum (merge old self.year_max) - let mut decade_max = RRA::new(CF::Maximum, 7*86400, 570); + let mut decade_max = RRA::new(CF::Maximum, 7 * 86400, 570); let (start, reso, data) = self.year_max.extract_data(); decade_max.insert_data(start, reso, data)?; @@ -286,11 +288,8 @@ impl RRDv1 { let source = DataSource { dst, last_value: f64::NAN, - last_update: self.hour_avg.last_update, // IMPORTANT! + last_update: self.hour_avg.last_update, // IMPORTANT! }; - Ok(RRD { - source, - rra_list, - }) + Ok(RRD { source, rra_list }) } } diff --git a/proxmox-rrd/tests/file_format_test.rs b/proxmox-rrd/tests/file_format_test.rs index 81e49ca3..372a4077 100644 --- a/proxmox-rrd/tests/file_format_test.rs +++ b/proxmox-rrd/tests/file_format_test.rs @@ -7,7 +7,6 @@ use proxmox_rrd::rrd::RRD; use proxmox_sys::fs::CreateOptions; fn compare_file(fn1: &str, fn2: &str) -> Result<(), Error> { - let status = Command::new("/usr/bin/cmp") .arg(fn1) .arg(fn2) @@ -27,7 +26,6 @@ const RRD_V2_FN: &str = "./tests/testdata/cpu.rrd_v2"; // make sure we can load and convert RRD v1 #[test] fn upgrade_from_rrd_v1() -> Result<(), Error> { - let rrd = RRD::load(Path::new(RRD_V1_FN), true)?; const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.upgraded"; @@ -44,7 +42,6 @@ fn upgrade_from_rrd_v1() -> Result<(), Error> { // make sure we can load and save RRD v2 #[test] fn load_and_save_rrd_v2() -> Result<(), Error> { - let rrd = RRD::load(Path::new(RRD_V2_FN), true)?; const RRD_V2_NEW_FN: &str = "./tests/testdata/cpu.rrd_v2.saved"; From 56dd83740dfc47fd62d43dd1cee187575eee08c8 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 13 Apr 2022 08:17:08 +0200 Subject: [PATCH 092/111] bump proxmox-router dependency to 1.2 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 9f6510e3..7b2cfaef 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" description = "Simple RRD database implementation." [dev-dependencies] -proxmox-router = "1.1" +proxmox-router = "1.2" [dependencies] anyhow = "1.0" From 71a7566cd32859fa567f640816f6ef2b3e29fb08 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 13 Apr 2022 08:20:27 +0200 Subject: [PATCH 093/111] bump proxmox-schema dependency to 1.3.1 for streaming attribute Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 7b2cfaef..d1332699 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -21,5 +21,5 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" -proxmox-schema = { version = "1.3", features = [ "api-macro" ] } +proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } proxmox-sys = "0.2" \ No newline at end of file From 6614f8840f9197774ef81f78563154d7a004fe03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Mon, 16 May 2022 15:02:07 +0200 Subject: [PATCH 094/111] build: bump required log version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit else logging using "{var}" in format strings doesn't work properly. Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index d1332699..93af86b5 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -13,7 +13,7 @@ anyhow = "1.0" bitflags = "1.2.1" crossbeam-channel = "0.5" libc = "0.2" -log = "0.4" +log = "0.4.17" nix = "0.19.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" From 8e042eb13003c634a05413088b16937d580b8ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 2 Jun 2022 13:10:33 +0200 Subject: [PATCH 095/111] update to nix 0.24 / rustyline 9 / proxmox-sys 0.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 93af86b5..b9ef4ee4 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -14,7 +14,7 @@ bitflags = "1.2.1" crossbeam-channel = "0.5" libc = "0.2" log = "0.4.17" -nix = "0.19.1" +nix = "0.24" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_cbor = "0.11.1" @@ -22,4 +22,4 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } -proxmox-sys = "0.2" \ No newline at end of file +proxmox-sys = "0.3" \ No newline at end of file From d84573627068e63e9f2c85e50ac4f8a3f46f298f Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 7 Jun 2022 09:22:45 +0200 Subject: [PATCH 096/111] tree wide: typo fixes through codespell Signed-off-by: Thomas Lamprecht --- proxmox-rrd/src/rrd.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 41af6242..4ae3ee93 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -8,7 +8,7 @@ //! ## Features //! //! * Well defined data format [CBOR](https://datatracker.ietf.org/doc/html/rfc8949) -//! * Plattform independent (big endian f64, hopefully a standard format?) +//! * Platform independent (big endian f64, hopefully a standard format?) //! * Arbitrary number of RRAs (dynamically changeable) use std::io::{Read, Write}; @@ -456,7 +456,7 @@ impl RRD { /// This selects the RRA with specified [CF] and (minimum) /// resolution, and extract data from `start` to `end`. /// - /// `start`: Start time. If not sepecified, we simply extract 10 data points. + /// `start`: Start time. If not specified, we simply extract 10 data points. /// `end`: End time. Default is to use the current time. pub fn extract_data( &self, @@ -600,7 +600,7 @@ mod tests { assert_eq!(reso, 60); assert_eq!(data, [Some(6.5), Some(8.5), Some(10.5), Some(12.5), None]); - // add much newer vaule (should delete all previous/outdated value) + // add much newer value (should delete all previous/outdated value) let i = 100; rrd.update((i as f64) * 30.0, i as f64); println!("TEST {:?}", serde_json::to_string_pretty(&rrd)); From f81e9b259a8fc64b299e5fab800456d54e7fd9e0 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 21 Jun 2022 10:43:11 +0200 Subject: [PATCH 097/111] bump proxmox-router dep to 1.2.4 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index b9ef4ee4..e838cd25 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" description = "Simple RRD database implementation." [dev-dependencies] -proxmox-router = "1.2" +proxmox-router = "1.2.4" [dependencies] anyhow = "1.0" From f3fd79be1352cce2c9370f82aafe2305ba104269 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 29 Jun 2022 09:44:42 +0200 Subject: [PATCH 098/111] bump proxmox-sys dep to 0.3.1 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index e838cd25..0a9c948c 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -22,4 +22,4 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } -proxmox-sys = "0.3" \ No newline at end of file +proxmox-sys = "0.3.1" \ No newline at end of file From 56b5c28930799681205d15f60aa39f1fe288bbdb Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 27 Jul 2022 13:43:04 +0200 Subject: [PATCH 099/111] rrd: Entry type and clippy fixes Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/examples/prrd.rs | 4 +- proxmox-rrd/src/cache.rs | 3 +- proxmox-rrd/src/cache/rrd_map.rs | 3 +- proxmox-rrd/src/lib.rs | 2 + proxmox-rrd/src/rrd.rs | 124 +++++++++++++++++++++++-------- 5 files changed, 104 insertions(+), 32 deletions(-) diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index 081cef98..5825f296 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -301,7 +301,9 @@ pub fn resize_rrd(path: String, rra_index: usize, slots: i64) -> Result<(), Erro let rra_end = rra.slot_end_time(rrd.source.last_update as u64); let rra_start = rra_end - rra.resolution * (rra.data.len() as u64); - let (start, reso, data) = rra.extract_data(rra_start, rra_end, rrd.source.last_update); + let (start, reso, data) = rra + .extract_data(rra_start, rra_end, rrd.source.last_update) + .into(); let mut new_rra = RRA::new(rra.cf, rra.resolution, new_slots as usize); new_rra.last_count = rra.last_count; diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 90e4e470..1321e58d 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -13,6 +13,7 @@ use crossbeam_channel::{bounded, TryRecvError}; use proxmox_sys::fs::{create_path, CreateOptions}; use crate::rrd::{CF, DST, RRA, RRD}; +use crate::Entry; mod journal; use journal::*; @@ -217,7 +218,7 @@ impl RRDCache { resolution: u64, start: Option, end: Option, - ) -> Result>)>, Error> { + ) -> Result, Error> { self.rrd_map .read() .unwrap() diff --git a/proxmox-rrd/src/cache/rrd_map.rs b/proxmox-rrd/src/cache/rrd_map.rs index 56dde2e6..f907d350 100644 --- a/proxmox-rrd/src/cache/rrd_map.rs +++ b/proxmox-rrd/src/cache/rrd_map.rs @@ -9,6 +9,7 @@ use proxmox_sys::fs::create_path; use crate::rrd::{CF, DST, RRD}; use super::CacheConfig; +use crate::Entry; pub struct RRDMap { config: Arc, @@ -87,7 +88,7 @@ impl RRDMap { resolution: u64, start: Option, end: Option, - ) -> Result>)>, Error> { + ) -> Result, Error> { match self.map.get(&format!("{}/{}", base, name)) { Some(rrd) => Ok(Some(rrd.extract_data(cf, resolution, start, end)?)), None => Ok(None), diff --git a/proxmox-rrd/src/lib.rs b/proxmox-rrd/src/lib.rs index 2038170d..80b39438 100644 --- a/proxmox-rrd/src/lib.rs +++ b/proxmox-rrd/src/lib.rs @@ -9,6 +9,8 @@ mod rrd_v1; pub mod rrd; +#[doc(inline)] +pub use rrd::Entry; mod cache; pub use cache::*; diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 4ae3ee93..6affa9a5 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -28,7 +28,7 @@ use crate::rrd_v1; pub const PROXMOX_RRD_MAGIC_2_0: [u8; 8] = [224, 200, 228, 27, 239, 112, 122, 159]; #[api()] -#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq)] +#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// RRD data source type pub enum DST { @@ -42,7 +42,7 @@ pub enum DST { } #[api()] -#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq)] +#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Consolidation function pub enum CF { @@ -68,6 +68,42 @@ pub struct DataSource { pub last_value: f64, } +/// An RRD entry. +/// +/// Serializes as a tuple. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde( + from = "(u64, u64, Vec>)", + into = "(u64, u64, Vec>)" +)] +pub struct Entry { + pub start: u64, + pub resolution: u64, + pub data: Vec>, +} + +impl Entry { + pub const fn new(start: u64, resolution: u64, data: Vec>) -> Self { + Self { + start, + resolution, + data, + } + } +} + +impl From for (u64, u64, Vec>) { + fn from(entry: Entry) -> (u64, u64, Vec>) { + (entry.start, entry.resolution, entry.data) + } +} + +impl From<(u64, u64, Vec>)> for Entry { + fn from(data: (u64, u64, Vec>)) -> Self { + Self::new(data.0, data.1, data.2) + } +} + impl DataSource { /// Create a new Instance pub fn new(dst: DST) -> Self { @@ -265,12 +301,7 @@ impl RRA { /// Extract data from `start` to `end`. The RRA itself does not /// store the `last_update` time, so you need to pass this a /// parameter (see [DataSource]). - pub fn extract_data( - &self, - start: u64, - end: u64, - last_update: f64, - ) -> (u64, u64, Vec>) { + pub fn extract_data(&self, start: u64, end: u64, last_update: f64) -> Entry { let last_update = last_update as u64; let reso = self.resolution; let num_entries = self.data.len() as u64; @@ -303,7 +334,7 @@ impl RRA { } } - (start, reso, list) + Entry::new(start, reso, list) } } @@ -464,7 +495,7 @@ impl RRD { resolution: u64, start: Option, end: Option, - ) -> Result<(u64, u64, Vec>), Error> { + ) -> Result { let mut rra: Option<&RRA> = None; for item in self.rra_list.iter() { if item.cf != cf { @@ -507,9 +538,13 @@ mod tests { rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Maximum, 60, Some(0), Some(5 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Maximum, 60, Some(0), Some(5 * 60))?; assert_eq!(start, 0); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]); Ok(()) @@ -524,9 +559,13 @@ mod tests { rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Minimum, 60, Some(0), Some(5 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Minimum, 60, Some(0), Some(5 * 60))?; assert_eq!(start, 0); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [None, Some(2.0), Some(4.0), Some(6.0), Some(8.0)]); Ok(()) @@ -547,9 +586,13 @@ mod tests { "CF::Average should not exist" ); - let (start, reso, data) = rrd.extract_data(CF::Last, 60, Some(0), Some(20 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Last, 60, Some(0), Some(20 * 60))?; assert_eq!(start, 0); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [None, Some(3.0), Some(5.0), Some(7.0), Some(9.0)]); Ok(()) @@ -564,9 +607,13 @@ mod tests { rrd.update((i as f64) * 30.0, (i * 60) as f64); } - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; assert_eq!(start, 60); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [Some(1.0), Some(2.0), Some(2.0), Some(2.0), None]); Ok(()) @@ -581,23 +628,35 @@ mod tests { rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; assert_eq!(start, 60); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [Some(2.5), Some(4.5), Some(6.5), Some(8.5), None]); for i in 10..14 { rrd.update((i as f64) * 30.0, i as f64); } - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Average, 60, Some(60), Some(5 * 60))?; assert_eq!(start, 60); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [None, Some(4.5), Some(6.5), Some(8.5), Some(10.5)]); - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(3 * 60), Some(8 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Average, 60, Some(3 * 60), Some(8 * 60))?; assert_eq!(start, 3 * 60); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [Some(6.5), Some(8.5), Some(10.5), Some(12.5), None]); // add much newer value (should delete all previous/outdated value) @@ -605,16 +664,23 @@ mod tests { rrd.update((i as f64) * 30.0, i as f64); println!("TEST {:?}", serde_json::to_string_pretty(&rrd)); - let (start, reso, data) = - rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(100 * 30 + 5 * 60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(100 * 30 + 5 * 60))?; assert_eq!(start, 100 * 30); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, [Some(100.0), None, None, None, None]); // extract with end time smaller than start time - let (start, reso, data) = rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(60))?; + let Entry { + start, + resolution, + data, + } = rrd.extract_data(CF::Average, 60, Some(100 * 30), Some(60))?; assert_eq!(start, 100 * 30); - assert_eq!(reso, 60); + assert_eq!(resolution, 60); assert_eq!(data, []); Ok(()) From 5d61126f5896d7613e4ad180207742bf3f33ec74 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 28 Jul 2022 13:40:07 +0200 Subject: [PATCH 100/111] bump proxmox-sys dep to 0.4 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 0a9c948c..496e1028 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -22,4 +22,4 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } -proxmox-sys = "0.3.1" \ No newline at end of file +proxmox-sys = "0.4" \ No newline at end of file From d97a86c15b05652ba96878d5a6fb881fc22c68c6 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Tue, 11 Oct 2022 15:45:52 +0200 Subject: [PATCH 101/111] cargo: rrd: set license in subcrate too in preparation of moving this out Signed-off-by: Thomas Lamprecht --- proxmox-rrd/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 496e1028..b92e262b 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -3,6 +3,7 @@ name = "proxmox-rrd" version = "0.1.0" authors = ["Proxmox Support Team "] edition = "2018" +license = "AGPL-3" description = "Simple RRD database implementation." [dev-dependencies] @@ -22,4 +23,4 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } -proxmox-sys = "0.4" \ No newline at end of file +proxmox-sys = "0.4" From 3bbdcf23e079d9dca860ca8a91d54813bd0cc66a Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 19 Oct 2022 14:22:38 +0200 Subject: [PATCH 102/111] bump sys dep to 0.4.1 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index b92e262b..c9caf7d5 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -23,4 +23,4 @@ serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } proxmox-time = "1" proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } -proxmox-sys = "0.4" +proxmox-sys = "0.4.1" From 3193237afd6d3ea37333a4b311cabbf296b030fe Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 24 Nov 2022 13:52:43 +0100 Subject: [PATCH 103/111] rrd: add Entry::get() to access the data Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/src/rrd.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 6affa9a5..7d05072f 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -90,6 +90,12 @@ impl Entry { data, } } + + /// Get a data point at a specific index which also does bound checking and returns `None` for + /// out of bounds indices. + pub fn get(&self, idx: usize) -> Option { + self.data.get(idx).copied().flatten() + } } impl From for (u64, u64, Vec>) { From 1aa6f0ea2221d48597adfa4d27c34b19b3aeabd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Mon, 5 Dec 2022 11:27:40 +0100 Subject: [PATCH 104/111] clippy 1.65 fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and rustfmt Signed-off-by: Fabian Grünbichler --- proxmox-rrd/src/cache.rs | 2 +- proxmox-rrd/src/rrd.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 1321e58d..8b004fd7 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -430,7 +430,7 @@ fn commit_journal_impl( for rel_path in files.iter() { let mut path = config.basedir.clone(); - path.push(&rel_path); + path.push(rel_path); fsync_file_or_dir(&path) .map_err(|err| format_err!("fsync rrd file {} failed - {}", rel_path, err))?; } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 7d05072f..cd1016e0 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -425,12 +425,12 @@ impl RRD { options: CreateOptions, avoid_page_cache: bool, ) -> Result<(), Error> { - let (fd, tmp_path) = make_tmp_file(&path, options)?; + let (fd, tmp_path) = make_tmp_file(path, options)?; let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) }; let mut try_block = || -> Result<(), Error> { let mut data: Vec = Vec::new(); - data.extend(&PROXMOX_RRD_MAGIC_2_0); + data.extend(PROXMOX_RRD_MAGIC_2_0); serde_cbor::to_writer(&mut data, self)?; file.write_all(&data)?; @@ -454,7 +454,7 @@ impl RRD { } } - if let Err(err) = std::fs::rename(&tmp_path, &path) { + if let Err(err) = std::fs::rename(&tmp_path, path) { let _ = nix::unistd::unlink(&tmp_path); bail!("Atomic rename failed - {}", err); } From 5acca01947e9139baea51277db6184f3ec1830c2 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 6 Dec 2022 11:19:41 +0100 Subject: [PATCH 105/111] tree-wide: bump edition to 2021 Signed-off-by: Wolfgang Bumiller --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index c9caf7d5..4e9bb317 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -2,7 +2,7 @@ name = "proxmox-rrd" version = "0.1.0" authors = ["Proxmox Support Team "] -edition = "2018" +edition = "2021" license = "AGPL-3" description = "Simple RRD database implementation." From 219af02796eca0e17ac08589d9beb4fd7d0b17f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Wed, 7 Dec 2022 11:33:47 +0100 Subject: [PATCH 106/111] workspace: inherit metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pbs-buildcfg is the only one that needs to inherit the version as well, since it stores it in the compiled crate. Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 4e9bb317..c79fc4b3 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "proxmox-rrd" version = "0.1.0" -authors = ["Proxmox Support Team "] -edition = "2021" -license = "AGPL-3" +authors.workspace = true +edition.workspace = true +license.workspace = true description = "Simple RRD database implementation." [dev-dependencies] From d75e305162a20800e9c36ee63174649918baf612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Fri, 9 Dec 2022 11:37:02 +0100 Subject: [PATCH 107/111] switch proxmox dependencies to workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit besides harmonizing versions, the only global change is that the tokio-io feature of pxar is now implied since its default anyway, instead of being spelled out. Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index c79fc4b3..0955b86f 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -7,7 +7,7 @@ license.workspace = true description = "Simple RRD database implementation." [dev-dependencies] -proxmox-router = "1.2.4" +proxmox-router = { workspace = true, features = ["cli", "server"] } [dependencies] anyhow = "1.0" @@ -21,6 +21,6 @@ serde_json = "1.0" serde_cbor = "0.11.1" #proxmox = { version = "0.15.3" } -proxmox-time = "1" -proxmox-schema = { version = "1.3.1", features = [ "api-macro" ] } -proxmox-sys = "0.4.1" +proxmox-time.workspace = true +proxmox-schema = { workspace = true, features = [ "api-macro" ] } +proxmox-sys.workspace = true From b659deb529dcc5d26bb47c568a357a469ecf1389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Fri, 9 Dec 2022 13:22:58 +0100 Subject: [PATCH 108/111] switch regular dependencies to workspace ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit where applicable. notable changes: - serde now uses 'derive' feature across the board - serde removed from pbs-tools (not used) - openssl bumped to 0.40 (and patched comment removed) - removed invalid zstd comment Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 0955b86f..b49809fd 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -10,17 +10,16 @@ description = "Simple RRD database implementation." proxmox-router = { workspace = true, features = ["cli", "server"] } [dependencies] -anyhow = "1.0" -bitflags = "1.2.1" -crossbeam-channel = "0.5" -libc = "0.2" -log = "0.4.17" -nix = "0.24" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +anyhow.workspace = true +bitflags.workspace = true +crossbeam-channel.workspace = true +libc.workspace = true +log.workspace = true +nix.workspace = true +serde.workspace = true +serde_json.workspace = true serde_cbor = "0.11.1" -#proxmox = { version = "0.15.3" } proxmox-time.workspace = true proxmox-schema = { workspace = true, features = [ "api-macro" ] } proxmox-sys.workspace = true From 32504b78db4f90cf8db9aa04d3fd80f9c8403712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Fri, 9 Dec 2022 13:52:03 +0100 Subject: [PATCH 109/111] switch remaining member dependencies to workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit these are only used by a single member at the moment, but we can move them to the workspace to have a single location for version + base feature set specification. Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index b49809fd..d8e901ec 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -18,7 +18,7 @@ log.workspace = true nix.workspace = true serde.workspace = true serde_json.workspace = true -serde_cbor = "0.11.1" +serde_cbor.workspace = true proxmox-time.workspace = true proxmox-schema = { workspace = true, features = [ "api-macro" ] } From 10f56e93585aa9a8b6190ba5b39058868889f879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Fri, 9 Dec 2022 13:58:19 +0100 Subject: [PATCH 110/111] sort dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- proxmox-rrd/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index d8e901ec..c0b7d708 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -17,9 +17,9 @@ libc.workspace = true log.workspace = true nix.workspace = true serde.workspace = true -serde_json.workspace = true serde_cbor.workspace = true +serde_json.workspace = true -proxmox-time.workspace = true proxmox-schema = { workspace = true, features = [ "api-macro" ] } proxmox-sys.workspace = true +proxmox-time.workspace = true From 109902fbf01c11f9c1262f7763de53ce08d4e809 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Wed, 29 Nov 2023 18:32:06 +0100 Subject: [PATCH 111/111] tree-wide: fix various typos found with codespell Signed-off-by: Thomas Lamprecht --- proxmox-rrd/examples/prrd.rs | 8 ++++---- proxmox-rrd/src/cache.rs | 4 ++-- proxmox-rrd/src/rrd.rs | 2 +- proxmox-rrd/src/rrd_v1.rs | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/proxmox-rrd/examples/prrd.rs b/proxmox-rrd/examples/prrd.rs index 5825f296..c7d29376 100644 --- a/proxmox-rrd/examples/prrd.rs +++ b/proxmox-rrd/examples/prrd.rs @@ -124,10 +124,10 @@ pub fn update_rrd(path: String, time: Option, value: f64) -> Result<(), Err type: CF, }, resolution: { - description: "Time resulution", + description: "Time resolution", }, start: { - description: "Start time. If not sepecified, we simply extract 10 data points.", + description: "Start time. If not specified, we simply extract 10 data points.", optional: true, }, end: { @@ -292,11 +292,11 @@ pub fn resize_rrd(path: String, rra_index: usize, slots: i64) -> Result<(), Erro let new_slots = (rra.data.len() as i64) + slots; if new_slots < 1 { - bail!("numer of new slots is too small ('{}' < 1)", new_slots); + bail!("number of new slots is too small ('{}' < 1)", new_slots); } if new_slots > 1024 * 1024 { - bail!("numer of new slots is too big ('{}' > 1M)", new_slots); + bail!("number of new slots is too big ('{}' > 1M)", new_slots); } let rra_end = rra.slot_end_time(rrd.source.last_update as u64); diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 8b004fd7..254010f3 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -102,7 +102,7 @@ impl RRDCache { /// * cf=average,r=7*86400,n=570 => 10years /// * cf=maximum,r=7*86400,n=570 => 10year /// - /// The resultion data file size is about 80KB. + /// The resulting data file size is about 80KB. pub fn create_proxmox_backup_default_rrd(dst: DST) -> RRD { let rra_list = vec![ // 1 min * 1440 => 1 day @@ -207,7 +207,7 @@ impl RRDCache { /// Extract data from cached RRD /// - /// `start`: Start time. If not sepecified, we simply extract 10 data points. + /// `start`: Start time. If not specified, we simply extract 10 data points. /// /// `end`: End time. Default is to use the current time. pub fn extract_cached_data( diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index cd1016e0..0b8ac460 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -147,7 +147,7 @@ impl DataSource { // we update last_value anyways, so that we can compute the diff // next time. self.last_value = value; - bail!("conter overflow/reset detected"); + bail!("counter overflow/reset detected"); } else { value - self.last_value }; diff --git a/proxmox-rrd/src/rrd_v1.rs b/proxmox-rrd/src/rrd_v1.rs index a1e7bf8e..2f4a25f8 100644 --- a/proxmox-rrd/src/rrd_v1.rs +++ b/proxmox-rrd/src/rrd_v1.rs @@ -13,7 +13,7 @@ pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; use crate::rrd::{DataSource, CF, DST, RRA, RRD}; bitflags! { - /// Flags to specify the data soure type and consolidation function + /// Flags to specify the data source type and consolidation function pub struct RRAFlags: u64 { // Data Source Types const DST_GAUGE = 1; @@ -34,9 +34,9 @@ bitflags! { /// RRD files. #[repr(C)] pub struct RRAv1 { - /// Defined the data soure type and consolidation function + /// Defined the data source type and consolidation function pub flags: RRAFlags, - /// Resulution (seconds) + /// Resolution (seconds) pub resolution: u64, /// Last update time (epoch) pub last_update: f64, @@ -213,7 +213,7 @@ impl RRDv1 { let (start, reso, data) = self.hour_max.extract_data(); day_max.insert_data(start, reso, data)?; - // compute montly average (merge old self.month_avg, + // compute monthly average (merge old self.month_avg, // self.week_avg and self.day_avg) let mut month_avg = RRA::new(CF::Average, 30 * 60, 1440); @@ -228,7 +228,7 @@ impl RRDv1 { let (start, reso, data) = self.day_avg.extract_data(); month_avg.insert_data(start, reso, data)?; - // compute montly maximum (merge old self.month_max, + // compute monthly maximum (merge old self.month_max, // self.week_max and self.day_max) let mut month_max = RRA::new(CF::Maximum, 30 * 60, 1440);