move RRD code into proxmox-rrd crate

proxmox-rrd/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "proxmox-rrd"
+version = "0.1.0"
+authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
+edition = "2018"
+description = "Simple RRD database implementation."
+
+[dependencies]
+anyhow = "1.0"
+bitflags = "1.2.1"
+serde = { version = "1.0", features = [] }
+
+proxmox = { version = "0.13.5", features = ["api-macro"] }

proxmox-rrd/src/cache.rs

@@ -0,0 +1,111 @@
+use std::path::{Path, PathBuf};
+use std::collections::HashMap;
+use std::sync::{RwLock};
+
+use anyhow::{format_err, Error};
+
+use proxmox::tools::fs::{create_path, CreateOptions};
+
+use crate::{RRDMode, RRDTimeFrameResolution};
+
+use super::*;
+
+/// RRD cache - keep RRD data in RAM, but write updates to disk
+///
+/// This cache is designed to run as single instance (no concurrent
+/// access from other processes).
+pub struct RRDCache {
+    basedir: PathBuf,
+    file_options: CreateOptions,
+    dir_options: CreateOptions,
+    cache: RwLock<HashMap<String, RRD>>,
+}
+
+impl RRDCache {
+
+    /// Creates a new instance
+    pub fn new<P: AsRef<Path>>(
+        basedir: P,
+        file_options: Option<CreateOptions>,
+        dir_options: Option<CreateOptions>,
+    ) -> Self {
+        let basedir = basedir.as_ref().to_owned();
+        Self {
+            basedir,
+            file_options: file_options.unwrap_or_else(|| CreateOptions::new()),
+            dir_options: dir_options.unwrap_or_else(|| CreateOptions::new()),
+            cache: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl RRDCache {
+
+    /// Create rrdd stat dir with correct permission
+    pub fn create_rrdb_dir(&self) -> Result<(), Error> {
+
+        create_path(&self.basedir, Some(self.dir_options.clone()), Some(self.file_options.clone()))
+            .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?;
+
+        Ok(())
+    }
+
+    /// Update data in RAM and write file back to disk (if `save` is set)
+    pub fn update_value(
+        &self,
+        rel_path: &str,
+        value: f64,
+        dst: DST,
+        save: bool,
+    ) -> Result<(), Error> {
+
+        let mut path = self.basedir.clone();
+        path.push(rel_path);
+
+        std::fs::create_dir_all(path.parent().unwrap())?; // fixme??
+
+        let mut map = self.cache.write().unwrap();
+        let now = proxmox::tools::time::epoch_f64();
+
+        if let Some(rrd) = map.get_mut(rel_path) {
+            rrd.update(now, value);
+            if save { rrd.save(&path, self.file_options.clone())?; }
+        } else {
+            let mut rrd = match RRD::load(&path) {
+                Ok(rrd) => rrd,
+                Err(err) => {
+                    if err.kind() != std::io::ErrorKind::NotFound {
+                        eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err);
+                    }
+                    RRD::new(dst)
+                },
+            };
+            rrd.update(now, value);
+            if save {
+                rrd.save(&path, self.file_options.clone())?;
+            }
+            map.insert(rel_path.into(), rrd);
+        }
+
+        Ok(())
+    }
+
+    /// Extract data from cached RRD
+    pub fn extract_cached_data(
+        &self,
+        base: &str,
+        name: &str,
+        now: f64,
+        timeframe: RRDTimeFrameResolution,
+        mode: RRDMode,
+    ) -> Option<(u64, u64, Vec<Option<f64>>)> {
+
+        let map = self.cache.read().unwrap();
+
+        match map.get(&format!("{}/{}", base, name)) {
+            Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)),
+            None => None,
+        }
+    }
+
+}

proxmox-rrd/src/lib.rs

@@ -0,0 +1,37 @@
+mod rrd;
+pub use rrd::*;
+
+mod cache;
+pub use cache::*;
+
+use serde::{Deserialize, Serialize};
+use proxmox::api::api;
+
+#[api()]
+#[derive(Copy, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+/// RRD consolidation mode
+pub enum RRDMode {
+    /// Maximum
+    Max,
+    /// Average
+    Average,
+}
+
+#[api()]
+#[repr(u64)]
+#[derive(Copy, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// RRD time frame resolution
+pub enum RRDTimeFrameResolution {
+    ///  1 min => last 70 minutes
+    Hour = 60,
+    /// 30 min => last 35 hours
+    Day = 60*30,
+    /// 3 hours => about 8 days
+    Week = 60*180,
+    /// 12 hours => last 35 days
+    Month = 60*720,
+    /// 1 week => last 490 days
+    Year = 60*10080,
+}

proxmox-rrd/src/rrd.rs

@@ -3,17 +3,21 @@ use std::path::Path;
 
 use anyhow::Error;
 
-use pbs_api_types::{RRDMode, RRDTimeFrameResolution};
+use proxmox::tools::{fs::replace_file, fs::CreateOptions};
 
+use crate::{RRDMode, RRDTimeFrameResolution};
+
+/// The number of data entries per RRA
 pub const RRD_DATA_ENTRIES: usize = 70;
 
+/// Proxmox RRD file magic number
 // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8];
 pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] =  [206, 46, 26, 212, 172, 158, 5, 186];
 
 use bitflags::bitflags;
 
 bitflags!{
-    pub struct RRAFlags: u64 {
+    struct RRAFlags: u64 {
         // Data Source Types
         const DST_GAUGE  = 1;
         const DST_DERIVE = 2;
@@ -27,6 +31,7 @@ bitflags!{
     }
 }
 
+/// RRD data source tyoe
 pub enum DST {
     Gauge,
     Derive,
@@ -227,6 +232,7 @@ impl RRD {
         timeframe: RRDTimeFrameResolution,
         mode: RRDMode,
     ) -> (u64, u64, Vec<Option<f64>>) {
+
         let epoch = time as u64;
         let reso = timeframe as u64;
 
@@ -296,25 +302,11 @@ impl RRD {
         Self::from_raw(&raw)
     }
 
-    pub fn save(&self, filename: &Path) -> Result<(), Error> {
-        use proxmox::tools::{fs::replace_file, fs::CreateOptions};
-
+    pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> {
         let rrd_slice = unsafe {
             std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::<RRD>())
         };
-
-        let backup_user = pbs_config::backup_user()?;
-        let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644);
-        // set the correct owner/group/permissions while saving file
-        // owner(rw) = backup, group(r)= backup
-        let options = CreateOptions::new()
-            .perm(mode)
-            .owner(backup_user.uid)
-            .group(backup_user.gid);
-
-        replace_file(filename, rrd_slice, options)?;
-
-        Ok(())
+        replace_file(filename, rrd_slice, options)
     }
 
 

src/rrd/cache.rs

@@ -1,81 +0,0 @@
-use std::path::PathBuf;
-use std::collections::HashMap;
-use std::sync::{RwLock};
-
-use anyhow::{format_err, Error};
-use lazy_static::lazy_static;
-
-use proxmox::tools::fs::{create_path, CreateOptions};
-
-use pbs_api_types::{RRDMode, RRDTimeFrameResolution};
-
-use super::*;
-
-const PBS_RRD_BASEDIR: &str = "/var/lib/proxmox-backup/rrdb";
-
-lazy_static!{
-    static ref RRD_CACHE: RwLock<HashMap<String, RRD>> = {
-        RwLock::new(HashMap::new())
-    };
-}
-
-/// Create rrdd stat dir with correct permission
-pub fn create_rrdb_dir() -> Result<(), Error> {
-
-    let backup_user = pbs_config::backup_user()?;
-    let opts = CreateOptions::new()
-        .owner(backup_user.uid)
-        .group(backup_user.gid);
-
-    create_path(PBS_RRD_BASEDIR, None, Some(opts))
-        .map_err(|err: Error| format_err!("unable to create rrdb stat dir - {}", err))?;
-
-    Ok(())
-}
-
-pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result<(), Error> {
-
-    let mut path = PathBuf::from(PBS_RRD_BASEDIR);
-    path.push(rel_path);
-
-    std::fs::create_dir_all(path.parent().unwrap())?;
-
-    let mut map = RRD_CACHE.write().unwrap();
-    let now = proxmox::tools::time::epoch_f64();
-
-    if let Some(rrd) = map.get_mut(rel_path) {
-        rrd.update(now, value);
-        if save { rrd.save(&path)?; }
-    } else {
-        let mut rrd = match RRD::load(&path) {
-            Ok(rrd) => rrd,
-            Err(err) => {
-                if err.kind() != std::io::ErrorKind::NotFound {
-                    eprintln!("overwriting RRD file {:?}, because of load error: {}", path, err);
-                }
-                RRD::new(dst)
-            },
-        };
-        rrd.update(now, value);
-        if save { rrd.save(&path)?; }
-        map.insert(rel_path.into(), rrd);
-    }
-
-    Ok(())
-}
-
-pub fn extract_cached_data(
-    base: &str,
-    name: &str,
-    now: f64,
-    timeframe: RRDTimeFrameResolution,
-    mode: RRDMode,
-) -> Option<(u64, u64, Vec<Option<f64>>)> {
-
-    let map = RRD_CACHE.read().unwrap();
-
-    match map.get(&format!("{}/{}", base, name)) {
-        Some(rrd) => Some(rrd.extract_data(now, timeframe, mode)),
-        None => None,
-    }
-}

src/rrd/mod.rs

@@ -1,5 +0,0 @@
-#[allow(clippy::module_inception)]
-mod rrd;
-pub use rrd::*;
-mod cache;
-pub use cache::*;