apt: add cache feature

Save/read package state from a file, and add the api functions to manipulate
that state.

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
This commit is contained in:
Dietmar Maurer 2024-07-02 09:37:30 +02:00 committed by Wolfgang Bumiller
parent f536a91b2f
commit f451a643ae
7 changed files with 706 additions and 3 deletions

View File

@ -22,3 +22,20 @@ rfc822-like = "0.2.1"
proxmox-apt-api-types.workspace = true
proxmox-config-digest = { workspace = true, features = ["openssl"] }
proxmox-sys.workspace = true
apt-pkg-native = { version = "0.3.2", optional = true }
regex = { workspace = true, optional = true }
nix = { workspace = true, optional = true }
log = { workspace = true, optional = true }
proxmox-schema = { workspace = true, optional = true }
[features]
default = []
cache = [
"dep:apt-pkg-native",
"dep:regex",
"dep:nix",
"dep:log",
"dep:proxmox-schema",
]

View File

@ -13,6 +13,7 @@ Build-Depends: debhelper (>= 12),
librust-proxmox-apt-api-types-1+default-dev <!nocheck>,
librust-proxmox-config-digest-0.1+default-dev <!nocheck>,
librust-proxmox-config-digest-0.1+openssl-dev <!nocheck>,
librust-proxmox-sys-0.5+default-dev (>= 0.5.7-~~) <!nocheck>,
librust-rfc822-like-0.2+default-dev (>= 0.2.1-~~) <!nocheck>,
librust-serde-1+default-dev <!nocheck>,
librust-serde-1+derive-dev <!nocheck>,
@ -37,10 +38,13 @@ Depends:
librust-proxmox-apt-api-types-1+default-dev,
librust-proxmox-config-digest-0.1+default-dev,
librust-proxmox-config-digest-0.1+openssl-dev,
librust-proxmox-sys-0.5+default-dev (>= 0.5.7-~~),
librust-rfc822-like-0.2+default-dev (>= 0.2.1-~~),
librust-serde-1+default-dev,
librust-serde-1+derive-dev,
librust-serde-json-1+default-dev
Suggests:
librust-proxmox-apt+cache-dev (= ${binary:Version})
Provides:
librust-proxmox-apt+default-dev (= ${binary:Version}),
librust-proxmox-apt-0-dev (= ${binary:Version}),
@ -51,3 +55,22 @@ Provides:
librust-proxmox-apt-0.10.10+default-dev (= ${binary:Version})
Description: Proxmox library for APT - Rust source code
Source code for Debianized Rust crate "proxmox-apt"
Package: librust-proxmox-apt+cache-dev
Architecture: any
Multi-Arch: same
Depends:
${misc:Depends},
librust-proxmox-apt-dev (= ${binary:Version}),
librust-apt-pkg-native-0.3+default-dev (>= 0.3.2-~~),
librust-log-0.4+default-dev (>= 0.4.17-~~),
librust-nix-0.26+default-dev (>= 0.26.1-~~),
librust-proxmox-schema-3+default-dev (>= 3.1.1-~~),
librust-regex-1+default-dev (>= 1.5-~~)
Provides:
librust-proxmox-apt-0+cache-dev (= ${binary:Version}),
librust-proxmox-apt-0.10+cache-dev (= ${binary:Version}),
librust-proxmox-apt-0.10.10+cache-dev (= ${binary:Version})
Description: Proxmox library for APT - feature "cache"
This metapackage enables feature "cache" for the Rust proxmox-apt crate, by
pulling in any additional dependencies needed by that feature.

143
proxmox-apt/src/api.rs Normal file
View File

@ -0,0 +1,143 @@
// API function that work without feature "cache"
use anyhow::{bail, Error};
use proxmox_apt_api_types::{
APTChangeRepositoryOptions, APTGetChangelogOptions, APTRepositoriesResult, APTRepositoryFile,
APTRepositoryHandle,
};
use proxmox_config_digest::ConfigDigest;
use crate::repositories::{APTRepositoryFileImpl, APTRepositoryImpl};
/// Retrieve the changelog of the specified package.
pub fn get_changelog(options: &APTGetChangelogOptions) -> Result<String, Error> {
let mut command = std::process::Command::new("apt-get");
command.arg("changelog");
command.arg("-qq"); // don't display download progress
if let Some(ver) = &options.version {
command.arg(format!("{}={}", options.name, ver));
} else {
command.arg(&options.name);
}
let output = proxmox_sys::command::run_command(command, None)?;
Ok(output)
}
/// Get APT repository information.
pub fn list_repositories(product: &str) -> Result<APTRepositoriesResult, Error> {
let (files, errors, digest) = crate::repositories::repositories()?;
let suite = crate::repositories::get_current_release_codename()?;
let infos = crate::repositories::check_repositories(&files, suite);
let standard_repos = crate::repositories::standard_repositories(&files, product, suite);
Ok(APTRepositoriesResult {
files,
errors,
digest,
infos,
standard_repos,
})
}
/// Add the repository identified by the `handle`.
/// If the repository is already configured, it will be set to enabled.
///
/// The `digest` parameter asserts that the configuration has not been modified.
pub fn add_repository_handle(
product: &str,
handle: APTRepositoryHandle,
digest: Option<ConfigDigest>,
) -> Result<(), Error> {
let (mut files, errors, current_digest) = crate::repositories::repositories()?;
current_digest.detect_modification(digest.as_ref())?;
let suite = crate::repositories::get_current_release_codename()?;
// check if it's already configured first
for file in files.iter_mut() {
for repo in file.repositories.iter_mut() {
if repo.is_referenced_repository(handle, "pbs", &suite.to_string()) {
if repo.enabled {
return Ok(());
}
repo.set_enabled(true);
file.write()?;
return Ok(());
}
}
}
let (repo, path) = crate::repositories::get_standard_repository(handle, product, suite);
if let Some(error) = errors.iter().find(|error| error.path == path) {
bail!(
"unable to parse existing file {} - {}",
error.path,
error.error,
);
}
if let Some(file) = files
.iter_mut()
.find(|file| file.path.as_ref() == Some(&path))
{
file.repositories.push(repo);
file.write()?;
} else {
let mut file = match APTRepositoryFile::new(&path)? {
Some(file) => file,
None => bail!("invalid path - {}", path),
};
file.repositories.push(repo);
file.write()?;
}
Ok(())
}
/// Change the properties of the specified repository.
///
/// The `digest` parameter asserts that the configuration has not been modified.
pub fn change_repository(
path: &str,
index: usize,
options: &APTChangeRepositoryOptions,
digest: Option<ConfigDigest>,
) -> Result<(), Error> {
let (mut files, errors, current_digest) = crate::repositories::repositories()?;
current_digest.detect_modification(digest.as_ref())?;
if let Some(error) = errors.iter().find(|error| error.path == path) {
bail!("unable to parse file {} - {}", error.path, error.error);
}
if let Some(file) = files
.iter_mut()
.find(|file| file.path.as_deref() == Some(path))
{
if let Some(repo) = file.repositories.get_mut(index) {
if let Some(enabled) = options.enabled {
repo.set_enabled(enabled);
}
file.write()?;
} else {
bail!("invalid index - {}", index);
}
} else {
bail!("invalid path - {}", path);
}
Ok(())
}

301
proxmox-apt/src/cache.rs Normal file
View File

@ -0,0 +1,301 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use anyhow::{bail, format_err, Error};
use apt_pkg_native::Cache;
use proxmox_schema::const_regex;
use proxmox_sys::fs::{file_read_optional_string, replace_file, CreateOptions};
use proxmox_apt_api_types::APTUpdateInfo;
#[derive(Debug, serde::Serialize, serde::Deserialize)]
/// Some information we cache about the package (update) state, like what pending update version
/// we already notfied an user about
pub struct PkgState {
/// simple map from package name to most recently notified (emailed) version
pub notified: Option<HashMap<String, String>>,
/// A list of pending updates
pub package_status: Vec<APTUpdateInfo>,
}
pub fn write_pkg_cache<P: AsRef<Path>>(apt_state_file: P, state: &PkgState) -> Result<(), Error> {
let serialized_state = serde_json::to_string(state)?;
replace_file(
apt_state_file,
serialized_state.as_bytes(),
CreateOptions::new(),
false,
)
.map_err(|err| format_err!("Error writing package cache - {}", err))?;
Ok(())
}
pub fn read_pkg_state<P: AsRef<Path>>(apt_state_file: P) -> Result<Option<PkgState>, Error> {
let serialized_state = match file_read_optional_string(apt_state_file) {
Ok(Some(raw)) => raw,
Ok(None) => return Ok(None),
Err(err) => bail!("could not read cached package state file - {}", err),
};
serde_json::from_str(&serialized_state)
.map(Some)
.map_err(|err| format_err!("could not parse cached package status - {}", err))
}
pub fn pkg_cache_expired<P: AsRef<Path>>(apt_state_file: P) -> Result<bool, Error> {
if let Ok(pbs_cache) = std::fs::metadata(apt_state_file) {
let apt_pkgcache = std::fs::metadata("/var/cache/apt/pkgcache.bin")?;
let dpkg_status = std::fs::metadata("/var/lib/dpkg/status")?;
let mtime = pbs_cache.modified()?;
if apt_pkgcache.modified()? <= mtime && dpkg_status.modified()? <= mtime {
return Ok(false);
}
}
Ok(true)
}
pub fn update_cache<P: AsRef<Path>>(apt_state_file: P) -> Result<PkgState, Error> {
let apt_state_file = apt_state_file.as_ref();
// update our cache
let all_upgradeable = list_installed_apt_packages(
|data| {
data.candidate_version == data.active_version
&& data.installed_version != Some(data.candidate_version)
},
None,
);
let cache = match read_pkg_state(apt_state_file) {
Ok(Some(mut cache)) => {
cache.package_status = all_upgradeable;
cache
}
_ => PkgState {
notified: None,
package_status: all_upgradeable,
},
};
write_pkg_cache(apt_state_file, &cache)?;
Ok(cache)
}
const_regex! {
VERSION_EPOCH_REGEX = r"^\d+:";
FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
}
pub struct FilterData<'a> {
/// package name
pub package: &'a str,
/// this is version info returned by APT
pub installed_version: Option<&'a str>,
pub candidate_version: &'a str,
/// this is the version info the filter is supposed to check
pub active_version: &'a str,
}
enum PackagePreSelect {
OnlyInstalled,
OnlyNew,
All,
}
pub fn list_installed_apt_packages<F: Fn(FilterData) -> bool>(
filter: F,
only_versions_for: Option<&str>,
) -> Vec<APTUpdateInfo> {
let mut ret = Vec::new();
let mut depends = HashSet::new();
// note: this is not an 'apt update', it just re-reads the cache from disk
let mut cache = Cache::get_singleton();
cache.reload();
let mut cache_iter = match only_versions_for {
Some(name) => cache.find_by_name(name),
None => cache.iter(),
};
loop {
match cache_iter.next() {
Some(view) => {
let di = if only_versions_for.is_some() {
query_detailed_info(PackagePreSelect::All, &filter, view, None)
} else {
query_detailed_info(
PackagePreSelect::OnlyInstalled,
&filter,
view,
Some(&mut depends),
)
};
if let Some(info) = di {
ret.push(info);
}
if only_versions_for.is_some() {
break;
}
}
None => {
drop(cache_iter);
// also loop through missing dependencies, as they would be installed
for pkg in depends.iter() {
let mut iter = cache.find_by_name(pkg);
let view = match iter.next() {
Some(view) => view,
None => continue, // package not found, ignore
};
let di = query_detailed_info(PackagePreSelect::OnlyNew, &filter, view, None);
if let Some(info) = di {
ret.push(info);
}
}
break;
}
}
}
ret
}
fn query_detailed_info<'a, F, V>(
pre_select: PackagePreSelect,
filter: F,
view: V,
depends: Option<&mut HashSet<String>>,
) -> Option<APTUpdateInfo>
where
F: Fn(FilterData) -> bool,
V: std::ops::Deref<Target = apt_pkg_native::sane::PkgView<'a>>,
{
let current_version = view.current_version();
let candidate_version = view.candidate_version();
let (current_version, candidate_version) = match pre_select {
PackagePreSelect::OnlyInstalled => match (current_version, candidate_version) {
(Some(cur), Some(can)) => (Some(cur), can), // package installed and there is an update
(Some(cur), None) => (Some(cur.clone()), cur), // package installed and up-to-date
(None, Some(_)) => return None, // package could be installed
(None, None) => return None, // broken
},
PackagePreSelect::OnlyNew => match (current_version, candidate_version) {
(Some(_), Some(_)) => return None,
(Some(_), None) => return None,
(None, Some(can)) => (None, can),
(None, None) => return None,
},
PackagePreSelect::All => match (current_version, candidate_version) {
(Some(cur), Some(can)) => (Some(cur), can),
(Some(cur), None) => (Some(cur.clone()), cur),
(None, Some(can)) => (None, can),
(None, None) => return None,
},
};
// get additional information via nested APT 'iterators'
let mut view_iter = view.versions();
while let Some(ver) = view_iter.next() {
let package = view.name();
let version = ver.version();
let mut origin_res = "unknown".to_owned();
let mut section_res = "unknown".to_owned();
let mut priority_res = "unknown".to_owned();
let mut short_desc = package.clone();
let mut long_desc = "".to_owned();
let fd = FilterData {
package: package.as_str(),
installed_version: current_version.as_deref(),
candidate_version: &candidate_version,
active_version: &version,
};
if filter(fd) {
if let Some(section) = ver.section() {
section_res = section;
}
if let Some(prio) = ver.priority_type() {
priority_res = prio;
}
// assume every package has only one origin file (not
// origin, but origin *file*, for some reason those seem to
// be different concepts in APT)
let mut origin_iter = ver.origin_iter();
let origin = origin_iter.next();
if let Some(origin) = origin {
if let Some(sd) = origin.short_desc() {
short_desc = sd;
}
if let Some(ld) = origin.long_desc() {
long_desc = ld;
}
// the package files appear in priority order, meaning
// the one for the candidate version is first - this is fine
// however, as the source package should be the same for all
// versions anyway
let mut pkg_iter = origin.file();
let pkg_file = pkg_iter.next();
if let Some(pkg_file) = pkg_file {
if let Some(origin_name) = pkg_file.origin() {
origin_res = origin_name;
}
}
}
if let Some(depends) = depends {
let mut dep_iter = ver.dep_iter();
loop {
let dep = match dep_iter.next() {
Some(dep) if dep.dep_type() != "Depends" => continue,
Some(dep) => dep,
None => break,
};
let dep_pkg = dep.target_pkg();
let name = dep_pkg.name();
depends.insert(name);
}
}
return Some(APTUpdateInfo {
package,
title: short_desc,
arch: view.arch(),
description: long_desc,
origin: origin_res,
version: candidate_version.clone(),
old_version: match current_version {
Some(vers) => vers,
None => "".to_owned(),
},
priority: priority_res,
section: section_res,
extra_info: None,
});
}
}
None
}
pub fn sort_package_list(packages: &mut Vec<APTUpdateInfo>) {
let cache = apt_pkg_native::Cache::get_singleton();
packages.sort_by(|left, right| {
cache
.compare_versions(&left.old_version, &right.old_version)
.reverse()
});
}

View File

@ -0,0 +1,208 @@
// API function that need feature "cache"
use std::path::Path;
use anyhow::{bail, format_err, Error};
use std::os::unix::prelude::OsStrExt;
use proxmox_apt_api_types::{APTUpdateInfo, APTUpdateOptions};
/// List available APT updates
///
/// Automatically updates an expired package cache.
pub fn list_available_apt_update<P: AsRef<Path>>(
apt_state_file: P,
) -> Result<Vec<APTUpdateInfo>, Error> {
let apt_state_file = apt_state_file.as_ref();
if let Ok(false) = crate::cache::pkg_cache_expired(apt_state_file) {
if let Ok(Some(cache)) = crate::cache::read_pkg_state(apt_state_file) {
return Ok(cache.package_status);
}
}
let cache = crate::cache::update_cache(apt_state_file)?;
Ok(cache.package_status)
}
/// Update the APT database
///
/// You should update the APT proxy configuration before running this.
pub fn update_database<P: AsRef<Path>>(
apt_state_file: P,
options: &APTUpdateOptions,
send_updates_available: impl Fn(&[&APTUpdateInfo]) -> Result<(), Error>,
) -> Result<(), Error> {
let apt_state_file = apt_state_file.as_ref();
let quiet = options.quiet.unwrap_or(false);
let notify = options.notify.unwrap_or(false);
if !quiet {
log::info!("starting apt-get update")
}
let mut command = std::process::Command::new("apt-get");
command.arg("update");
// apt "errors" quite easily, and run_command is a bit rigid, so handle this inline for now.
let output = command
.output()
.map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
if !quiet {
log::info!("{}", String::from_utf8(output.stdout)?);
}
// TODO: improve run_command to allow outputting both, stderr and stdout
if !output.status.success() {
if output.status.code().is_some() {
let msg = String::from_utf8(output.stderr)
.map(|m| {
if m.is_empty() {
String::from("no error message")
} else {
m
}
})
.unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
log::warn!("{msg}");
} else {
bail!("terminated by signal");
}
}
let mut cache = crate::cache::update_cache(apt_state_file)?;
if notify {
let mut notified = match cache.notified {
Some(notified) => notified,
None => std::collections::HashMap::new(),
};
let mut to_notify: Vec<&APTUpdateInfo> = Vec::new();
for pkg in &cache.package_status {
match notified.insert(pkg.package.to_owned(), pkg.version.to_owned()) {
Some(notified_version) => {
if notified_version != pkg.version {
to_notify.push(pkg);
}
}
None => to_notify.push(pkg),
}
}
if !to_notify.is_empty() {
to_notify.sort_unstable_by_key(|k| &k.package);
send_updates_available(&to_notify)?;
}
cache.notified = Some(notified);
crate::cache::write_pkg_cache(apt_state_file, &cache)?;
}
Ok(())
}
/// Get package information for a list of important product packages.
///
/// We first list the product virtual package (i.e. `proxmox-backup`), with extra
/// information about the running kernel.
///
/// Next is the api_server_package, with extra information abnout the running api
/// server version.
///
/// The list of installed kernel packages follows.
///
/// We the add an entry for all packages in package_list, even if they are
/// not installed.
pub fn get_package_versions(
product_virtual_package: &str,
api_server_package: &str,
running_api_server_version: &str,
package_list: &[&str],
) -> Result<Vec<APTUpdateInfo>, Error> {
fn unknown_package(package: String, extra_info: Option<String>) -> APTUpdateInfo {
APTUpdateInfo {
package,
title: "unknown".into(),
arch: "unknown".into(),
description: "unknown".into(),
version: "unknown".into(),
old_version: "unknown".into(),
origin: "unknown".into(),
priority: "unknown".into(),
section: "unknown".into(),
extra_info,
}
}
let mut packages: Vec<APTUpdateInfo> = Vec::new();
let is_kernel =
|name: &str| name.starts_with("pve-kernel-") || name.starts_with("proxmox-kernel");
let installed_packages = crate::cache::list_installed_apt_packages(
|filter| {
filter.installed_version == Some(filter.active_version)
&& (is_kernel(filter.package)
|| (filter.package == product_virtual_package)
|| (filter.package == api_server_package)
|| package_list.contains(&filter.package))
},
None,
);
let running_kernel = format!(
"running kernel: {}",
std::str::from_utf8(nix::sys::utsname::uname()?.release().as_bytes())?.to_owned()
);
if let Some(product_virtual_package_info) = installed_packages
.iter()
.find(|pkg| pkg.package == product_virtual_package)
{
let mut product_virtual_package_info = product_virtual_package_info.clone();
product_virtual_package_info.extra_info = Some(running_kernel);
packages.push(product_virtual_package_info);
} else {
packages.push(unknown_package(
product_virtual_package.into(),
Some(running_kernel),
));
}
if let Some(api_server_package_info) = installed_packages
.iter()
.find(|pkg| pkg.package == api_server_package)
{
let mut api_server_package_info = api_server_package_info.clone();
api_server_package_info.extra_info = Some(running_api_server_version.into());
packages.push(api_server_package_info);
} else {
packages.push(unknown_package(
api_server_package.into(),
Some(running_api_server_version.into()),
));
}
let mut kernel_pkgs: Vec<APTUpdateInfo> = installed_packages
.iter()
.filter(|pkg| is_kernel(&pkg.package))
.cloned()
.collect();
crate::cache::sort_package_list(&mut kernel_pkgs);
packages.append(&mut kernel_pkgs);
// add entry for all packages we're interested in, even if not installed
for pkg in package_list.iter() {
if *pkg == product_virtual_package || *pkg == api_server_package {
continue;
}
match installed_packages.iter().find(|item| &item.package == pkg) {
Some(apt_pkg) => packages.push(apt_pkg.to_owned()),
None => packages.push(unknown_package(pkg.to_string(), None)),
}
}
Ok(packages)
}

View File

@ -1,5 +1,14 @@
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
mod api;
pub use api::{add_repository_handle, change_repository, get_changelog, list_repositories};
#[cfg(feature = "cache")]
pub mod cache;
#[cfg(feature = "cache")]
mod cache_api;
#[cfg(feature = "cache")]
pub use cache_api::{get_package_versions, list_available_apt_update, update_database};
pub mod config;
pub mod deb822;
pub mod repositories;

View File

@ -5,11 +5,13 @@ use anyhow::{bail, format_err, Error};
use proxmox_apt::config::APTConfig;
use proxmox_apt::repositories::{
check_repositories, get_current_release_codename, standard_repositories, APTRepositoryFile,
APTRepositoryHandle, APTRepositoryInfo, APTStandardRepository, DebianCodename,
check_repositories, get_current_release_codename, standard_repositories, DebianCodename,
};
use proxmox_apt::repositories::{
APTRepositoryFileImpl, APTRepositoryHandleImpl, APTRepositoryImpl, APTStandardRepositoryImpl,
APTRepositoryFileImpl, APTRepositoryImpl, APTStandardRepositoryImpl,
};
use proxmox_apt_api_types::{
APTRepositoryFile, APTRepositoryHandle, APTRepositoryInfo, APTStandardRepository,
};
fn create_clean_directory(path: &PathBuf) -> Result<(), Error> {