auto-updates: Cache cached-update GVariant to disk

Rather than recalculating `cached-update` as part of transaction
cleanups and RpmostreedOS internal reloads, write it directly to a file
from `deploy_transaction_execute`. This gives two major benefits:

1. Auto-updates now has virtually zero impact to daemon startup time.
2. We get to directly use the `DnfSack` created during metadata refresh
   rather than reconstructing it later on. This greatly simplifies code.

This makes use of new APIs in libdnf to skip filelists and load
updateinfo metadata right from the start.

Closes: #1268
Approved by: cgwalters
This commit is contained in:
Jonathan Lebon 2018-03-05 18:08:38 +00:00 committed by Atomic Bot
parent 1b580ce201
commit 05e09cd72c
7 changed files with 181 additions and 179 deletions

View File

@ -81,7 +81,7 @@
'signatures' (type 'av')
'gpg-enabled' (type 'b')
'ref-has-new-commit' (type 'b')
TRUE if 'checksum' refers to a new commit we're not booted in.
TRUE if 'checksum' refers to a new base commit we're not booted in.
'rpm-diff' (type 'a{sv}')
'upgraded' (type 'a(us(ss)(ss))')
'downgraded' (type 'a(us(ss)(ss))')

View File

@ -678,11 +678,9 @@ find_newer_package (DnfSack *sack,
* @rpm_diff, and all new pkgs in @out_newer_packages (these are used later for advisories).
* */
static gboolean
rpmmd_diff (OstreeSysroot *sysroot,
/* these are just to avoid refetching them */
OstreeRepo *repo,
OstreeDeployment *deployment,
rpmmd_diff (OstreeRepo *repo,
const char *base_checksum,
const char *layered_checksum,
DnfSack *sack,
RpmDiff *rpm_diff,
GPtrArray **out_newer_packages,
@ -694,7 +692,6 @@ rpmmd_diff (OstreeSysroot *sysroot,
* layered_pkgs. */
g_autoptr(GPtrArray) all_layered_pkgs = NULL;
const char *layered_checksum = ostree_deployment_get_csum (deployment);
RpmOstreeDbDiffExtFlags flags = RPM_OSTREE_DB_DIFF_EXT_ALLOW_NOENT;
if (!rpm_ostree_db_diff_ext (repo, base_checksum, layered_checksum, flags, NULL,
&all_layered_pkgs, NULL, NULL, NULL, error))
@ -878,104 +875,20 @@ rpm_ostree_pkgs_to_dnf (DnfSack *sack,
return g_steal_pointer (&dnf_pkgs);
}
static gboolean
get_cached_rpmmd_sack (OstreeSysroot *sysroot,
OstreeRepo *repo,
OstreeDeployment *deployment,
DnfSack **out_sack,
GError **error)
{
/* we don't need the full force of the core ctx here; we just want a DnfContext so that it
* can load the repos and deal with releasever for us */
g_autoptr(DnfContext) ctx = dnf_context_new ();
/* We have to point to the same source root for releasever to hit the right cache: an
* interesting point here is that if there's a newer $releasever pending (i.e. 'deploy'
* auto update policy), we'll still be using the previous releasever -- this is OK though,
* we should be special-casing these rebases later re. how to display them; at least
* status already shows endoflife. See also deploy_transaction_execute(). */
g_autofree char *deployment_root = rpmostree_get_deployment_root (sysroot, deployment);
dnf_context_set_source_root (ctx, deployment_root);
g_autofree char *reposdir = g_build_filename (deployment_root, "etc/yum.repos.d", NULL);
dnf_context_set_repo_dir (ctx, reposdir);
dnf_context_set_cache_dir (ctx, RPMOSTREE_CORE_CACHEDIR RPMOSTREE_DIR_CACHE_REPOMD);
dnf_context_set_solv_dir (ctx, RPMOSTREE_CORE_CACHEDIR RPMOSTREE_DIR_CACHE_SOLV);
if (!dnf_context_setup (ctx, NULL, error))
return FALSE;
/* add the repos but strictly from cache; we should have already *just* checked &
* refreshed metadata as part of the DeployTransaction; but we gracefully handle bad cache
* too (e.g. if we start using the new dnf_context_clean_cache() on rebases?) */
GPtrArray *repos = dnf_context_get_repos (ctx);
/* need a new DnfSackAddFlags flag for dnf_sack_add_repos to say "don't fallback to
* updating if cache invalid/absent"; for now, just do it ourselves */
GPtrArray *cached_enabled_repos = g_ptr_array_new ();
for (guint i = 0; i < repos->len; i++)
{
DnfRepo *repo = repos->pdata[i];
if ((dnf_repo_get_enabled (repo) & DNF_REPO_ENABLED_PACKAGES) == 0)
continue;
/* TODO: We need to expand libdnf here to somehow do a dnf_repo_check() without it
* triggering a download if there's no cache at all. Here, we just physically check
* for the location. */
const char *location = dnf_repo_get_location (repo);
if (!glnx_fstatat_allow_noent (AT_FDCWD, location, NULL, 0, error))
return FALSE;
if (errno == ENOENT)
continue;
g_autoptr(GError) local_error = NULL;
g_autoptr(DnfState) state = dnf_state_new ();
if (!dnf_repo_check (repo, G_MAXUINT, state, &local_error))
sd_journal_print (LOG_WARNING, "Couldn't load cache for repo %s: %s",
dnf_repo_get_id (repo), local_error->message);
else
g_ptr_array_add (cached_enabled_repos, repo);
}
g_autoptr(DnfSack) sack = NULL;
if (cached_enabled_repos->len > 0)
{
/* Set up our own sack and point it to the solv cache. TODO: We could've used the sack
* from dnf_context_setup_sack(), but we need to extend libdnf to specify flags like
* UPDATEINFO beforehand. Otherwise we have to add_repos() twice which almost double
* startup time. */
sack = dnf_sack_new ();
dnf_sack_set_cachedir (sack, RPMOSTREE_CORE_CACHEDIR RPMOSTREE_DIR_CACHE_SOLV);
if (!dnf_sack_setup (sack, DNF_SACK_SETUP_FLAG_MAKE_CACHE_DIR, error))
return FALSE;
/* we still use add_repos rather than add_repo separately above because it does nice
* things like process excludes */
g_autoptr(DnfState) state = dnf_state_new ();
if (!dnf_sack_add_repos (sack, cached_enabled_repos, G_MAXUINT,
DNF_SACK_ADD_FLAG_UPDATEINFO, state, error))
return FALSE;
}
*out_sack = g_steal_pointer (&sack);
return TRUE;
}
/* The variant returned by this function is backwards compatible with the one returned by
* rpmostreed_commit_generate_cached_details_variant(). However, it also includes a base
* tree db diff, layered pkgs diff, state, advisories, etc... Also, it will happily return
* NULL if no updates are available. */
gboolean
rpmostreed_update_generate_variant (OstreeSysroot *sysroot,
OstreeDeployment *deployment,
OstreeRepo *repo,
GVariant **out_update,
GError **error)
rpmostreed_update_generate_variant (OstreeDeployment *deployment,
OstreeRepo *repo,
DnfSack *sack,
GVariant **out_update,
GCancellable *cancellable,
GError **error)
{
/* We try to minimize I/O in this function. We're in the daemon startup path, and thus
* directly contribute to lag from a cold `rpm-ostree status`. Anyway, as a principle we
* shouldn't do long-running operations outside of transactions. */
g_autoptr(RpmOstreeOrigin) origin = rpmostree_origin_parse_deployment (deployment, error);
g_autoptr(RpmOstreeOrigin) origin =
rpmostree_origin_parse_deployment (deployment, error);
if (!origin)
return FALSE;
@ -1003,22 +916,23 @@ rpmostreed_update_generate_variant (OstreeSysroot *sysroot,
return FALSE;
const char *current_checksum = ostree_deployment_get_csum (deployment);
const char *current_base_checksum = current_checksum;
gboolean is_layered;
g_autofree char *current_checksum_owned = NULL;
g_autofree char *current_base_checksum_owned = NULL;
if (!rpmostree_deployment_get_layered_info (repo, deployment, &is_layered,
&current_checksum_owned, NULL, NULL, NULL,
&current_base_checksum_owned, NULL, NULL, NULL,
error))
return FALSE;
if (is_layered)
current_checksum = current_checksum_owned;
current_base_checksum = current_base_checksum_owned;
/* Graciously handle rev no longer in repo; e.g. mucking around with rebase/rollback; we
* still want to do the rpm-md phase. In that case, just use the current csum. */
gboolean is_new_checksum = FALSE;
if (!new_checksum)
new_checksum = g_strdup (current_checksum);
new_checksum = g_strdup (current_base_checksum);
else
is_new_checksum = !g_str_equal (new_checksum, current_checksum);
is_new_checksum = !g_str_equal (new_checksum, current_base_checksum);
g_autoptr(GVariant) commit = NULL;
if (!ostree_repo_load_commit (repo, new_checksum, &commit, NULL, error))
@ -1050,8 +964,9 @@ rpmostreed_update_generate_variant (OstreeSysroot *sysroot,
/* Note we allow_noent here; we'll just skip over the rpm diff if there's no data */
RpmOstreeDbDiffExtFlags flags = RPM_OSTREE_DB_DIFF_EXT_ALLOW_NOENT;
if (!rpm_ostree_db_diff_ext (repo, current_checksum, new_checksum, flags, &removed,
&added, &modified_old, &ostree_modified_new, NULL, error))
if (!rpm_ostree_db_diff_ext (repo, current_base_checksum, new_checksum, flags,
&removed, &added, &modified_old, &ostree_modified_new,
cancellable, error))
return FALSE;
/* check if allow_noent kicked in */
@ -1062,18 +977,13 @@ rpmostreed_update_generate_variant (OstreeSysroot *sysroot,
/* now we look at the rpm-md side */
/* first we try to set up a sack (NULL --> no cache available) */
g_autoptr(DnfSack) sack = NULL;
if (!get_cached_rpmmd_sack (sysroot, repo, deployment, &sack, error))
return FALSE;
g_autoptr(GPtrArray) rpmmd_modified_new = NULL;
GHashTable *layered_pkgs = rpmostree_origin_get_packages (origin);
/* check that it's actually layered (i.e. the requests are not all just dormant) */
if (sack && is_layered && g_hash_table_size (layered_pkgs) > 0)
{
if (!rpmmd_diff (sysroot, repo, deployment, current_checksum, sack, &rpm_diff,
if (!rpmmd_diff (repo, current_base_checksum, current_checksum, sack, &rpm_diff,
&rpmmd_modified_new, error))
return FALSE;
}
@ -1112,7 +1022,13 @@ rpmostreed_update_generate_variant (OstreeSysroot *sysroot,
/* but if there are no updates, then just ditch the whole thing and return NULL */
if (is_new_checksum || rpmmd_modified_new)
*out_update = g_variant_ref_sink (g_variant_dict_end (&dict));
{
/* include a "state" checksum for cache invalidation; for now this is just the
* checksum of the deployment against which we ran, though we could base it off more
* things later if needed */
g_variant_dict_insert (&dict, "update-sha256", "s", current_checksum);
*out_update = g_variant_ref_sink (g_variant_dict_end (&dict));
}
else
*out_update = NULL;

View File

@ -19,6 +19,7 @@
#pragma once
#include <ostree.h>
#include <libdnf/libdnf.h>
#include "rpmostreed-types.h"
@ -45,9 +46,10 @@ GVariant * rpmostreed_commit_generate_cached_details_variant (OstreeDeploym
const char *refspec,
const char *checksum,
GError **error);
gboolean
rpmostreed_update_generate_variant (OstreeSysroot *sysroot,
OstreeDeployment *deployment,
OstreeRepo *repo,
GVariant **out_update,
GError **error);
gboolean rpmostreed_update_generate_variant (OstreeDeployment *deployment,
OstreeRepo *repo,
DnfSack *sack,
GVariant **out_update,
GCancellable *cancellable,
GError **error);

View File

@ -914,15 +914,15 @@ os_handle_update_deployment (RPMOSTreeOS *interface,
}
/* compat shim for call completer */
static void automatic_update_trigger_completer (RPMOSTreeOS *os,
GDBusMethodInvocation *invocation,
GUnixFDList *dummy,
const gchar *address)
static void
automatic_update_trigger_completer (RPMOSTreeOS *os,
GDBusMethodInvocation *invocation,
GUnixFDList *dummy,
const gchar *address)
{ /* enabled */
rpmostree_os_complete_automatic_update_trigger (os, invocation, TRUE, address);
}
/* we make this a separate method to keep the D-Bus API clean, but the actual
* implementation is done by our dear friend deploy_transaction_execute().
*/
@ -932,11 +932,26 @@ os_handle_automatic_update_trigger (RPMOSTreeOS *interface,
GDBusMethodInvocation *invocation,
GVariant *arg_options)
{
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
const char *osname = rpmostree_os_get_name (interface);
OstreeSysroot *sysroot = rpmostreed_sysroot_get_root (rpmostreed_sysroot_get ());
OstreeDeployment *booted = ostree_sysroot_get_booted_deployment (sysroot);
if (!booted || !g_str_equal (osname, ostree_deployment_get_osname (booted)))
{
/* We're not even booted in our osname; let's just not handle this case for now until
* someone shows up with a good use case for it. We only want the booted OS to use the
* current /var/cache, since it's per-OS. If we ever allow this, we'll want to
* invalidate the auto-update cache on the osname as well. */
glnx_throw (error, "Cannot trigger auto-update for offline OS '%s'", osname);
g_dbus_method_invocation_take_error (invocation, g_steal_pointer (&local_error));
return TRUE;
}
g_auto(GVariantDict) dict;
g_variant_dict_init (&dict, arg_options);
const char *mode = vardict_lookup_ptr (&dict, "mode", "&s") ?: "auto";
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
RpmostreedAutomaticUpdatePolicy autoupdate_policy;
if (g_str_equal (mode, "auto"))
@ -1696,53 +1711,72 @@ out:
return TRUE;
}
/* split out for easier handling of the NULL case */
static gboolean
refresh_cached_update_impl (RpmostreedOS *self,
GVariant **out_cached_update,
GError **error)
{
g_autoptr(GVariant) cached_update = NULL;
/* if we're not booted into our OS, don't look at cache; it's for another OS interface */
const char *osname = rpmostree_os_get_name (RPMOSTREE_OS (self));
OstreeSysroot *sysroot = rpmostreed_sysroot_get_root (rpmostreed_sysroot_get ());
OstreeDeployment *booted = ostree_sysroot_get_booted_deployment (sysroot);
if (!booted || !g_str_equal (osname, ostree_deployment_get_osname (booted)))
return TRUE; /* Note early return */
glnx_autofd int fd = -1;
g_autoptr(GError) local_error = NULL;
if (!glnx_openat_rdonly (AT_FDCWD, RPMOSTREE_AUTOUPDATES_CACHE_FILE, TRUE, &fd,
&local_error))
{
if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
return g_propagate_error (error, g_steal_pointer (&local_error)), FALSE;
return TRUE; /* Note early return */
}
/* sanity check there isn't something fishy going on before even reading it in */
struct stat stbuf;
if (!glnx_fstat (fd, &stbuf, error))
return FALSE;
if (!rpmostree_check_size_within_limit (stbuf.st_size, OSTREE_MAX_METADATA_SIZE,
RPMOSTREE_AUTOUPDATES_CACHE_FILE, error))
return FALSE;
g_autoptr(GBytes) data = glnx_fd_readall_bytes (fd, NULL, error);
if (!data)
return FALSE;
cached_update =
g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, data, FALSE));
/* check if cache is still valid -- see rpmostreed_update_generate_variant() */
g_auto(GVariantDict) dict;
g_variant_dict_init (&dict, cached_update);
const char *state = vardict_lookup_ptr (&dict, "update-sha256", "&s");
if (g_strcmp0 (state, ostree_deployment_get_csum (booted)) != 0)
{
sd_journal_print (LOG_INFO, "Deleting outdated cached update for OS '%s'", osname);
g_clear_pointer (&cached_update, (GDestroyNotify)g_variant_unref);
if (!glnx_unlinkat (AT_FDCWD, RPMOSTREE_AUTOUPDATES_CACHE_FILE, 0, error))
return FALSE;
}
*out_cached_update = g_steal_pointer (&cached_update);
return TRUE;
}
static gboolean
refresh_cached_update (RpmostreedOS *self, GError **error)
{
const char *name = rpmostree_os_get_name (RPMOSTREE_OS (self));
OstreeSysroot *sysroot = rpmostreed_sysroot_get_root (rpmostreed_sysroot_get ());
OstreeRepo *repo = ostree_sysroot_repo (sysroot);
/* if we're not handling the system sysroot, then let's just skip all this (e.g. `make
* check` tests) */
const char *sysroot_path = gs_file_get_path_cached (ostree_sysroot_get_path (sysroot));
if (!g_str_equal (sysroot_path, "/"))
return TRUE;
/* Note here we're *not* using rpmostree_syscore_get_origin_merge_deployment(): cached
* updates are always relative to the booted/merge deployment; e.g. we still want to be
* able to show details about a pending deployment. */
g_autoptr(OstreeDeployment) merge_deployment =
ostree_sysroot_get_merge_deployment (sysroot, name);
g_autoptr(GVariant) cached_update = NULL;
if (merge_deployment)
{
RpmostreedAutomaticUpdatePolicy autoupdate_policy =
rpmostreed_get_automatic_update_policy (rpmostreed_daemon_get ());
/* if auto-updates is off, just fill in the backcompat stuff */
if (autoupdate_policy == RPMOSTREED_AUTOMATIC_UPDATE_POLICY_NONE)
{
cached_update = rpmostreed_commit_generate_cached_details_variant (merge_deployment,
repo, NULL, NULL,
error);
if (!cached_update)
return FALSE;
}
else
{
if (!rpmostreed_update_generate_variant (sysroot, merge_deployment, repo,
&cached_update, error))
return FALSE;
}
}
if (!refresh_cached_update_impl (self, &cached_update, error))
return FALSE;
rpmostree_os_set_cached_update (RPMOSTREE_OS (self), cached_update);
/* for backwards compatibility */
gboolean has_cached_updates = (cached_update != NULL);
rpmostree_os_set_has_cached_update_rpm_diff (RPMOSTREE_OS (self), has_cached_updates);
rpmostree_os_set_has_cached_update_rpm_diff (RPMOSTREE_OS (self), cached_update != NULL);
return TRUE;
}

View File

@ -20,6 +20,7 @@
#include "ostree.h"
#include <libglnx.h>
#include <systemd/sd-journal.h>
#include "rpmostreed-transaction-types.h"
#include "rpmostreed-transaction.h"
@ -1002,6 +1003,19 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
* have pkgs layered). This is still just a heuristic, since e.g. an InactiveRequest
* may in fact become active in the new base, but we don't have the full tree. */
/* Note here that we use the booted deployment for releasever: the download metadata
* only path is currently used only by the auto-update checker, and there we always
* want to show updates/vulnerabilities relative to the *booted* releasever. */
OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (sysroot);
/* this is checked in AutomaticUpdateTrigger, but check here too to be safe */
if (!booted_deployment ||
!g_str_equal (self->osname, ostree_deployment_get_osname (booted_deployment)))
return glnx_throw (error, "Refusing to download rpm-md for offline OS '%s'",
self->osname);
g_autoptr(DnfSack) sack = NULL;
/* XXX: in rojig mode we'll want to do this unconditionally */
if (g_hash_table_size (rpmostree_origin_get_packages (origin)) > 0)
{
@ -1009,27 +1023,46 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
g_autoptr(RpmOstreeContext) ctx =
rpmostree_context_new_system (repo, cancellable, error);
/* Note here that we use the cfg merge deployment for releasever: the download
* metadata only path is currently used only by the auto-update checker, and there
* we want to show updates/vulnerabilities relative to the *booted* releasever.
* Anyway, given that we don't yet do etc merges on boot, it shouldn't be too
* common for users to stay long on e.g. f26 when they have f27 already deployed
* and ready to reboot into. */
g_autoptr(OstreeDeployment) cfg_merge_deployment =
ostree_sysroot_get_merge_deployment (sysroot, self->osname);
g_autofree char *source_root =
rpmostree_get_deployment_root (sysroot, cfg_merge_deployment);
rpmostree_get_deployment_root (sysroot, booted_deployment);
if (!rpmostree_context_setup (ctx, NULL, source_root, NULL, cancellable, error))
return FALSE;
/* we always want to force a refetch of the metadata */
dnf_context_set_cache_age (rpmostree_context_get_dnf (ctx), 0);
DnfContext *dnfctx = rpmostree_context_get_dnf (ctx);
dnf_context_set_cache_age (dnfctx, 0);
/* point libdnf to our repos dir */
rpmostree_context_configure_from_deployment (ctx, sysroot, cfg_merge_deployment);
rpmostree_context_configure_from_deployment (ctx, sysroot, booted_deployment);
if (!rpmostree_context_download_metadata (ctx, cancellable, error))
/* streamline: we don't need rpmdb or filelists, but we *do* need updateinfo */
if (!rpmostree_context_download_metadata (ctx,
DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB |
DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_FILELISTS |
DNF_CONTEXT_SETUP_SACK_FLAG_LOAD_UPDATEINFO, cancellable, error))
return FALSE;
sack = g_object_ref (dnf_context_get_sack (dnfctx));
}
/* now generate the variant and cache it to disk */
/* always delete first since we might not be replacing it at all */
if (!glnx_shutil_rm_rf_at (AT_FDCWD, RPMOSTREE_AUTOUPDATES_CACHE_FILE,
cancellable, error))
return FALSE;
g_autoptr(GVariant) update = NULL;
if (!rpmostreed_update_generate_variant (booted_deployment, repo, sack, &update,
cancellable, error))
return FALSE;
if (update != NULL)
{
if (!glnx_file_replace_contents_at (AT_FDCWD, RPMOSTREE_AUTOUPDATES_CACHE_FILE,
g_variant_get_data (update),
g_variant_get_size (update),
0, cancellable, error))
return FALSE;
}

View File

@ -36,6 +36,9 @@
#define RPMOSTREE_SYSIMAGE_DIR "usr/lib/sysimage"
#define RPMOSTREE_BASE_RPMDB RPMOSTREE_SYSIMAGE_DIR "/rpm-ostree-base-db"
/* put it in cache dir so it gets destroyed naturally with a `cleanup -m` */
#define RPMOSTREE_AUTOUPDATES_CACHE_FILE RPMOSTREE_CORE_CACHEDIR "cached-update.gv"
#define RPMOSTREE_TYPE_CONTEXT (rpmostree_context_get_type ())
G_DECLARE_FINAL_TYPE (RpmOstreeContext, rpmostree_context, RPMOSTREE, CONTEXT, GObject)

View File

@ -135,11 +135,18 @@ EOF
vm_rpmostree reload
}
vm_rpmostree cleanup -m
# make sure that off means off
change_policy off
vm_rpmostree status | grep 'auto updates disabled'
vm_rpmostree upgrade --trigger-automatic-update-policy > out.txt
assert_file_has_content out.txt "Automatic updates are not enabled; exiting"
# check we didn't download any metadata (skip starting dir)
vm_cmd find /var/cache/rpm-ostree | tail -n +2 > out.txt
if [ -s out.txt ]; then
cat out.txt && assert_not_reached "rpmmd downloaded!"
fi
echo "ok disabled"
# ok, let's test out check
@ -167,6 +174,13 @@ assert_file_has_content out.txt "Upgraded: layered-cake 2.1-3 -> 2.1-4"
! grep -A999 'Available update' out.txt | grep "Commit"
echo "ok check mode layered only"
# confirm no filelists were fetched
vm_cmd find /var/cache/rpm-ostree -iname '*filelists*' > out.txt
if [ -s out.txt ]; then
cat out.txt && assert_not_reached "Filelists were downloaded!"
fi
echo "ok no filelists"
# now add some advisory updates
vm_build_rpm layered-enh version 2.0 uinfo VMCHECK-ENH
vm_build_rpm layered-sec-none version 2.0 uinfo VMCHECK-SEC-NONE