From affa50fbc172f42b08448b228d7d855c8aa013ef Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 21 Dec 2017 12:29:49 +0100 Subject: [PATCH] Lower jigdo client logic into core Introduce a new `rpmostree_context_execute_jigdo()` that fills the same role as `ostree_repo_pull_with_options()`. This will be used by the sysroot upgrader. I didn't change the jigdo client code much yet; as a TODO says there's a lot more we can do to improve things. Some of the public APIs we added to the core no longer need to be public, such as `rpmostree_context_set_packages()`. But let's try to do things incrementally. I did at least change the `g_print()`s to `rpmostree_output_message()`. I dropped the `commit_and_print()`; at some point will come back and clean things up so we consistently journal/print stats. Closes: #1168 Approved by: jlebon --- Makefile-libpriv.am | 1 + src/app/rpmostree-ex-builtin-jigdo2commit.c | 222 +---------------- src/libpriv/rpmostree-core.h | 5 + src/libpriv/rpmostree-jigdo-client.c | 263 ++++++++++++++++++++ 4 files changed, 271 insertions(+), 220 deletions(-) create mode 100644 src/libpriv/rpmostree-jigdo-client.c diff --git a/Makefile-libpriv.am b/Makefile-libpriv.am index 9bb57892..320e57ec 100644 --- a/Makefile-libpriv.am +++ b/Makefile-libpriv.am @@ -51,6 +51,7 @@ librpmostreepriv_la_SOURCES = \ src/libpriv/rpmostree-jigdo-assembler.c \ src/libpriv/rpmostree-jigdo-assembler.h \ src/libpriv/rpmostree-jigdo-core.h \ + src/libpriv/rpmostree-jigdo-client.c \ src/libpriv/rpmostree-unpacker-core.c \ src/libpriv/rpmostree-unpacker-core.h \ src/libpriv/rpmostree-output.c \ diff --git a/src/app/rpmostree-ex-builtin-jigdo2commit.c b/src/app/rpmostree-ex-builtin-jigdo2commit.c index 5031df67..a1c34d1a 100644 --- a/src/app/rpmostree-ex-builtin-jigdo2commit.c +++ b/src/app/rpmostree-ex-builtin-jigdo2commit.c @@ -24,9 +24,6 @@ #include #include #include -// For the jigdo Requires parsing -#include -#include #include #include #include @@ -104,52 +101,6 @@ rpm_ostree_jigdo2commit_context_new (RpmOstreeJigdo2CommitContext **out_context, return TRUE; } -static DnfPackage * -query_jigdo_pkg (DnfContext *dnfctx, - const char *name, - const char *evr, - GError **error) -{ - hy_autoquery HyQuery query = hy_query_create (dnf_context_get_sack (dnfctx)); - /* See also similar examples of queries in e.g. dnf_context_update() */ - hy_query_filter (query, HY_PKG_NAME, HY_EQ, name); - hy_query_filter (query, HY_PKG_ARCH, HY_NEQ, "src"); - hy_query_filter (query, HY_PKG_EVR, HY_EQ, evr); - g_autoptr(GPtrArray) pkglist = hy_query_run (query); - if (pkglist->len == 0) - return glnx_null_throw (error, "Failed to find package %s-%s", name, evr); - return g_object_ref (pkglist->pdata[0]); -} - -static gboolean -commit_and_print (RpmOstreeJigdo2CommitContext *self, - RpmOstreeRepoAutoTransaction *txn, - GCancellable *cancellable, - GError **error) -{ - OstreeRepoTransactionStats stats; - if (!ostree_repo_commit_transaction (self->repo, &stats, cancellable, error)) - return FALSE; - txn->initialized = FALSE; - - g_print ("Metadata Total: %u\n", stats.metadata_objects_total); - g_print ("Metadata Written: %u\n", stats.metadata_objects_written); - g_print ("Content Total: %u\n", stats.content_objects_total); - g_print ("Content Written: %u\n", stats.content_objects_written); - g_print ("Content Bytes Written: %" G_GUINT64_FORMAT "\n", stats.content_bytes_written); - - return TRUE; -} - -static int -compare_pkgs (gconstpointer ap, - gconstpointer bp) -{ - DnfPackage **a = (gpointer)ap; - DnfPackage **b = (gpointer)bp; - return dnf_package_cmp (*a, *b); -} - static gboolean impl_jigdo2commit (RpmOstreeJigdo2CommitContext *self, const char *jigdo_id, @@ -176,177 +127,8 @@ impl_jigdo2commit (RpmOstreeJigdo2CommitContext *self, return FALSE; if (!rpmostree_context_prepare_jigdo (self->ctx, cancellable, error)) return FALSE; - - DnfPackage* oirpm_pkg = rpmostree_context_get_jigdo_pkg (self->ctx); - const char *provided_commit = rpmostree_context_get_jigdo_checksum (self->ctx); - - DnfContext *dnfctx = rpmostree_context_get_dnf (self->ctx); - g_print ("oirpm: %s (%s) commit=%s\n", dnf_package_get_nevra (oirpm_pkg), - dnf_package_get_reponame (oirpm_pkg), provided_commit); - - { OstreeRepoCommitState commitstate; - gboolean has_commit; - if (!ostree_repo_has_object (self->repo, OSTREE_OBJECT_TYPE_COMMIT, provided_commit, - &has_commit, cancellable, error)) - return FALSE; - if (has_commit) - { - if (!ostree_repo_load_commit (self->repo, provided_commit, NULL, - &commitstate, error)) - return FALSE; - if (!(commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL)) - { - g_print ("Commit is already written, nothing to do\n"); - return TRUE; /* 🔚 Early return */ - } - } - } - - g_autoptr(GPtrArray) pkgs_required = g_ptr_array_new_with_free_func (g_object_unref); - - /* Look at the Requires of the jigdoRPM. Note that we don't want to do - * dependency resolution here - that's part of the whole idea, we're doing - * deterministic imaging. - */ - g_autoptr(DnfReldepList) requires = dnf_package_get_requires (oirpm_pkg); - const gint n_requires = dnf_reldep_list_count (requires); - Pool *pool = dnf_sack_get_pool (dnf_context_get_sack (dnfctx)); - for (int i = 0; i < n_requires; i++) - { - DnfReldep *req = dnf_reldep_list_index (requires, i); - Id reqid = dnf_reldep_get_id (req); - if (!ISRELDEP (reqid)) - continue; - Reldep *rdep = GETRELDEP (pool, reqid); - /* This is the core hack; we're searching for Requires that - * have exact '=' versions. This assumes that the rpmbuild - * process won't inject such requirements. - */ - if (!(rdep->flags & REL_EQ)) - continue; - - const char *name = pool_id2str (pool, rdep->name); - const char *evr = pool_id2str (pool, rdep->evr); - - DnfPackage *pkg = query_jigdo_pkg (dnfctx, name, evr, error); - // FIXME: Possibly we shouldn't require a package to be in the repos if we - // already have it imported? This would help support downgrades if the - // repo owner has pruned. - if (!pkg) - return FALSE; - g_ptr_array_add (pkgs_required, g_object_ref (pkg)); - } - g_ptr_array_sort (pkgs_required, compare_pkgs); - - g_print ("Jigdo from %u packages\n", pkgs_required->len); - - /* For now we first serially download the oirpm, but down the line we can do - * this async. Doing so will require putting more of the jigdo logic into the - * core, so it knows not to import the jigdoRPM. - */ - { g_autoptr(GPtrArray) oirpm_singleton_pkglist = g_ptr_array_new (); - g_ptr_array_add (oirpm_singleton_pkglist, oirpm_pkg); - if (!rpmostree_context_set_packages (self->ctx, oirpm_singleton_pkglist, cancellable, error)) - return FALSE; - } - - if (!rpmostree_context_download (self->ctx, cancellable, error)) - return FALSE; - - glnx_fd_close int oirpm_fd = -1; - if (!rpmostree_context_consume_package (self->ctx, oirpm_pkg, &oirpm_fd, error)) - return FALSE; - - g_autoptr(RpmOstreeJigdoAssembler) jigdo = rpmostree_jigdo_assembler_new_take_fd (&oirpm_fd, oirpm_pkg, error); - if (!jigdo) - return FALSE; - g_autofree char *checksum = NULL; - g_autoptr(GVariant) commit = NULL; - g_autoptr(GVariant) commit_meta = NULL; - if (!rpmostree_jigdo_assembler_read_meta (jigdo, &checksum, &commit, &commit_meta, - cancellable, error)) - return FALSE; - - if (!g_str_equal (checksum, provided_commit)) - return glnx_throw (error, "Package '%s' commit mismatch; Provides=%s, actual=%s", - dnf_package_get_nevra (oirpm_pkg), provided_commit, checksum); - - g_printerr ("TODO implement GPG verification\n"); - - g_auto(RpmOstreeRepoAutoTransaction) txn = { 0, }; - if (!rpmostree_repo_auto_transaction_start (&txn, self->repo, FALSE, cancellable, error)) - return FALSE; - - if (!ostree_repo_write_commit_detached_metadata (self->repo, checksum, commit_meta, - cancellable, error)) - return FALSE; - /* Mark as partial until we're done */ - if (!ostree_repo_mark_commit_partial (self->repo, checksum, TRUE, error)) - return FALSE; - { g_autofree guint8*csum = NULL; - if (!ostree_repo_write_metadata (self->repo, OSTREE_OBJECT_TYPE_COMMIT, - checksum, commit, &csum, - cancellable, error)) - return FALSE; - } - - if (!rpmostree_jigdo_assembler_write_new_objects (jigdo, self->repo, cancellable, error)) - return FALSE; - - if (!commit_and_print (self, &txn, cancellable, error)) - return FALSE; - - /* And now, process the jigdo set */ - if (!rpmostree_context_set_packages (self->ctx, pkgs_required, cancellable, error)) - return FALSE; - - /* See what packages we need to import, print their size. TODO clarify between - * download/import. - */ - g_autoptr(GHashTable) pkgset_to_import = g_hash_table_new (NULL, NULL); - { g_autoptr(GPtrArray) pkgs_to_import = rpmostree_context_get_packages_to_import (self->ctx); - guint64 dlsize = 0; - for (guint i = 0; i < pkgs_to_import->len; i++) - { - DnfPackage *pkg = pkgs_to_import->pdata[i]; - dlsize += dnf_package_get_size (pkg); - g_hash_table_add (pkgset_to_import, pkg); - } - g_autofree char *dlsize_fmt = g_format_size (dlsize); - g_print ("%u packages to import, download size: %s\n", pkgs_to_import->len, dlsize_fmt); - } - - /* Parse the xattr data in the jigdoRPM */ - g_autoptr(GHashTable) pkg_to_xattrs = g_hash_table_new_full (NULL, NULL, - (GDestroyNotify)g_object_unref, - (GDestroyNotify)g_variant_unref); - - for (guint i = 0; i < pkgs_required->len; i++) - { - DnfPackage *pkg = pkgs_required->pdata[i]; - const gboolean should_import = g_hash_table_contains (pkgset_to_import, pkg); - g_autoptr(GVariant) objid_to_xattrs = NULL; - if (!rpmostree_jigdo_assembler_next_xattrs (jigdo, &objid_to_xattrs, cancellable, error)) - return FALSE; - if (!objid_to_xattrs) - return glnx_throw (error, "missing xattr entry: %s", dnf_package_get_name (pkg)); - if (!should_import) - continue; - g_hash_table_insert (pkg_to_xattrs, g_object_ref (pkg), g_steal_pointer (&objid_to_xattrs)); - } - - /* Start the download and import, using the xattr data from the jigdoRPM */ - if (!rpmostree_context_download (self->ctx, cancellable, error)) - return FALSE; - g_autoptr(GVariant) xattr_table = rpmostree_jigdo_assembler_get_xattr_table (jigdo); - if (!rpmostree_context_import_jigdo (self->ctx, xattr_table, pkg_to_xattrs, - cancellable, error)) - return FALSE; - - /* Last thing is to delete the partial marker, just like - * ostree_repo_pull_with_options(). - */ - if (!ostree_repo_mark_commit_partial (self->repo, checksum, FALSE, error)) + gboolean jigdo_changed; + if (!rpmostree_context_execute_jigdo (self->ctx, &jigdo_changed, cancellable, error)) return FALSE; return TRUE; diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index 1c7a79ea..9427a515 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -149,6 +149,11 @@ gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error); +gboolean rpmostree_context_execute_jigdo (RpmOstreeContext *self, + gboolean *out_changed, + GCancellable *cancellable, + GError **error); + gboolean rpmostree_context_consume_package (RpmOstreeContext *self, DnfPackage *package, diff --git a/src/libpriv/rpmostree-jigdo-client.c b/src/libpriv/rpmostree-jigdo-client.c new file mode 100644 index 00000000..f64377b0 --- /dev/null +++ b/src/libpriv/rpmostree-jigdo-client.c @@ -0,0 +1,263 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2017 Red Hat, Inc. + * + * Licensed under the GNU Lesser General Public License Version 2.1 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + + +/* This file contains the client-side portions of jigdo that are "private" + * implementation detials of RpmOstreeContext. A better model down the line + * might be to have RpmOstreeJigdoContext or so. + */ + +#include "config.h" + +#include +#include +#include "rpmostree-jigdo-assembler.h" +#include "rpmostree-core-private.h" +#include "rpmostree-rpm-util.h" +#include "rpmostree-output.h" +// For the jigdo Requires parsing +#include +#include + +#include +#include + +static DnfPackage * +query_jigdo_pkg (DnfContext *dnfctx, + const char *name, + const char *evr, + GError **error) +{ + hy_autoquery HyQuery query = hy_query_create (dnf_context_get_sack (dnfctx)); + /* See also similar examples of queries in e.g. dnf_context_update() */ + hy_query_filter (query, HY_PKG_NAME, HY_EQ, name); + hy_query_filter (query, HY_PKG_ARCH, HY_NEQ, "src"); + hy_query_filter (query, HY_PKG_EVR, HY_EQ, evr); + g_autoptr(GPtrArray) pkglist = hy_query_run (query); + if (pkglist->len == 0) + return glnx_null_throw (error, "Failed to find package %s-%s", name, evr); + return g_object_ref (pkglist->pdata[0]); +} + +static int +compare_pkgs (gconstpointer ap, + gconstpointer bp) +{ + DnfPackage **a = (gpointer)ap; + DnfPackage **b = (gpointer)bp; + return dnf_package_cmp (*a, *b); +} + +/* Core logic for performing a jigdo assembly client side. The high level flow is: + * + * - Download rpm-md + * - query for jigdoRPM + * - query for jigdoSet (dependencies of above) + * - download and parse jigdoRPM + * - download and import jigdoSet + * - commit all data to ostree + */ +gboolean +rpmostree_context_execute_jigdo (RpmOstreeContext *self, + gboolean *out_changed, + GCancellable *cancellable, + GError **error) +{ + OstreeRepo *repo = self->ostreerepo; + DnfPackage* oirpm_pkg = rpmostree_context_get_jigdo_pkg (self); + const char *provided_commit = rpmostree_context_get_jigdo_checksum (self); + + DnfContext *dnfctx = rpmostree_context_get_dnf (self); + rpmostree_output_message ("jigdoRPM: %s (%s) commit=%s", dnf_package_get_nevra (oirpm_pkg), + dnf_package_get_reponame (oirpm_pkg), provided_commit); + + { OstreeRepoCommitState commitstate; + gboolean has_commit; + if (!ostree_repo_has_object (repo, OSTREE_OBJECT_TYPE_COMMIT, provided_commit, + &has_commit, cancellable, error)) + return FALSE; + if (has_commit) + { + if (!ostree_repo_load_commit (repo, provided_commit, NULL, + &commitstate, error)) + return FALSE; + if (!(commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL)) + { + rpmostree_output_message ("Commit is already written, nothing to do"); + *out_changed = FALSE; + return TRUE; /* 🔚 Early return */ + } + } + } + + g_autoptr(GPtrArray) pkgs_required = g_ptr_array_new_with_free_func (g_object_unref); + + /* Look at the Requires of the jigdoRPM. Note that we don't want to do + * dependency resolution here - that's part of the whole idea, we're doing + * deterministic imaging. + */ + g_autoptr(DnfReldepList) requires = dnf_package_get_requires (oirpm_pkg); + const gint n_requires = dnf_reldep_list_count (requires); + Pool *pool = dnf_sack_get_pool (dnf_context_get_sack (dnfctx)); + for (int i = 0; i < n_requires; i++) + { + DnfReldep *req = dnf_reldep_list_index (requires, i); + Id reqid = dnf_reldep_get_id (req); + if (!ISRELDEP (reqid)) + continue; + Reldep *rdep = GETRELDEP (pool, reqid); + /* This is the core hack; we're searching for Requires that + * have exact '=' versions. This assumes that the rpmbuild + * process won't inject such requirements. + */ + if (!(rdep->flags & REL_EQ)) + continue; + + const char *name = pool_id2str (pool, rdep->name); + const char *evr = pool_id2str (pool, rdep->evr); + + DnfPackage *pkg = query_jigdo_pkg (dnfctx, name, evr, error); + // FIXME: Possibly we shouldn't require a package to be in the repos if we + // already have it imported? This would help support downgrades if the + // repo owner has pruned. + if (!pkg) + return FALSE; + g_ptr_array_add (pkgs_required, g_object_ref (pkg)); + } + g_ptr_array_sort (pkgs_required, compare_pkgs); + + rpmostree_output_message ("Jigdo from %u packages", pkgs_required->len); + + /* For now we first serially download the oirpm, but down the line we can do + * this async. Doing so will require putting more of the jigdo logic into the + * core, so it knows not to import the jigdoRPM. + */ + { g_autoptr(GPtrArray) oirpm_singleton_pkglist = g_ptr_array_new (); + g_ptr_array_add (oirpm_singleton_pkglist, oirpm_pkg); + if (!rpmostree_context_set_packages (self, oirpm_singleton_pkglist, cancellable, error)) + return FALSE; + } + + if (!rpmostree_context_download (self, cancellable, error)) + return FALSE; + + glnx_fd_close int oirpm_fd = -1; + if (!rpmostree_context_consume_package (self, oirpm_pkg, &oirpm_fd, error)) + return FALSE; + + g_autoptr(RpmOstreeJigdoAssembler) jigdo = rpmostree_jigdo_assembler_new_take_fd (&oirpm_fd, oirpm_pkg, error); + if (!jigdo) + return FALSE; + g_autofree char *checksum = NULL; + g_autoptr(GVariant) commit = NULL; + g_autoptr(GVariant) commit_meta = NULL; + if (!rpmostree_jigdo_assembler_read_meta (jigdo, &checksum, &commit, &commit_meta, + cancellable, error)) + return FALSE; + + if (!g_str_equal (checksum, provided_commit)) + return glnx_throw (error, "Package '%s' commit mismatch; Provides=%s, actual=%s", + dnf_package_get_nevra (oirpm_pkg), provided_commit, checksum); + + g_printerr ("TODO implement GPG verification\n"); + + g_auto(RpmOstreeRepoAutoTransaction) txn = { 0, }; + if (!rpmostree_repo_auto_transaction_start (&txn, repo, FALSE, cancellable, error)) + return FALSE; + + if (!ostree_repo_write_commit_detached_metadata (repo, checksum, commit_meta, + cancellable, error)) + return FALSE; + /* Mark as partial until we're done */ + if (!ostree_repo_mark_commit_partial (repo, checksum, TRUE, error)) + return FALSE; + { g_autofree guint8*csum = NULL; + if (!ostree_repo_write_metadata (repo, OSTREE_OBJECT_TYPE_COMMIT, + checksum, commit, &csum, + cancellable, error)) + return FALSE; + } + + if (!rpmostree_jigdo_assembler_write_new_objects (jigdo, repo, cancellable, error)) + return FALSE; + + if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error)) + return FALSE; + txn.initialized = FALSE; + + /* And now, process the jigdo set */ + if (!rpmostree_context_set_packages (self, pkgs_required, cancellable, error)) + return FALSE; + + /* See what packages we need to import, print their size. TODO clarify between + * download/import. + */ + g_autoptr(GHashTable) pkgset_to_import = g_hash_table_new (NULL, NULL); + { g_autoptr(GPtrArray) pkgs_to_import = rpmostree_context_get_packages_to_import (self); + guint64 dlsize = 0; + for (guint i = 0; i < pkgs_to_import->len; i++) + { + DnfPackage *pkg = pkgs_to_import->pdata[i]; + dlsize += dnf_package_get_size (pkg); + g_hash_table_add (pkgset_to_import, pkg); + } + g_autofree char *dlsize_fmt = g_format_size (dlsize); + rpmostree_output_message ("%u packages to import, download size: %s", pkgs_to_import->len, dlsize_fmt); + } + + /* Parse the xattr data in the jigdoRPM */ + g_autoptr(GHashTable) pkg_to_xattrs = g_hash_table_new_full (NULL, NULL, + (GDestroyNotify)g_object_unref, + (GDestroyNotify)g_variant_unref); + + for (guint i = 0; i < pkgs_required->len; i++) + { + DnfPackage *pkg = pkgs_required->pdata[i]; + const gboolean should_import = g_hash_table_contains (pkgset_to_import, pkg); + g_autoptr(GVariant) objid_to_xattrs = NULL; + if (!rpmostree_jigdo_assembler_next_xattrs (jigdo, &objid_to_xattrs, cancellable, error)) + return FALSE; + if (!objid_to_xattrs) + return glnx_throw (error, "missing xattr entry: %s", dnf_package_get_name (pkg)); + if (!should_import) + continue; + g_hash_table_insert (pkg_to_xattrs, g_object_ref (pkg), g_steal_pointer (&objid_to_xattrs)); + } + + /* Start the download and import, using the xattr data from the jigdoRPM */ + if (!rpmostree_context_download (self, cancellable, error)) + return FALSE; + g_autoptr(GVariant) xattr_table = rpmostree_jigdo_assembler_get_xattr_table (jigdo); + if (!rpmostree_context_import_jigdo (self, xattr_table, pkg_to_xattrs, + cancellable, error)) + return FALSE; + + /* Last thing is to delete the partial marker, just like + * ostree_repo_pull_with_options(). + */ + if (!ostree_repo_mark_commit_partial (repo, checksum, FALSE, error)) + return FALSE; + + *out_changed = TRUE; + + return TRUE; +} +