bin/compose: Expose phases as [install, postprocess, commit] cmds

Right now `rpm-ostree compose tree` is very prescriptive about how things work.
Trying to add anything that isn't an RPM is absolutely fighting the system. Our
postprocessing system *enforces* no network access (good for reproducibilty, but
still prescriptive).

There's really a logical split between three phases:

 - install: "build a rootfs that installs packages"
 - postprocess: "run magical ostree postprocessing like kernel"
 - commit: "commit result to ostree"

So there are two high level flows I'd like to enable here. First is to allow
people to do *arbitrary* postprocessing between `install` and `commit`. For
example, run Ansible and change `/etc`. This path basically is like what we have
today with `postprocess-script.sh`, except the builder can do anything they want
with network access enabled.

Going much farther, this helps us support a "build with Dockerfile" style flow.
We can then provide tooling to extract the container image, and combine
`postprocess` and `commit`.

Or completely the other way - if for example someone wants to use `rpm-ostree
compose install`, they could tar up the result as a Docker/OCI image. That's now
easier; an advantage of this flow over e.g. `yum --installroot` is the "change
detection" code we have.

Related issues/PRs:

 - https://github.com/projectatomic/rpm-ostree/pull/96
 - https://github.com/projectatomic/rpm-ostree/issues/471

One disadvantage of this approach right now is that if one *does* go for
the split approach, we lose the "input hash" metadata for example.  And
down the line, I'd like to add even more metadata, like the input rpm repos,
which could also be rendered on the client side.

But, I think we can address that later by e.g. caching the metadata in a file in
the install root and picking it back up or something.

Closes: #1039
Approved by: jlebon
This commit is contained in:
Colin Walters 2017-10-12 14:56:08 -04:00 committed by Atomic Bot
parent 1f31cbb99a
commit 63af4bbdda
8 changed files with 324 additions and 51 deletions

View File

@ -106,10 +106,12 @@ Boston, MA 02111-1307, USA.
<listitem>
<para>
Entrypoint for tree composition; most typically used on
servers to prepare trees for replication by client systems.
Currently has two subcommands, <literal>tree</literal> and
<literal>sign</literal>.
Entrypoint for tree composition; most typically used on servers to
prepare trees for replication by client systems. The
<literal>tree</literal> subcommand processes a treefile, installs
packages, and commits the result to an OSTree repository. There are
also split commands <literal>install</literal>,
<literal>postprocess</literal>, and <literal>commit</literal>.
</para>
</listitem>
</varlistentry>

View File

@ -30,8 +30,17 @@
static RpmOstreeCommand compose_subcommands[] = {
{ "tree", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT,
"Install packages and commit the result to an OSTree repository",
"Process a \"treefile\"; install packages and commit the result to an OSTree repository",
rpmostree_compose_builtin_tree },
{ "install", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT,
"Install packages into a target path",
rpmostree_compose_builtin_install },
{ "postprocess", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT,
"Perform final postprocessing on an installation root",
rpmostree_compose_builtin_postprocess },
{ "commit", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT,
"Commit a target path to an OSTree repository",
rpmostree_compose_builtin_commit },
{ NULL, 0, NULL, NULL }
};

View File

@ -58,20 +58,29 @@ static gboolean opt_dry_run;
static gboolean opt_print_only;
static char *opt_write_commitid_to;
static GOptionEntry option_entries[] = {
{ "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Append given key and value (in string format) to metadata", "KEY=VALUE" },
{ "add-metadata-from-json", 0, 0, G_OPTION_ARG_STRING, &opt_metadata_json, "Parse the given JSON file as object, convert to GVariant, append to OSTree commit", "JSON" },
{ "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" },
{ "workdir-tmpfs", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL },
{ "output-repodata-dir", 0, 0, G_OPTION_ARG_STRING, &opt_output_repodata_dir, "Save downloaded repodata in DIR", "DIR" },
{ "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
static GOptionEntry install_option_entries[] = {
{ "force-nocache", 0, 0, G_OPTION_ARG_NONE, &opt_force_nocache, "Always create a new OSTree commit, even if nothing appears to have changed", NULL },
{ "cache-only", 0, 0, G_OPTION_ARG_NONE, &opt_cache_only, "Assume cache is present, do not attempt to update it", NULL },
{ "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
{ "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
{ "proxy", 0, 0, G_OPTION_ARG_STRING, &opt_proxy, "HTTP proxy", "PROXY" },
{ "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" },
{ "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Just print the transaction and exit", NULL },
{ "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
{ "output-repodata-dir", 0, 0, G_OPTION_ARG_STRING, &opt_output_repodata_dir, "Save downloaded repodata in DIR", "DIR" },
{ "print-only", 0, 0, G_OPTION_ARG_NONE, &opt_print_only, "Just expand any includes and print treefile", NULL },
{ "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" },
{ "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" },
{ "workdir-tmpfs", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL },
{ NULL }
};
static GOptionEntry postprocess_option_entries[] = {
{ NULL }
};
static GOptionEntry commit_option_entries[] = {
{ "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Append given key and value (in string format) to metadata", "KEY=VALUE" },
{ "add-metadata-from-json", 0, 0, G_OPTION_ARG_STRING, &opt_metadata_json, "Parse the given JSON file as object, convert to GVariant, append to OSTree commit", "JSON" },
{ "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
{ "write-commitid-to", 0, 0, G_OPTION_ARG_STRING, &opt_write_commitid_to, "File to write the composed commitid to instead of updating the ref", "FILE" },
{ NULL }
};
@ -757,23 +766,34 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr,
cancellable, error))
return FALSE;
g_autoptr(GHashTable) varsubsts = rpmostree_dnfcontext_get_varsubsts (rpmostree_context_get_hif (self->corectx));
const char *input_ref = _rpmostree_jsonutil_object_require_string_member (self->treefile, "ref", error);
if (!input_ref)
return FALSE;
self->ref = _rpmostree_varsubst_string (input_ref, varsubsts, error);
if (!self->ref)
return FALSE;
*out_context = g_steal_pointer (&self);
return TRUE;
}
static gboolean
impl_compose_tree (const char *treefile_pathstr,
impl_install_tree (RpmOstreeTreeComposeContext *self,
gboolean *out_changed,
GCancellable *cancellable,
GError **error)
{
g_autoptr(RpmOstreeTreeComposeContext) self = NULL;
if (!rpm_ostree_compose_context_new (treefile_pathstr, &self, cancellable, error))
return FALSE;
/* FIXME - is this still necessary? */
if (fchdir (self->workdir_dfd) != 0)
return glnx_throw_errno_prefix (error, "fchdir");
/* Set this early here, so we only have to set it one more time in the
* complete exit path too.
*/
*out_changed = FALSE;
if (opt_print_only)
{
glnx_unref_object JsonGenerator *generator = json_generator_new ();
@ -787,16 +807,6 @@ impl_compose_tree (const char *treefile_pathstr,
return TRUE;
}
g_autoptr(GHashTable) varsubsts = rpmostree_dnfcontext_get_varsubsts (rpmostree_context_get_hif (self->corectx));
{ const char *input_ref = _rpmostree_jsonutil_object_require_string_member (self->treefile, "ref", error);
if (!input_ref)
return FALSE;
self->ref = _rpmostree_varsubst_string (input_ref, varsubsts, error);
if (!self->ref)
return FALSE;
}
/* Read the previous commit */
{ g_autoptr(GError) temp_error = NULL;
if (!ostree_repo_read_commit (self->repo, self->ref, &self->previous_root, &self->previous_checksum,
@ -984,6 +994,15 @@ impl_compose_tree (const char *treefile_pathstr,
g_hash_table_replace (self->metadata, g_strdup ("rpmostree.inputhash"),
g_variant_ref_sink (g_variant_new_string (new_inputhash)));
*out_changed = TRUE;
return TRUE;
}
static gboolean
impl_commit_tree (RpmOstreeTreeComposeContext *self,
GCancellable *cancellable,
GError **error)
{
const char *gpgkey = NULL;
if (!_rpmostree_jsonutil_object_get_optional_string_member (self->treefile, "gpg_key", &gpgkey, error))
return FALSE;
@ -1012,6 +1031,13 @@ impl_compose_tree (const char *treefile_pathstr,
}
}
if (!rpmostree_rootfs_postprocess_common (self->rootfs_dfd, cancellable, error))
return EXIT_FAILURE;
if (!rpmostree_postprocess_final (self->rootfs_dfd, self->treefile,
cancellable, error))
return EXIT_FAILURE;
/* The penultimate step, just basically `ostree commit` */
g_autofree char *new_revision = NULL;
if (!rpmostree_commit (self->rootfs_dfd, self->repo, self->ref, opt_write_commitid_to,
metadata, gpgkey, selinux, NULL,
@ -1021,22 +1047,127 @@ impl_compose_tree (const char *treefile_pathstr,
g_print ("%s => %s\n", self->ref, new_revision);
if (!process_touch_if_changed (error))
return FALSE;
return TRUE;
}
int
rpmostree_compose_builtin_tree (int argc,
char **argv,
RpmOstreeCommandInvocation *invocation,
GCancellable *cancellable,
GError **error)
rpmostree_compose_builtin_install (int argc,
char **argv,
RpmOstreeCommandInvocation *invocation,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE");
g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE DESTDIR");
if (!rpmostree_option_context_parse (context,
option_entries,
install_option_entries,
&argc, &argv,
invocation,
cancellable,
NULL, NULL, NULL, NULL,
error))
return EXIT_FAILURE;
if (argc != 3)
{
rpmostree_usage_error (context, "TREEFILE and DESTDIR required", error);
return EXIT_FAILURE;
}
if (!opt_repo)
{
rpmostree_usage_error (context, "--repo must be specified", error);
return EXIT_FAILURE;
}
if (opt_workdir)
{
rpmostree_usage_error (context, "--workdir is ignored with install-root", error);
return EXIT_FAILURE;
}
const char *treefile_path = argv[1];
/* Destination is turned into workdir */
const char *destdir = argv[2];
opt_workdir = g_strdup (destdir);
g_autoptr(RpmOstreeTreeComposeContext) self = NULL;
if (!rpm_ostree_compose_context_new (treefile_path, &self, cancellable, error))
return FALSE;
gboolean changed;
if (!impl_install_tree (self, &changed, cancellable, error))
return EXIT_FAILURE;
/* Keep the dir around */
g_print ("rootfs: %s/rootfs\n", self->workdir_tmp.path);
glnx_tmpdir_unset (&self->workdir_tmp);
return EXIT_SUCCESS;
}
int
rpmostree_compose_builtin_postprocess (int argc,
char **argv,
RpmOstreeCommandInvocation *invocation,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GOptionContext) context = g_option_context_new ("postprocess ROOTFS [TREEFILE]");
if (!rpmostree_option_context_parse (context,
postprocess_option_entries,
&argc, &argv,
invocation,
cancellable,
NULL, NULL, NULL, NULL,
error))
return EXIT_FAILURE;
if (argc < 2 || argc > 3)
{
rpmostree_usage_error (context, "ROOTFS must be specified", error);
return EXIT_FAILURE;
}
const char *rootfs_path = argv[1];
/* Here we *optionally* process a treefile; some things like `tmp-is-dir` and
* `boot_location` are configurable and relevant here, but a lot of users
* will also probably be OK with the defaults, and part of the idea here is
* to avoid at least some of the use cases requiring a treefile.
*/
const char *treefile_path = argc > 2 ? argv[2] : NULL;
glnx_unref_object JsonParser *treefile_parser = NULL;
JsonObject *treefile = NULL; /* Owned by parser */
if (treefile_path)
{
treefile_parser = json_parser_new ();
if (!json_parser_load_from_file (treefile_parser, treefile_path, error))
return EXIT_FAILURE;
JsonNode *treefile_rootval = json_parser_get_root (treefile_parser);
if (!JSON_NODE_HOLDS_OBJECT (treefile_rootval))
return glnx_throw (error, "Treefile root is not an object"), EXIT_FAILURE;
treefile = json_node_get_object (treefile_rootval);
}
glnx_fd_close int rootfs_dfd = -1;
if (!glnx_opendirat (AT_FDCWD, rootfs_path, TRUE, &rootfs_dfd, error))
return EXIT_FAILURE;
if (!rpmostree_rootfs_postprocess_common (rootfs_dfd, cancellable, error))
return EXIT_FAILURE;
if (!rpmostree_postprocess_final (rootfs_dfd, treefile,
cancellable, error))
return EXIT_FAILURE;
return EXIT_SUCCESS;
}
int
rpmostree_compose_builtin_commit (int argc,
char **argv,
RpmOstreeCommandInvocation *invocation,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE ROOTFS");
if (!rpmostree_option_context_parse (context,
commit_option_entries,
&argc, &argv,
invocation,
cancellable,
@ -1056,8 +1187,68 @@ rpmostree_compose_builtin_tree (int argc,
return EXIT_FAILURE;
}
if (!impl_compose_tree (argv[1], cancellable, error))
const char *treefile_path = argv[1];
const char *rootfs_path = argv[2];
g_autoptr(RpmOstreeTreeComposeContext) self = NULL;
if (!rpm_ostree_compose_context_new (treefile_path, &self, cancellable, error))
return EXIT_FAILURE;
if (!glnx_opendirat (AT_FDCWD, rootfs_path, TRUE, &self->rootfs_dfd, error))
return EXIT_FAILURE;
if (!impl_commit_tree (self, cancellable, error))
return EXIT_FAILURE;
return EXIT_SUCCESS;
}
int
rpmostree_compose_builtin_tree (int argc,
char **argv,
RpmOstreeCommandInvocation *invocation,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE");
g_option_context_add_main_entries (context, install_option_entries, NULL);
g_option_context_add_main_entries (context, postprocess_option_entries, NULL);
if (!rpmostree_option_context_parse (context,
commit_option_entries,
&argc, &argv,
invocation,
cancellable,
NULL, NULL, NULL, NULL,
error))
return EXIT_FAILURE;
if (argc < 2)
{
rpmostree_usage_error (context, "TREEFILE must be specified", error);
return EXIT_FAILURE;
}
if (!opt_repo)
{
rpmostree_usage_error (context, "--repo must be specified", error);
return EXIT_FAILURE;
}
const char *treefile_path = argv[1];
g_autoptr(RpmOstreeTreeComposeContext) self = NULL;
if (!rpm_ostree_compose_context_new (treefile_path, &self, cancellable, error))
return EXIT_FAILURE;
gboolean changed;
if (!impl_install_tree (self, &changed, cancellable, error))
return EXIT_FAILURE;
if (changed)
{
/* Do the ostree commit */
if (!impl_commit_tree (self, cancellable, error))
return EXIT_FAILURE;
/* Finally process the --touch-if-changed option */
if (!process_touch_if_changed (error))
return FALSE;
}
return EXIT_SUCCESS;
}

View File

@ -27,6 +27,9 @@
G_BEGIN_DECLS
gboolean rpmostree_compose_builtin_tree (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
gboolean rpmostree_compose_builtin_install (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
gboolean rpmostree_compose_builtin_postprocess (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
gboolean rpmostree_compose_builtin_commit (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
G_END_DECLS

View File

@ -856,12 +856,24 @@ postprocess_selinux_policy_store_location (int rootfs_dfd,
/* All "final" processing; things that are really required to use
* rpm-ostree on the target host.
*/
static gboolean
postprocess_final (int rootfs_dfd,
JsonObject *treefile,
GCancellable *cancellable,
GError **error)
gboolean
rpmostree_postprocess_final (int rootfs_dfd,
JsonObject *treefile,
GCancellable *cancellable,
GError **error)
{
GLNX_AUTO_PREFIX_ERROR ("Finalizing rootfs", error);
/* Use installation of the tmpfiles integration as an "idempotence" marker to
* avoid doing postprocessing twice, which can happen when mixing `compose
* postprocess-root` with `compose commit`.
*/
const char tmpfiles_integration_path[] = "usr/lib/tmpfiles.d/rpm-ostree-0-integration.conf";
if (!glnx_fstatat_allow_noent (rootfs_dfd, tmpfiles_integration_path, NULL, AT_SYMLINK_NOFOLLOW, error))
return FALSE;
if (errno == 0)
return TRUE;
gboolean selinux = TRUE;
if (!_rpmostree_jsonutil_object_get_optional_boolean_member (treefile,
"selinux",
@ -926,7 +938,7 @@ postprocess_final (int rootfs_dfd,
return FALSE;
if (!glnx_file_copy_at (pkglibdir_dfd, "rpm-ostree-0-integration.conf", NULL,
rootfs_dfd, "usr/lib/tmpfiles.d/rpm-ostree-0-integration.conf",
rootfs_dfd, tmpfiles_integration_path,
GLNX_FILE_COPY_NOXATTRS, /* Don't take selinux label */
cancellable, error))
return FALSE;
@ -1664,10 +1676,6 @@ rpmostree_prepare_rootfs_for_commit (int src_rootfs_dfd,
}
}
/* And call into the final postprocessing function */
if (!postprocess_final (target_rootfs_dfd, treefile,
cancellable, error))
return glnx_prefix_error (error, "Finalizing rootfs");
return TRUE;
}

View File

@ -65,6 +65,12 @@ rpmostree_prepare_rootfs_for_commit (int src_rootfs_dfd,
GCancellable *cancellable,
GError **error);
gboolean
rpmostree_postprocess_final (int rootfs_dfd,
JsonObject *treefile,
GCancellable *cancellable,
GError **error);
gboolean
rpmostree_commit (int rootfs_dfd,
OstreeRepo *repo,

View File

@ -33,8 +33,9 @@ prepare_compose_test() {
export treeref=fedora/stable/x86_64/${name}
}
compose_base_argv="--repo=${repobuild} --cache-only --cachedir=${test_compose_datadir}/cache"
runcompose() {
rpm-ostree compose --repo=${repobuild} tree --cache-only --cachedir=${test_compose_datadir}/cache ${treefile} "$@"
rpm-ostree compose tree ${compose_base_argv} ${treefile} "$@"
ostree --repo=${repo} pull-local ${repobuild}
}

View File

@ -0,0 +1,53 @@
#!/bin/bash
set -xeuo pipefail
dn=$(cd $(dirname $0) && pwd)
. ${dn}/libcomposetest.sh
prepare_compose_test "installroot"
# This is used to test postprocessing with treefile vs not
pysetjsonmember "boot_location" '"new"'
instroot_tmp=$(mktemp -d /var/tmp/rpm-ostree-instroot.XXXXXX)
rpm-ostree compose install ${compose_base_argv} ${treefile} ${instroot_tmp}
instroot=${instroot_tmp}/rootfs
assert_not_has_dir ${instroot}/usr/lib/ostree-boot
assert_not_has_dir ${instroot}/etc
test -L ${instroot}/home
assert_has_dir ${instroot}/usr/etc
# Clone the root - we'll test direct commit, as well as postprocess with
# and without treefile.
mv ${instroot}{,-postprocess}
cp -al ${instroot}{-postprocess,-directcommit}
cp -al ${instroot}{-postprocess,-postprocess-treefile}
integrationconf=usr/lib/tmpfiles.d/rpm-ostree-0-integration.conf
assert_not_has_file ${instroot}-postprocess/${integrationconf}
rpm-ostree compose postprocess ${instroot}-postprocess
assert_has_file ${instroot}-postprocess/${integrationconf}
# Without treefile, kernels end up in "both" mode
ls ${instroot}-postprocess/boot > ls.txt
assert_file_has_content ls.txt '^vmlinuz-'
rm -f ls.txt
ostree --repo=${repobuild} commit -b test-directcommit --selinux-policy ${instroot}-postprocess --tree=dir=${instroot}-postprocess
echo "ok postprocess + direct commit"
rpm-ostree compose postprocess ${instroot}-postprocess-treefile ${treefile}
assert_has_file ${instroot}-postprocess-treefile/${integrationconf}
# with treefile, no kernels in /boot
ls ${instroot}-postprocess-treefile/boot > ls.txt
assert_not_file_has_content ls.txt '^vmlinuz-'
rm -f ls.txt
echo "ok postprocess with treefile"
testdate=$(date)
echo "${testdate}" > ${instroot}-directcommit/usr/share/rpm-ostree-composetest-split.txt
assert_not_has_file ${instroot}-directcommit/${integrationconf}
rpm-ostree compose commit --repo=${repobuild} ${treefile} ${instroot}-directcommit
ostree --repo=${repobuild} ls ${treeref} /usr/bin/bash
ostree --repo=${repobuild} cat ${treeref} /usr/share/rpm-ostree-composetest-split.txt >out.txt
assert_file_has_content_literal out.txt "${testdate}"
ostree --repo=${repobuild} cat ${treeref} /${integrationconf}
echo "ok installroot"