keremet/rpm-ostree
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

treecompose: Remove internal caching, require --repo argument

Browse Source 
It's a lot clearer if the inputs, outputs, and cache state are cleanly
separated.  At least the "lorax" tool relies on a local HTTP cache
instead of keeping around the yum repos - let's do the same.

This commit causes treecompose to require a --repo argument, and it
also gains an optional --proxy argument.
This commit is contained in:
Colin Walters 2014-05-03 06:55:35 -04:00
parent 2d5ed249e0
commit 0ad262b2c4
3 changed files with 80 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -58,11 +58,13 @@ Running rpm-ostree
The core "rpm-ostree" takes as input a "treefile". There is a demo
one in `doc/demo-treefile.json`.
# rpm-ostree create sometreefile.json
# rpm-ostree treecompose --repo=/srv/rpm-ostree/repo --proxy=http://127.0.0.1:8123 sometreefile.json
All this does is use yum to download RPMs from the referenced repos,
and commit the result to the OSTree repository, using the ref named by
`ref`.
`ref`. Note that we've specified a local caching proxy (`polipo` in
this case) - otherwise we you will download the packages for each
treecompose.
You can export `/srv/rpm-ostree/repo` via any static webserver.

2
src/autobuilder/js/tasks/task-treecompose.js
View File

@ -60,7 +60,7 @@ const TaskTreeCompose = new Lang.Class({
let treefilePath = Gio.File.new_for_path('treefile.json');
JsonUtil.writeJsonFileAtomic(treefilePath, treefileData, cancellable);
argv.push.apply(argv, ['treecompose', '--workdir=' + this.workdir.get_path(), treefilePath.get_path()]);
argv.push.apply(argv, ['treecompose', treefilePath.get_path()]);
let productNameUnix = ref.replace(/\//g, '_');
let buildOutputPath = Gio.File.new_for_path('log-' + productNameUnix + '.txt');
print("Running: " + argv.map(GLib.shell_quote).join(' '));

223
src/rpmostree-builtin-treecompose.c
View File

@ -30,9 +30,15 @@
#include "libgsystem.h"
static char *opt_workdir;
static char *opt_cachedir;
static char *opt_proxy;
static char *opt_repo;
static GOptionEntry option_entries[] = {
{ "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "REPO" },
{ "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" },
{ "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
{ "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
{ "proxy", 0, 0, G_OPTION_ARG_STRING, &opt_proxy, "HTTP proxy", "PROXY" },
{ NULL }
};
@ -53,21 +59,6 @@ subprocess_context_print_args (GSSubprocessContext *ctx)
return g_string_free (ret, FALSE);
}
static const char *
object_require_string_member (JsonObject *object,
const char *member_name,
GError **error)
{
const char *ret = json_object_get_string_member (object, member_name);
if (!ret)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"No member '%s' found", member_name);
return NULL;
}
return ret;
}
static gboolean
object_get_optional_string_member (JsonObject *object,
const char *member_name,
@ -95,6 +86,23 @@ object_get_optional_string_member (JsonObject *object,
return ret;
}
static const char *
object_require_string_member (JsonObject *object,
const char *member_name,
GError **error)
{
const char *ret;
if (!object_get_optional_string_member (object, member_name, &ret, error))
return NULL;
if (!ret)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Member '%s' not found", member_name);
return NULL;
}
return ret;
}
static const char *
array_require_string_element (JsonArray *array,
guint i,
@ -257,7 +265,7 @@ void cleanup_keyfile_unref (void *loc)
static gboolean
append_repo_and_cache_opts (JsonObject *treedata,
GFile *cachedir,
GFile *workdir,
GPtrArray *args,
GCancellable *cancellable,
GError **error)
@ -268,11 +276,11 @@ append_repo_and_cache_opts (JsonObject *treedata,
gs_unref_object GFile *yumcache_lookaside = NULL;
gs_unref_object GFile *repos_tmpdir = NULL;
yumcache_lookaside = g_file_resolve_relative_path (cachedir, "yum-cache");
yumcache_lookaside = g_file_resolve_relative_path (workdir, "yum-cache");
if (!gs_file_ensure_directory (yumcache_lookaside, TRUE, cancellable, error))
goto out;
repos_tmpdir = g_file_resolve_relative_path (cachedir, "tmp-repos");
repos_tmpdir = g_file_resolve_relative_path (workdir, "tmp-repos");
if (!gs_shutil_rm_rf (repos_tmpdir, cancellable, error))
goto out;
if (!gs_file_ensure_directory (repos_tmpdir, TRUE, cancellable, error))
@ -283,6 +291,9 @@ append_repo_and_cache_opts (JsonObject *treedata,
g_ptr_array_add (args, g_strdup ("--disablerepo=*"));
if (opt_proxy)
g_ptr_array_add (args, g_strconcat ("--setopt=proxy=", opt_proxy, NULL));
if (json_object_has_member (treedata, "repos"))
enable_repos = json_object_get_array_member (treedata, "repos");
if (enable_repos)
@ -370,7 +381,7 @@ append_repo_and_cache_opts (JsonObject *treedata,
static YumContext *
yum_context_new (JsonObject *treedata,
GFile *yumroot,
GFile *cachedir,
GFile *workdir,
GCancellable *cancellable,
GError **error)
{
@ -384,7 +395,7 @@ yum_context_new (JsonObject *treedata,
g_ptr_array_add (yum_argv, g_strdup ("yum"));
g_ptr_array_add (yum_argv, g_strdup ("-y"));
if (!append_repo_and_cache_opts (treedata, cachedir, yum_argv,
if (!append_repo_and_cache_opts (treedata, workdir, yum_argv,
cancellable, error))
goto out;
@ -472,7 +483,7 @@ yum_context_command (YumContext *yumctx,
static gboolean
yuminstall (JsonObject *treedata,
GFile *yumroot,
GFile *cachedir,
GFile *workdir,
char **packages,
GCancellable *cancellable,
GError **error)
@ -481,7 +492,7 @@ yuminstall (JsonObject *treedata,
char **strviter;
YumContext *yumctx;
yumctx = yum_context_new (treedata, yumroot, cachedir, cancellable, error);
yumctx = yum_context_new (treedata, yumroot, workdir, cancellable, error);
if (!yumctx)
goto out;
@ -531,6 +542,7 @@ rpmostree_builtin_treecompose (int argc,
JsonArray *units = NULL;
guint len;
gs_free char *ref_unix = NULL;
gs_unref_object GFile *workdir = NULL;
gs_unref_object GFile *cachedir = NULL;
gs_unref_object GFile *yumroot = NULL;
gs_unref_object GFile *targetroot = NULL;
@ -539,7 +551,9 @@ rpmostree_builtin_treecompose (int argc,
gs_unref_ptrarray GPtrArray *bootstrap_packages = NULL;
gs_unref_ptrarray GPtrArray *packages = NULL;
gs_unref_object GFile *treefile_path = NULL;
gs_unref_object GFile *repo_path = NULL;
gs_unref_object JsonParser *treefile_parser = NULL;
gboolean workdir_is_tmp = FALSE;
g_option_context_add_main_entries (context, option_entries, NULL);
@ -554,9 +568,32 @@ rpmostree_builtin_treecompose (int argc,
goto out;
}
if (!opt_repo)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"--repo must be specified");
goto out;
}
repo_path = g_file_new_for_path (opt_repo);
repo = ostree_repo_new (repo_path);
if (!ostree_repo_open (repo, cancellable, error))
goto out;
treefile_path = g_file_new_for_path (argv[1]);
if (opt_workdir && chdir (opt_workdir) != 0)
if (opt_workdir)
{
workdir = g_file_new_for_path (opt_workdir);
}
else
{
gs_free char *tmpd = g_mkdtemp (g_strdup ("/var/tmp/rpm-ostree.XXXXXX"));
workdir = g_file_new_for_path (tmpd);
workdir_is_tmp = TRUE;
}
if (chdir (gs_file_get_path_cached (workdir)) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to chdir to '%s': %s",
@ -579,14 +616,10 @@ rpmostree_builtin_treecompose (int argc,
}
treefile = json_node_get_object (treefile_root);
cachedir = g_file_new_for_path ("cache");
if (!gs_file_ensure_directory (cachedir, TRUE, cancellable, error))
goto out;
yumroot = g_file_get_child (cachedir, "yum");
yumroot = g_file_get_child (workdir, "rootfs.tmp");
if (!gs_shutil_rm_rf (yumroot, cancellable, error))
goto out;
targetroot = g_file_resolve_relative_path (cachedir, "rootfs");
targetroot = g_file_get_child (workdir, "rootfs");
ref = object_require_string_member (treefile, "ref", error);
if (!ref)
@ -607,113 +640,10 @@ rpmostree_builtin_treecompose (int argc,
{
gs_free char *cached_packageset_name =
g_strconcat ("packageset-", ref_unix, ".txt", NULL);
gs_unref_object GFile *rpmtextlist_path =
g_file_resolve_relative_path (cachedir, cached_packageset_name);
gs_free char *cached_packageset_name_new =
g_strconcat (cached_packageset_name, ".new", NULL);
gs_unref_object GFile *rpmtextlist_path_new =
g_file_resolve_relative_path (cachedir, cached_packageset_name_new);
GPtrArray *repoquery_argv = g_ptr_array_new_with_free_func (g_free);
gs_unref_object GSSubprocessContext *repoquery_proc_ctx = NULL;
gs_unref_object GSSubprocess *repoquery_proc = NULL;
guint i;
GString *repoquery_arg_string = NULL;
gboolean first = TRUE;
g_ptr_array_add (repoquery_argv, g_strdup (PKGLIBDIR "/repoquery-sorted"));
if (!append_repo_and_cache_opts (treefile, cachedir, repoquery_argv,
cancellable, error))
goto out;
g_ptr_array_add (repoquery_argv, g_strdup ("--recursive"));
g_ptr_array_add (repoquery_argv, g_strdup ("--requires"));
g_ptr_array_add (repoquery_argv, g_strdup ("--resolve"));
repoquery_arg_string = g_string_new ("");
for (i = 0; i < packages->len; i++)
{
const char *package = packages->pdata[i];
if (!package)
continue;
g_ptr_array_add (repoquery_argv, g_strdup (package));
if (first)
first = FALSE;
else
g_string_append_c (repoquery_arg_string, ' ');
g_string_append (repoquery_arg_string, package);
}
g_print ("Running repoquery: %s\n", repoquery_arg_string->str);
g_string_free (repoquery_arg_string, TRUE);
g_ptr_array_add (repoquery_argv, NULL);
repoquery_proc_ctx = gs_subprocess_context_new ((char**)repoquery_argv->pdata);
gs_subprocess_context_set_stdout_file_path (repoquery_proc_ctx, gs_file_get_path_cached (rpmtextlist_path_new));
gs_subprocess_context_set_stderr_disposition (repoquery_proc_ctx, GS_SUBPROCESS_STREAM_DISPOSITION_INHERIT);
g_print ("Resolving dependencies...\n");
repoquery_proc = gs_subprocess_new (repoquery_proc_ctx, cancellable, error);
if (!repoquery_proc)
goto out;
if (!gs_subprocess_wait_sync_check (repoquery_proc, cancellable, error))
goto out;
if (g_file_query_exists (rpmtextlist_path, NULL))
{
GError *temp_error = NULL;
gs_unref_object GSSubprocess *diff_proc = NULL;
gboolean differs = FALSE;
g_print ("Comparing diff of previous tree\n");
diff_proc =
gs_subprocess_new_simple_argl (GS_SUBPROCESS_STREAM_DISPOSITION_INHERIT,
GS_SUBPROCESS_STREAM_DISPOSITION_INHERIT,
cancellable,
error,
"diff", "-u",
gs_file_get_path_cached (rpmtextlist_path),
gs_file_get_path_cached (rpmtextlist_path_new),
NULL);
if (!gs_subprocess_wait_sync_check (diff_proc, cancellable, &temp_error))
{
int ecode;
if (temp_error->domain != G_SPAWN_EXIT_ERROR)
{
g_propagate_error (error, temp_error);
goto out;
}
ecode = temp_error->code;
g_assert (ecode != 0);
if (ecode == 1)
differs = TRUE;
else
{
g_propagate_error (error, temp_error);
goto out;
}
}
if (!differs)
{
g_print ("No changes in package set\n");
if (!gs_file_unlink (rpmtextlist_path_new, cancellable, error))
goto out;
goto out;
}
}
else
{
g_print ("No previous diff file found at '%s'\n",
gs_file_get_path_cached (rpmtextlist_path));
}
/* Ensure we have enough to modify NSS */
if (!yuminstall (treefile, yumroot, cachedir,
if (!yuminstall (treefile, yumroot, workdir,
(char**)bootstrap_packages->pdata,
cancellable, error))
goto out;
@ -741,7 +671,7 @@ rpmostree_builtin_treecompose (int argc,
}
{
if (!yuminstall (treefile, yumroot, cachedir,
if (!yuminstall (treefile, yumroot, workdir,
(char**)packages->pdata,
cancellable, error))
goto out;
@ -752,6 +682,13 @@ rpmostree_builtin_treecompose (int argc,
if (g_strcmp0 (g_getenv ("RPM_OSTREE_BREAK"), "post-yum") == 0)
goto out;
/* Clean cached packages now */
{
gs_unref_object GFile *yumcache_lookaside = g_file_resolve_relative_path (workdir, "yum-cache");
if (!gs_shutil_rm_rf (yumcache_lookaside, cancellable, error))
goto out;
}
if (!rpmostree_postprocess (yumroot, cancellable, error))
goto out;
@ -842,15 +779,6 @@ rpmostree_builtin_treecompose (int argc,
goto out;
}
{
gs_unref_object GFile *repo_path = g_file_new_for_path ("repo");
repo = ostree_repo_new (repo_path);
}
if (!ostree_repo_open (repo, cancellable, error))
goto out;
{
const char *gpgkey;
if (!object_get_optional_string_member (treefile, "gpg_key", &gpgkey, error))
@ -861,14 +789,13 @@ rpmostree_builtin_treecompose (int argc,
cancellable, error))
goto out;
}
if (!gs_file_rename (rpmtextlist_path_new, rpmtextlist_path,
cancellable, error))
goto out;
}
g_print ("Complete\n");
if (workdir_is_tmp)
(void) gs_shutil_rm_rf (workdir, cancellable, NULL);
out:
return ret;
}