mirror of
https://github.com/ostreedev/ostree.git
synced 2024-12-25 01:34:34 +03:00
core: Change compose to operate purely in-memory
This is *significantly* faster than checking out each branch into the real filesystem, then importing it again.
This commit is contained in:
Colin Walters
parent
add55849ab
commit
e9fd921afe
@ -95,6 +95,31 @@ ostree_mutable_tree_set_contents_checksum (OstreeMutableTree *self,
|
||||
const char *
|
||||
ostree_mutable_tree_get_contents_checksum (OstreeMutableTree *self)
|
||||
{
|
||||
GHashTableIter iter;
|
||||
gpointer key, value;
|
||||
|
||||
if (!self->contents_checksum)
|
||||
return NULL;
|
||||
|
||||
/* Ensure the cache is valid; this implementation is a bit
|
||||
* lame in that we walk the whole tree every time this
|
||||
* getter is called; a better approach would be to invalidate
|
||||
* all of the parents whenever a child is modified.
|
||||
*
|
||||
* However, we only call this function once right now.
|
||||
*/
|
||||
g_hash_table_iter_init (&iter, self->subdirs);
|
||||
while (g_hash_table_iter_next (&iter, &key, &value))
|
||||
{
|
||||
OstreeMutableTree *subdir = value;
|
||||
if (!ostree_mutable_tree_get_contents_checksum (subdir))
|
||||
{
|
||||
g_free (self->contents_checksum);
|
||||
self->contents_checksum = NULL;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return self->contents_checksum;
|
||||
}
|
||||
|
||||
@ -122,6 +147,7 @@ ostree_mutable_tree_replace_file (OstreeMutableTree *self,
|
||||
goto out;
|
||||
}
|
||||
|
||||
ostree_mutable_tree_set_contents_checksum (self, NULL);
|
||||
g_hash_table_replace (self->files,
|
||||
g_strdup (name),
|
||||
g_strdup (checksum));
|
||||
@ -153,6 +179,7 @@ ostree_mutable_tree_ensure_dir (OstreeMutableTree *self,
|
||||
if (!ret_dir)
|
||||
{
|
||||
ret_dir = ostree_mutable_tree_new ();
|
||||
ostree_mutable_tree_set_contents_checksum (self, NULL);
|
||||
g_hash_table_insert (self->subdirs, g_strdup (name), g_object_ref (ret_dir));
|
||||
}
|
||||
|
||||
|
||||
@ -280,6 +280,12 @@ ostree_repo_file_ensure_resolved (OstreeRepoFile *self,
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
const char *
|
||||
ostree_repo_file_get_commit (OstreeRepoFile *self)
|
||||
{
|
||||
return ostree_repo_file_get_root (self)->commit;
|
||||
}
|
||||
|
||||
gboolean
|
||||
ostree_repo_file_get_xattrs (OstreeRepoFile *self,
|
||||
GVariant **out_xattrs,
|
||||
|
||||
@ -75,6 +75,8 @@ gboolean ostree_repo_file_is_tree (OstreeRepoFile *self);
|
||||
|
||||
const char * ostree_repo_file_get_checksum (OstreeRepoFile *self);
|
||||
|
||||
const char * ostree_repo_file_get_commit (OstreeRepoFile *self);
|
||||
|
||||
GFile *ostree_repo_file_nontree_get_local (OstreeRepoFile *self);
|
||||
|
||||
int ostree_repo_file_tree_find_child (OstreeRepoFile *self,
|
||||
|
||||
@ -1380,6 +1380,7 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo *self,
|
||||
GChecksum *child_file_checksum = NULL;
|
||||
GVariant *xattrs = NULL;
|
||||
GInputStream *file_input = NULL;
|
||||
gboolean repo_dir_was_empty = FALSE;
|
||||
|
||||
/* We can only reuse checksums directly if there's no modifier */
|
||||
if (OSTREE_IS_REPO_FILE (dir) && modifier == NULL)
|
||||
@ -1388,7 +1389,9 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo *self,
|
||||
if (repo_dir)
|
||||
{
|
||||
ostree_mutable_tree_set_metadata_checksum (mtree, ostree_repo_file_get_checksum (repo_dir));
|
||||
ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir));
|
||||
repo_dir_was_empty =
|
||||
g_hash_table_size (ostree_mutable_tree_get_files (mtree)) == 0
|
||||
&& g_hash_table_size (ostree_mutable_tree_get_subdirs (mtree)) == 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -1484,6 +1487,9 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo *self,
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (repo_dir && repo_dir_was_empty)
|
||||
ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir));
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
g_clear_object (&dir_enum);
|
||||
@ -1593,10 +1599,12 @@ ostree_repo_stage_mtree (OstreeRepo *self,
|
||||
GVariant *serialized_tree = NULL;
|
||||
GHashTableIter hash_iter;
|
||||
gpointer key, value;
|
||||
const char *existing_checksum;
|
||||
|
||||
if (ostree_mutable_tree_get_contents_checksum (mtree))
|
||||
existing_checksum = ostree_mutable_tree_get_contents_checksum (mtree);
|
||||
if (existing_checksum)
|
||||
{
|
||||
ret_contents_checksum = g_strdup (ostree_mutable_tree_get_contents_checksum (mtree));
|
||||
ret_contents_checksum = g_strdup (existing_checksum);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -27,97 +27,43 @@
|
||||
|
||||
#include <glib/gi18n.h>
|
||||
|
||||
static char *compose_metadata_path;
|
||||
static char *subject;
|
||||
static char *body;
|
||||
static char *branch;
|
||||
|
||||
static GOptionEntry options[] = {
|
||||
{ "out-metadata", 0, 0, G_OPTION_ARG_FILENAME, &compose_metadata_path, "Output a file containing serialized metadata about the compose, in host endianness", "path" },
|
||||
{ "subject", 's', 0, G_OPTION_ARG_STRING, &subject, "One line subject", "subject" },
|
||||
{ "body", 'm', 0, G_OPTION_ARG_STRING, &body, "Full description", "body" },
|
||||
{ "branch", 'b', 0, G_OPTION_ARG_STRING, &branch, "Branch", "branch" },
|
||||
{ NULL }
|
||||
};
|
||||
|
||||
static void
|
||||
rm_rf (GFile *path)
|
||||
{
|
||||
GFileInfo *finfo = NULL;
|
||||
GFileEnumerator *path_enum = NULL;
|
||||
guint32 type;
|
||||
|
||||
finfo = g_file_query_info (path, OSTREE_GIO_FAST_QUERYINFO,
|
||||
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
|
||||
NULL, NULL);
|
||||
if (!finfo)
|
||||
goto out;
|
||||
|
||||
type = g_file_info_get_attribute_uint32 (finfo, "standard::type");
|
||||
if (type == G_FILE_TYPE_DIRECTORY)
|
||||
{
|
||||
path_enum = g_file_enumerate_children (path, OSTREE_GIO_FAST_QUERYINFO,
|
||||
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
|
||||
NULL, NULL);
|
||||
if (!path_enum)
|
||||
goto out;
|
||||
|
||||
|
||||
g_clear_object (&finfo);
|
||||
while ((finfo = g_file_enumerator_next_file (path_enum, NULL, NULL)) != NULL)
|
||||
{
|
||||
GFile *child = g_file_get_child (path, g_file_info_get_attribute_byte_string (finfo, "standard::name"));
|
||||
rm_rf (child);
|
||||
g_clear_object (&child);
|
||||
g_clear_object (&finfo);
|
||||
}
|
||||
}
|
||||
|
||||
(void) unlink (ot_gfile_get_path_cached (path));
|
||||
|
||||
out:
|
||||
g_clear_object (&finfo);
|
||||
g_clear_object (&path_enum);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
compose_branch_on_dir (OstreeRepo *repo,
|
||||
GFile *destination,
|
||||
const char *branch,
|
||||
GVariantBuilder *metadata_builder,
|
||||
GError **error)
|
||||
add_branch (OstreeRepo *repo,
|
||||
OstreeMutableTree *mtree,
|
||||
const char *branch,
|
||||
GVariantBuilder *metadata_builder,
|
||||
GError **error)
|
||||
{
|
||||
char *destpath = NULL;
|
||||
char *branchpath = NULL;
|
||||
GFile *branchf = NULL;
|
||||
GFileEnumerator *enumerator = NULL;
|
||||
gboolean ret = FALSE;
|
||||
char *branchrev = NULL;
|
||||
GFile *branchf = NULL;
|
||||
const char *branch_rev;
|
||||
|
||||
if (!ostree_repo_resolve_rev (repo, branch, FALSE, &branchrev, error))
|
||||
if (!ostree_repo_read_commit (repo, branch, &branchf, NULL, error))
|
||||
goto out;
|
||||
|
||||
branch_rev = ostree_repo_file_get_commit ((OstreeRepoFile*)branchf);
|
||||
|
||||
if (!ostree_repo_stage_directory_to_mtree (repo, branchf, mtree, NULL,
|
||||
NULL, error))
|
||||
goto out;
|
||||
|
||||
destpath = g_strdup (ot_gfile_get_path_cached (destination));
|
||||
if (g_str_has_suffix (destpath, "/"))
|
||||
destpath[strlen (destpath) - 1] = '\0';
|
||||
branchpath = g_strconcat (destpath, "-tmp-checkout-", branchrev, NULL);
|
||||
branchf = ot_gfile_new_for_path (branchpath);
|
||||
|
||||
g_print ("Checking out %s (commit %s)...\n", branch, branchrev);
|
||||
if (!ostree_repo_checkout (repo, OSTREE_REPO_CHECKOUT_MODE_NONE,
|
||||
branchrev, branchf, NULL, error))
|
||||
goto out;
|
||||
g_print ("...done\n");
|
||||
g_print ("Merging over destination...\n");
|
||||
if (!ot_gfile_merge_dirs (destination, branchf, NULL, error))
|
||||
goto out;
|
||||
|
||||
if (metadata_builder)
|
||||
g_variant_builder_add (metadata_builder, "(ss)", branch, branchrev);
|
||||
g_variant_builder_add (metadata_builder, "(ss)", branch, branch_rev);
|
||||
|
||||
ret = TRUE;
|
||||
out:
|
||||
if (branchf)
|
||||
rm_rf (branchf);
|
||||
g_free (destpath);
|
||||
g_clear_object (&enumerator);
|
||||
g_clear_object (&branchf);
|
||||
g_free (branchrev);
|
||||
g_free (branchpath);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -128,17 +74,21 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
|
||||
gboolean ret = FALSE;
|
||||
OstreeRepo *repo = NULL;
|
||||
OstreeCheckout *checkout = NULL;
|
||||
const char *destination;
|
||||
char *parent = NULL;
|
||||
GFile *destf = NULL;
|
||||
gboolean compose_metadata_builder_initialized = FALSE;
|
||||
GVariantBuilder compose_metadata_builder;
|
||||
gboolean commit_metadata_builder_initialized = FALSE;
|
||||
GVariantBuilder commit_metadata_builder;
|
||||
GVariant *commit_metadata = NULL;
|
||||
char *contents_checksum = NULL;
|
||||
char *commit_checksum = NULL;
|
||||
GCancellable *cancellable = NULL;
|
||||
GFile *metadata_f = NULL;
|
||||
OstreeMutableTree *mtree = NULL;
|
||||
int i;
|
||||
|
||||
context = g_option_context_new ("DESTINATION BRANCH1 BRANCH2 ... - Merge multiple commits into a single filesystem tree");
|
||||
context = g_option_context_new ("BRANCH1 BRANCH2 ... - Merge multiple commits into a single commit tree");
|
||||
g_option_context_add_main_entries (context, options, NULL);
|
||||
|
||||
if (!g_option_context_parse (context, &argc, &argv, error))
|
||||
@ -148,50 +98,66 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
|
||||
if (!ostree_repo_check (repo, error))
|
||||
goto out;
|
||||
|
||||
if (argc < 3)
|
||||
if (!branch)
|
||||
{
|
||||
gchar *help = g_option_context_get_help (context, TRUE, NULL);
|
||||
g_printerr ("%s\n", help);
|
||||
g_free (help);
|
||||
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"DESTINATION and at least one COMMIT must be specified");
|
||||
"A branch must be specified with --branch");
|
||||
goto out;
|
||||
}
|
||||
|
||||
destination = argv[1];
|
||||
destf = ot_gfile_new_for_path (destination);
|
||||
|
||||
if (compose_metadata_path)
|
||||
if (!subject)
|
||||
{
|
||||
compose_metadata_builder_initialized = TRUE;
|
||||
g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)"));
|
||||
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||
"A subject must be specified with --subject");
|
||||
goto out;
|
||||
}
|
||||
|
||||
compose_metadata_builder_initialized = TRUE;
|
||||
g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)"));
|
||||
|
||||
if (!ostree_repo_resolve_rev (repo, branch, TRUE, &parent, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_prepare_transaction (repo, cancellable, error))
|
||||
goto out;
|
||||
|
||||
mtree = ostree_mutable_tree_new ();
|
||||
|
||||
for (i = 2; i < argc; i++)
|
||||
for (i = 1; i < argc; i++)
|
||||
{
|
||||
const char *branch = argv[i];
|
||||
const char *src_branch = argv[i];
|
||||
|
||||
if (!compose_branch_on_dir (repo, destf, branch, compose_metadata_builder_initialized ? &compose_metadata_builder : NULL, error))
|
||||
if (!add_branch (repo, mtree, src_branch,
|
||||
compose_metadata_builder_initialized ? &compose_metadata_builder : NULL,
|
||||
error))
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (compose_metadata_path)
|
||||
{
|
||||
commit_metadata_builder_initialized = TRUE;
|
||||
g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}"));
|
||||
commit_metadata_builder_initialized = TRUE;
|
||||
g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}"));
|
||||
|
||||
g_variant_builder_add (&commit_metadata_builder, "{sv}",
|
||||
"ostree-compose", g_variant_builder_end (&compose_metadata_builder));
|
||||
compose_metadata_builder_initialized = FALSE;
|
||||
g_variant_builder_add (&commit_metadata_builder, "{sv}",
|
||||
"ostree-compose", g_variant_builder_end (&compose_metadata_builder));
|
||||
commit_metadata = g_variant_builder_end (&commit_metadata_builder);
|
||||
g_variant_ref_sink (commit_metadata);
|
||||
|
||||
metadata_f = ot_gfile_new_for_path (compose_metadata_path);
|
||||
if (!ostree_repo_stage_mtree (repo, mtree, &contents_checksum, cancellable, error))
|
||||
goto out;
|
||||
|
||||
commit_metadata = g_variant_builder_end (&commit_metadata_builder);
|
||||
if (!ot_util_variant_save (metadata_f, commit_metadata, NULL, error))
|
||||
goto out;
|
||||
}
|
||||
if (!ostree_repo_stage_commit (repo, branch, parent, subject, body, commit_metadata,
|
||||
contents_checksum,
|
||||
ostree_mutable_tree_get_metadata_checksum (mtree),
|
||||
&commit_checksum, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_commit_transaction (repo, cancellable, error))
|
||||
goto out;
|
||||
|
||||
if (!ostree_repo_write_ref (repo, NULL, branch, commit_checksum, error))
|
||||
goto out;
|
||||
|
||||
ret = TRUE;
|
||||
g_print ("%s\n", commit_checksum);
|
||||
out:
|
||||
if (compose_metadata_builder_initialized)
|
||||
g_variant_builder_clear (&compose_metadata_builder);
|
||||
@ -199,10 +165,14 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
|
||||
g_variant_builder_clear (&commit_metadata_builder);
|
||||
if (context)
|
||||
g_option_context_free (context);
|
||||
g_free (parent);
|
||||
g_free (contents_checksum);
|
||||
g_free (commit_checksum);
|
||||
ot_clear_gvariant (&commit_metadata);
|
||||
g_clear_object (&repo);
|
||||
g_clear_object (&checkout);
|
||||
g_clear_object (&destf);
|
||||
g_clear_object (&metadata_f);
|
||||
g_clear_object (&mtree);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
|
||||
set -e
|
||||
|
||||
echo "1..8"
|
||||
echo "1..5"
|
||||
|
||||
. libtest.sh
|
||||
|
||||
@ -58,10 +58,11 @@ $OSTREE commit -b artifact-barapp -s 'Build 42 of barapp'
|
||||
echo 'ok artifacts committed'
|
||||
|
||||
cd "${test_tmpdir}"
|
||||
$OSTREE compose some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
|
||||
echo 'ok compose command'
|
||||
$OSTREE compose -s "compose 1" -b some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
|
||||
echo 'ok compose'
|
||||
|
||||
cd some-compose
|
||||
$OSTREE checkout some-compose some-compose-checkout
|
||||
cd some-compose-checkout
|
||||
assert_file_has_content ./usr/bin/bar 'another ELF file'
|
||||
assert_file_has_content ./usr/share/doc/foo.txt 'some documentation'
|
||||
find | md5sum > ../some-compose-md5
|
||||
@ -70,36 +71,15 @@ assert_file_has_content ../some-compose-md5 9038703e43d2ff2745fb7dd844de65c8
|
||||
echo 'ok compose content'
|
||||
|
||||
cd "${test_tmpdir}"
|
||||
rm -rf some-compose
|
||||
$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
|
||||
echo 'ok compose output metadata'
|
||||
|
||||
cd some-compose
|
||||
$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Initial commit of some-compose'
|
||||
echo 'ok compose commit with metadata'
|
||||
|
||||
$OSTREE show --print-compose some-compose > ${test_tmpdir}/some-compose-contents
|
||||
assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-runtime
|
||||
assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-devel
|
||||
echo 'ok compose verify metadata'
|
||||
|
||||
cd "${test_tmpdir}"
|
||||
rm -rf some-compose some-compose-metadata
|
||||
rm -rf some-compose-checkout some-compose-metadata
|
||||
cd "${test_tmpdir}"/artifact-barapp
|
||||
echo 'updated bar ELF file' > usr/bin/bar
|
||||
$OSTREE commit -b artifact-barapp -s 'Build 43 of barapp'
|
||||
$OSTREE compose -s "compose 2" -b some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
|
||||
echo 'ok compose update commit'
|
||||
|
||||
cd "${test_tmpdir}"
|
||||
$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
|
||||
cd some-compose
|
||||
assert_file_has_content ./usr/bin/bar 'updated bar ELF file'
|
||||
|
||||
echo 'ok updated artifact barapp'
|
||||
$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Updated some-compose'
|
||||
cd ${test_tmpdir}
|
||||
rm -rf some-compose
|
||||
|
||||
$OSTREE checkout some-compose some-compose-checkout
|
||||
cd some-compose-checkout
|
||||
assert_file_has_content ./usr/bin/bar 'updated bar ELF file'
|
||||
echo 'ok updated compose commit'
|
||||
echo 'ok compose update contents'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user