shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-17 14:50:35 +03:00
Code Issues Projects Releases Wiki Activity

core: Store "pending" objects explicitly, fix pull to use them

Browse Source 
Before we were creating randomly-named temporary files in repo/tmp
when downloading via pull, but that means if the download process is
interrupted, we have to redownload everything again.

Let's still keep the concept of a "transaction" where files are
stored in the repository as atomically as possible (i.e. we
do a bunch of rename() calls), but now we also have an explicit
"tmp/pending/objects" directory that contains named objects.

This allows us to then skip redownloading things that are pending.
This commit is contained in:
Colin Walters 2012-02-23 18:19:00 -05:00
parent 802959f398
commit b2ad353b42
3 changed files with 302 additions and 228 deletions

250
src/libostree/ostree-repo.c

@ -56,6 +56,7 @@ typedef struct _OstreeRepoPrivate OstreeRepoPrivate;
struct _OstreeRepoPrivate { struct _OstreeRepoPrivate {
GFile *repodir; GFile *repodir;
GFile *tmp_dir; GFile *tmp_dir;
GFile *pending_dir;
GFile *local_heads_dir; GFile *local_heads_dir;
GFile *remote_heads_dir; GFile *remote_heads_dir;
GFile *objects_dir; GFile *objects_dir;
@ -63,12 +64,11 @@ struct _OstreeRepoPrivate {
gboolean inited; gboolean inited;
gboolean in_transaction; gboolean in_transaction;
GFile *transaction_dir;
GKeyFile *config; GKeyFile *config;
OstreeRepoMode mode; OstreeRepoMode mode;
GHashTable *pending_transaction_tmpfiles; GHashTable *pending_transaction;
}; };
static void static void
@ -79,12 +79,12 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&priv->repodir); g_clear_object (&priv->repodir);
g_clear_object (&priv->tmp_dir); g_clear_object (&priv->tmp_dir);
g_clear_object (&priv->pending_dir);
g_clear_object (&priv->local_heads_dir); g_clear_object (&priv->local_heads_dir);
g_clear_object (&priv->remote_heads_dir); g_clear_object (&priv->remote_heads_dir);
g_clear_object (&priv->objects_dir); g_clear_object (&priv->objects_dir);
g_clear_object (&priv->config_file); g_clear_object (&priv->config_file);
g_clear_object (&priv->transaction_dir); g_hash_table_destroy (priv->pending_transaction);
g_hash_table_destroy (priv->pending_transaction_tmpfiles);
if (priv->config) if (priv->config)
g_key_file_free (priv->config); g_key_file_free (priv->config);
@ -149,6 +149,7 @@ ostree_repo_constructor (GType gtype,
g_assert (priv->repodir != NULL); g_assert (priv->repodir != NULL);
priv->tmp_dir = g_file_resolve_relative_path (priv->repodir, "tmp"); priv->tmp_dir = g_file_resolve_relative_path (priv->repodir, "tmp");
priv->pending_dir = g_file_resolve_relative_path (priv->repodir, "tmp/pending");
priv->local_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/heads"); priv->local_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/heads");
priv->remote_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/remotes"); priv->remote_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/remotes");
@ -184,9 +185,9 @@ ostree_repo_init (OstreeRepo *self)
{ {
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
priv->pending_transaction_tmpfiles = g_hash_table_new_full (g_str_hash, g_str_equal, priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal,
g_free, g_free,
g_free); NULL);
} }
OstreeRepo* OstreeRepo*
@ -600,6 +601,9 @@ ostree_repo_check (OstreeRepo *self, GError **error)
ot_gfile_get_path_cached (priv->objects_dir)); ot_gfile_get_path_cached (priv->objects_dir));
goto out; goto out;
} }
if (!ot_gfile_ensure_directory (priv->pending_dir, FALSE, error))
goto out;
priv->config = g_key_file_new (); priv->config = g_key_file_new ();
if (!g_key_file_load_from_file (priv->config, ot_gfile_get_path_cached (priv->config_file), 0, error)) if (!g_key_file_load_from_file (priv->config, ot_gfile_get_path_cached (priv->config_file), 0, error))
@ -700,35 +704,60 @@ create_checksum_and_objtype (const char *checksum,
return g_strconcat (checksum, ".", ostree_object_type_to_string (objtype), NULL); return g_strconcat (checksum, ".", ostree_object_type_to_string (objtype), NULL);
} }
static GFile *
get_pending_object_path (OstreeRepo *self,
const char *checksum,
OstreeObjectType objtype)
{
OstreeRepoPrivate *priv = GET_PRIVATE (self);
char *relpath;
GFile *ret;
relpath = ostree_get_relative_object_path (checksum, objtype);
ret = g_file_resolve_relative_path (priv->pending_dir, relpath);
g_free (relpath);
return ret;
}
gboolean gboolean
ostree_repo_has_object (OstreeRepo *self, ostree_repo_find_object (OstreeRepo *self,
OstreeObjectType objtype, OstreeObjectType objtype,
const char *checksum, const char *checksum,
gboolean *out_have_object, GFile **out_stored_path,
GCancellable *cancellable, GFile **out_pending_path,
GError **error) GCancellable *cancellable,
GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self);
char *tmp_key = NULL;
GFile *object_path = NULL; GFile *object_path = NULL;
struct stat stbuf;
tmp_key = create_checksum_and_objtype (checksum, objtype); g_return_val_if_fail (out_stored_path, FALSE);
g_return_val_if_fail (out_pending_path, FALSE);
if (g_hash_table_lookup (priv->pending_transaction_tmpfiles, tmp_key)) object_path = ostree_repo_get_object_path (self, checksum, objtype);
*out_stored_path = NULL;
*out_pending_path = NULL;
if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
{ {
*out_have_object = TRUE; *out_stored_path = object_path;
object_path = NULL;
} }
else else
{ {
object_path = ostree_repo_get_object_path (self, checksum, objtype); g_clear_object (&object_path);
object_path = get_pending_object_path (self, checksum, objtype);
*out_have_object = g_file_query_exists (object_path, cancellable); if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
{
*out_pending_path = object_path;
object_path = NULL;
}
} }
ret = TRUE; ret = TRUE;
/* out: */ /* out: */
g_free (tmp_key);
g_clear_object (&object_path); g_clear_object (&object_path);
return ret; return ret;
} }
@ -755,36 +784,53 @@ stage_object_impl (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
static gboolean static void
insert_into_transaction (OstreeRepo *self, insert_into_transaction (OstreeRepo *self,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype)
GFile *tempfile_path,
GError **error)
{ {
gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
char *key; char *key;
key = create_checksum_and_objtype (checksum, objtype); key = create_checksum_and_objtype (checksum, objtype);
/* Takes ownership */
g_hash_table_replace (priv->pending_transaction, key, NULL);
}
if (g_hash_table_lookup (priv->pending_transaction_tmpfiles, key)) static gboolean
stage_tmpfile_trusted (OstreeRepo *self,
const char *checksum,
OstreeObjectType objtype,
GFile *tempfile_path,
GError **error)
{
gboolean ret = FALSE;
GFile *pending_path = NULL;
GFile *checksum_dir = NULL;
pending_path = get_pending_object_path (self, checksum, objtype);
checksum_dir = g_file_get_parent (pending_path);
if (!ot_gfile_ensure_directory (checksum_dir, TRUE, error))
goto out;
if (link (ot_gfile_get_path_cached (tempfile_path), ot_gfile_get_path_cached (pending_path)) < 0)
{ {
if (unlink (ot_gfile_get_path_cached (tempfile_path)) < 0) if (errno != EEXIST)
{ {
ot_util_set_error_from_errno (error, errno); ot_util_set_error_from_errno (error, errno);
goto out; goto out;
} }
g_free (key);
}
else
{
g_hash_table_insert (priv->pending_transaction_tmpfiles,
key, g_strdup (ot_gfile_get_basename_cached (tempfile_path)));
} }
insert_into_transaction (self, checksum, objtype);
(void) unlink (ot_gfile_get_path_cached (tempfile_path));
ret = TRUE; ret = TRUE;
out: out:
g_clear_object (&pending_path);
g_clear_object (&checksum_dir);
return ret; return ret;
} }
@ -816,7 +862,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
g_variant_get_size (serialized), g_variant_get_size (serialized),
NULL); NULL);
if (!ostree_create_temp_file_from_input (priv->transaction_dir, if (!ostree_create_temp_file_from_input (priv->tmp_dir,
"archive-tmp-", NULL, "archive-tmp-", NULL,
NULL, NULL, mem, NULL, NULL, mem,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
@ -826,7 +872,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
goto out; goto out;
temp_info = dup_file_info_owned_by_me (file_info); temp_info = dup_file_info_owned_by_me (file_info);
if (!ostree_create_temp_file_from_input (priv->transaction_dir, if (!ostree_create_temp_file_from_input (priv->tmp_dir,
"archive-tmp-", NULL, "archive-tmp-", NULL,
temp_info, NULL, input, temp_info, NULL, input,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
@ -861,12 +907,12 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
else else
actual_checksum = g_checksum_get_string (ret_checksum); actual_checksum = g_checksum_get_string (ret_checksum);
if (!insert_into_transaction (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
content_temp_file, error)) content_temp_file, error))
goto out; goto out;
if (!insert_into_transaction (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
meta_temp_file, error)) meta_temp_file, error))
goto out; goto out;
ret = TRUE; ret = TRUE;
@ -898,7 +944,8 @@ stage_object_impl (OstreeRepo *self,
GChecksum *ret_checksum = NULL; GChecksum *ret_checksum = NULL;
GFileInfo *temp_info = NULL; GFileInfo *temp_info = NULL;
GFile *temp_file = NULL; GFile *temp_file = NULL;
gboolean already_exists; GFile *stored_path = NULL;
GFile *pending_path = NULL;
const char *actual_checksum; const char *actual_checksum;
g_return_val_if_fail (priv->in_transaction, FALSE); g_return_val_if_fail (priv->in_transaction, FALSE);
@ -910,29 +957,28 @@ stage_object_impl (OstreeRepo *self,
if (expected_checksum) if (expected_checksum)
{ {
if (!ostree_repo_has_object (self, objtype, expected_checksum, &already_exists, cancellable, error)) if (!ostree_repo_find_object (self, objtype, expected_checksum, &stored_path, &pending_path,
cancellable, error))
goto out; goto out;
} }
else
already_exists = FALSE;
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META);
if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) if (stored_path == NULL && pending_path == NULL)
{ {
g_assert (file_info != NULL); if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE)
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) {
g_assert (input != NULL); g_assert (file_info != NULL);
} if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
else if (OSTREE_OBJECT_TYPE_IS_META (objtype)) g_assert (input != NULL);
{ }
g_assert (xattrs == NULL); else if (OSTREE_OBJECT_TYPE_IS_META (objtype))
g_assert (input != NULL); {
} g_assert (xattrs == NULL);
g_assert (input != NULL);
}
if (!already_exists)
{
if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE && priv->mode == OSTREE_REPO_MODE_ARCHIVE) if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE && priv->mode == OSTREE_REPO_MODE_ARCHIVE)
{ {
if (!impl_stage_archive_file_object_from_raw (self, file_info, xattrs, input, if (!impl_stage_archive_file_object_from_raw (self, file_info, xattrs, input,
@ -943,7 +989,7 @@ stage_object_impl (OstreeRepo *self,
} }
else else
{ {
if (!ostree_create_temp_file_from_input (priv->transaction_dir, if (!ostree_create_temp_file_from_input (priv->tmp_dir,
ostree_object_type_to_string (objtype), NULL, ostree_object_type_to_string (objtype), NULL,
file_info, xattrs, input, file_info, xattrs, input,
objtype, objtype,
@ -967,13 +1013,23 @@ stage_object_impl (OstreeRepo *self,
} }
} }
if (!insert_into_transaction (self, actual_checksum, objtype, if (!stage_tmpfile_trusted (self, actual_checksum, objtype,
temp_file, error)) temp_file, error))
goto out; goto out;
g_clear_object (&temp_file); g_clear_object (&temp_file);
} }
} }
else if (pending_path)
{
g_assert (expected_checksum);
insert_into_transaction (self, expected_checksum, objtype);
}
else
{
g_assert (stored_path != NULL);
/* Nothing to do */
}
ret = TRUE; ret = TRUE;
ot_transfer_out_value(out_checksum, &ret_checksum); ot_transfer_out_value(out_checksum, &ret_checksum);
out: out:
@ -981,6 +1037,8 @@ stage_object_impl (OstreeRepo *self,
(void) unlink (ot_gfile_get_path_cached (temp_file)); (void) unlink (ot_gfile_get_path_cached (temp_file));
g_clear_object (&temp_file); g_clear_object (&temp_file);
g_clear_object (&temp_info); g_clear_object (&temp_info);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
ot_clear_checksum (&ret_checksum); ot_clear_checksum (&ret_checksum);
return ret; return ret;
} }
@ -1030,30 +1088,13 @@ ostree_repo_prepare_transaction (OstreeRepo *self,
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
GString *tmpdir = NULL;
tmpdir = g_string_new ("");
g_return_val_if_fail (priv->in_transaction == FALSE, FALSE); g_return_val_if_fail (priv->in_transaction == FALSE, FALSE);
priv->in_transaction = TRUE; priv->in_transaction = TRUE;
g_clear_object (&priv->transaction_dir);
g_string_append (tmpdir, ot_gfile_get_path_cached (priv->tmp_dir));
g_string_append_c (tmpdir, '/');
g_string_append_printf (tmpdir, "trans-%lu.XXXXXX",
(gulong)getpid ());
if (!mkdtemp (tmpdir->str))
{
ot_util_set_error_from_errno (error, errno);
goto out;
}
priv->transaction_dir = ot_gfile_new_for_path (tmpdir->str);
ret = TRUE; ret = TRUE;
out: /* out: */
if (tmpdir)
g_string_free (tmpdir, TRUE);
return ret; return ret;
} }
@ -1071,11 +1112,10 @@ ostree_repo_commit_transaction (OstreeRepo *self,
g_return_val_if_fail (priv->in_transaction == TRUE, FALSE); g_return_val_if_fail (priv->in_transaction == TRUE, FALSE);
g_hash_table_iter_init (&iter, priv->pending_transaction_tmpfiles); g_hash_table_iter_init (&iter, priv->pending_transaction);
while (g_hash_table_iter_next (&iter, &key, &value)) while (g_hash_table_iter_next (&iter, &key, &value))
{ {
const char *checksum_and_type = key; const char *checksum_and_type = key;
const char *filename = value;
const char *type_str; const char *type_str;
OstreeObjectType objtype; OstreeObjectType objtype;
@ -1087,7 +1127,7 @@ ostree_repo_commit_transaction (OstreeRepo *self,
objtype = ostree_object_type_from_string (type_str + 1); objtype = ostree_object_type_from_string (type_str + 1);
g_clear_object (&f); g_clear_object (&f);
f = g_file_get_child (priv->transaction_dir, filename); f = get_pending_object_path (self, checksum, objtype);
if (!commit_staged_file (self, f, checksum, objtype, cancellable, error)) if (!commit_staged_file (self, f, checksum, objtype, cancellable, error))
goto out; goto out;
@ -1095,12 +1135,10 @@ ostree_repo_commit_transaction (OstreeRepo *self,
ret = TRUE; ret = TRUE;
out: out:
(void) rmdir (ot_gfile_get_path_cached (priv->transaction_dir));
priv->in_transaction = FALSE; priv->in_transaction = FALSE;
g_clear_object (&priv->transaction_dir);
g_free (checksum); g_free (checksum);
g_hash_table_remove_all (priv->pending_transaction_tmpfiles); g_hash_table_remove_all (priv->pending_transaction);
g_clear_object (&f); g_clear_object (&f);
return ret; return ret;
} }
@ -1112,30 +1150,12 @@ ostree_repo_abort_transaction (OstreeRepo *self,
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self); OstreeRepoPrivate *priv = GET_PRIVATE (self);
GFile *f = NULL;
GHashTableIter iter;
gpointer key, value;
g_return_val_if_fail (priv->in_transaction == TRUE, FALSE);
g_hash_table_iter_init (&iter, priv->pending_transaction_tmpfiles);
while (g_hash_table_iter_next (&iter, &key, &value))
{
const char *filename = value;
g_clear_object (&f);
f = g_file_get_child (priv->transaction_dir, filename);
(void) unlink (ot_gfile_get_path_cached (f));
}
/* For now, let's not delete pending files */
g_hash_table_remove_all (priv->pending_transaction);
priv->in_transaction = FALSE; priv->in_transaction = FALSE;
(void) rmdir (ot_gfile_get_path_cached (priv->transaction_dir));
g_clear_object (&priv->transaction_dir);
ret = TRUE; ret = TRUE;
g_hash_table_remove_all (priv->pending_transaction_tmpfiles);
g_clear_object (&f);
return ret; return ret;
} }
@ -1179,35 +1199,21 @@ ostree_repo_load_variant (OstreeRepo *self,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self);
GFile *object_path = NULL; GFile *object_path = NULL;
GFile *tmpfile = NULL; GFile *tmpfile = NULL;
GVariant *ret_variant = NULL; GVariant *ret_variant = NULL;
char *pending_key = NULL;
const char *pending_tmpfile;
g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (expected_type), FALSE); g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (expected_type), FALSE);
pending_key = create_checksum_and_objtype (sha256, expected_type); object_path = ostree_repo_get_object_path (self, sha256, expected_type);
if ((pending_tmpfile = g_hash_table_lookup (priv->pending_transaction_tmpfiles, pending_key)) != NULL) if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error))
{ goto out;
tmpfile = g_file_get_child (priv->transaction_dir, pending_tmpfile);
if (!ostree_map_metadata_file (tmpfile, expected_type, &ret_variant, error))
goto out;
}
else
{
object_path = ostree_repo_get_object_path (self, sha256, expected_type);
if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error))
goto out;
}
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant); ot_transfer_out_value (out_variant, &ret_variant);
out: out:
g_clear_object (&object_path); g_clear_object (&object_path);
g_clear_object (&tmpfile); g_clear_object (&tmpfile);
g_free (pending_key);
ot_clear_gvariant (&ret_variant); ot_clear_gvariant (&ret_variant);
return ret; return ret;
} }

13
src/libostree/ostree-repo.h

@ -92,12 +92,13 @@ gboolean ostree_repo_abort_transaction (OstreeRepo *self,
GCancellable *cancellable, GCancellable *cancellable,
GError **error); GError **error);
gboolean ostree_repo_has_object (OstreeRepo *self, gboolean ostree_repo_find_object (OstreeRepo *self,
OstreeObjectType objtype, OstreeObjectType objtype,
const char *checksum, const char *checksum,
gboolean *out_have_object, GFile **out_stored_path,
GCancellable *cancellable, GFile **out_pending_path,
GError **error); GCancellable *cancellable,
GError **error);
gboolean ostree_repo_stage_object (OstreeRepo *self, gboolean ostree_repo_stage_object (OstreeRepo *self,
OstreeObjectType objtype, OstreeObjectType objtype,

267
src/ostree/ostree-pull.c

@ -117,119 +117,141 @@ fetch_object (OstreeRepo *repo,
SoupURI *baseuri, SoupURI *baseuri,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype,
gboolean *did_exist, GFile **out_temp_path,
GFile **out_file,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GFile *ret_file = NULL;
char *objpath = NULL; char *objpath = NULL;
char *relpath = NULL; char *relpath = NULL;
SoupURI *obj_uri = NULL; SoupURI *obj_uri = NULL;
gboolean exists; GFile *ret_temp_path = NULL;
g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); objpath = ostree_get_relative_object_path (checksum, objtype);
obj_uri = soup_uri_copy (baseuri);
if (!ostree_repo_has_object (repo, objtype, checksum, &exists, NULL, error)) relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL);
soup_uri_set_path (obj_uri, relpath);
if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path, error))
goto out; goto out;
if (!exists)
{
objpath = ostree_get_relative_object_path (checksum, objtype);
obj_uri = soup_uri_copy (baseuri);
relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL);
soup_uri_set_path (obj_uri, relpath);
if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_file, error))
goto out;
*did_exist = FALSE;
}
else
*did_exist = TRUE;
ret = TRUE; ret = TRUE;
ot_transfer_out_value (out_file, &ret_file); ot_transfer_out_value (out_temp_path, &ret_temp_path);
out: out:
if (obj_uri) if (obj_uri)
soup_uri_free (obj_uri); soup_uri_free (obj_uri);
g_clear_object (&ret_file); g_clear_object (&ret_temp_path);
g_free (objpath); g_free (objpath);
g_free (relpath); g_free (relpath);
return ret; return ret;
} }
static gboolean static gboolean
store_object (OstreeRepo *repo, fetch_and_store_object (OstreeRepo *repo,
SoupSession *soup, SoupSession *soup,
SoupURI *baseuri, SoupURI *baseuri,
const char *checksum, const char *checksum,
OstreeObjectType objtype, OstreeObjectType objtype,
gboolean *did_exist, gboolean *out_is_pending,
GError **error) GVariant **out_metadata,
GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GFile *filename = NULL;
GFileInfo *file_info = NULL; GFileInfo *file_info = NULL;
GInputStream *input = NULL; GInputStream *input = NULL;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
GFile *temp_path = NULL;
GVariant *ret_metadata = NULL;
gboolean ret_is_pending;
if (!fetch_object (repo, soup, baseuri, checksum, objtype, did_exist, &filename, error)) g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
if (!ostree_repo_find_object (repo, objtype, checksum,
&stored_path, &pending_path, NULL, error))
goto out; goto out;
if (!*did_exist) if (!(stored_path || pending_path))
{ {
file_info = g_file_query_info (filename, OSTREE_GIO_FAST_QUERYINFO, if (!fetch_object (repo, soup, baseuri, checksum, objtype, &temp_path, error))
goto out;
}
if (temp_path)
{
file_info = g_file_query_info (temp_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, error); G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, error);
if (!file_info) if (!file_info)
goto out; goto out;
input = (GInputStream*)g_file_read (filename, NULL, error); input = (GInputStream*)g_file_read (temp_path, NULL, error);
if (!input) if (!input)
goto out; goto out;
}
if (pending_path || temp_path)
{
if (!ostree_repo_stage_object (repo, objtype, checksum, file_info, NULL, input, NULL, error)) if (!ostree_repo_stage_object (repo, objtype, checksum, file_info, NULL, input, NULL, error))
goto out; goto out;
log_verbose ("Staged object: %s.%s", checksum, ostree_object_type_to_string (objtype));
ret_is_pending = TRUE;
if (out_metadata)
{
if (!ostree_map_metadata_file (pending_path ? pending_path : temp_path, objtype, &ret_metadata, error))
goto out;
}
}
else
{
ret_is_pending = FALSE;
} }
ret = TRUE; ret = TRUE;
if (out_is_pending)
*out_is_pending = ret_is_pending;
ot_transfer_out_value (out_metadata, &ret_metadata);
out: out:
if (filename) if (temp_path)
(void) unlink (ot_gfile_get_path_cached (filename)); (void) unlink (ot_gfile_get_path_cached (temp_path));
ot_clear_gvariant (&ret_metadata);
g_clear_object (&temp_path);
g_clear_object (&file_info); g_clear_object (&file_info);
g_clear_object (&input); g_clear_object (&input);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
return ret; return ret;
} }
static gboolean static gboolean
store_tree_recurse (OstreeRepo *repo, fetch_and_store_tree_recurse (OstreeRepo *repo,
SoupSession *soup, SoupSession *soup,
SoupURI *base_uri, SoupURI *base_uri,
const char *rev, const char *rev,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GVariant *tree = NULL; GVariant *tree = NULL;
GVariant *files_variant = NULL; GVariant *files_variant = NULL;
GVariant *dirs_variant = NULL; GVariant *dirs_variant = NULL;
gboolean did_exist; gboolean is_pending;
int i, n; int i, n;
GVariant *archive_metadata = NULL; GVariant *archive_metadata = NULL;
GFileInfo *archive_file_info = NULL; GFileInfo *archive_file_info = NULL;
GVariant *archive_xattrs = NULL; GVariant *archive_xattrs = NULL;
GFile *meta_file = NULL; GFile *meta_temp_path = NULL;
GFile *content_file = NULL; GFile *content_temp_path = NULL;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
GInputStream *input = NULL; GInputStream *input = NULL;
if (!store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE, &did_exist, error)) if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE, &is_pending, &tree, error))
goto out; goto out;
if (did_exist) if (!is_pending)
log_verbose ("Already have tree %s", rev); log_verbose ("Already have tree %s", rev);
else else
{ {
if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_DIR_TREE, rev, &tree, error))
goto out;
/* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */
files_variant = g_variant_get_child_value (tree, 2); files_variant = g_variant_get_child_value (tree, 2);
dirs_variant = g_variant_get_child_value (tree, 3); dirs_variant = g_variant_get_child_value (tree, 3);
@ -247,43 +269,79 @@ store_tree_recurse (OstreeRepo *repo,
if (!ostree_validate_checksum_string (checksum, error)) if (!ostree_validate_checksum_string (checksum, error))
goto out; goto out;
g_clear_object (&meta_file); g_clear_object (&stored_path);
g_clear_object (&pending_path);
if (!fetch_object (repo, soup, base_uri, checksum, /* If we're fetching from an archive into a bare repository, we need
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, * to explicitly check for raw file types locally.
&did_exist, */
&meta_file, if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE)
error))
goto out;
if (!ostree_map_metadata_file (meta_file, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
&archive_metadata, error))
goto out;
if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error))
goto out;
g_clear_object (&input);
g_clear_object (&content_file);
if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR)
{ {
if (!fetch_object (repo, soup, base_uri, checksum, if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, &stored_path, &pending_path, NULL, error))
&did_exist,
&content_file,
error))
goto out; goto out;
}
input = (GInputStream*)g_file_read (content_file, NULL, error); else
if (!input) {
if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, checksum,
&stored_path, &pending_path, NULL, error))
goto out; goto out;
} }
if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, g_clear_object (&input);
checksum, g_clear_object (&archive_file_info);
archive_file_info, archive_xattrs, input, ot_clear_gvariant (&archive_xattrs);
NULL, error)) if (!(stored_path || pending_path))
goto out; {
g_clear_object (&meta_temp_path);
if (!fetch_object (repo, soup, base_uri, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
&meta_temp_path,
error))
goto out;
if (!ostree_map_metadata_file (meta_temp_path,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
&archive_metadata, error))
goto out;
if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error))
goto out;
if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR)
{
if (!fetch_object (repo, soup, base_uri, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
&content_temp_path,
error))
goto out;
input = (GInputStream*)g_file_read (content_temp_path, NULL, error);
if (!input)
goto out;
}
}
if (!stored_path)
{
log_verbose ("Staged file object: %s", checksum);
if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE,
checksum,
archive_file_info, archive_xattrs, input,
NULL, error))
goto out;
}
if (meta_temp_path)
{
(void) unlink (ot_gfile_get_path_cached (meta_temp_path));
g_clear_object (&meta_temp_path);
}
if (content_temp_path)
{
(void) unlink (ot_gfile_get_path_cached (content_temp_path));
g_clear_object (&content_temp_path);
}
} }
n = g_variant_n_children (dirs_variant); n = g_variant_n_children (dirs_variant);
@ -303,10 +361,10 @@ store_tree_recurse (OstreeRepo *repo,
if (!ostree_validate_checksum_string (meta_checksum, error)) if (!ostree_validate_checksum_string (meta_checksum, error))
goto out; goto out;
if (!store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, &did_exist, error)) if (!fetch_and_store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, NULL, NULL, error))
goto out; goto out;
if (!store_tree_recurse (repo, soup, base_uri, tree_checksum, error)) if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_checksum, error))
goto out; goto out;
} }
} }
@ -320,40 +378,49 @@ store_tree_recurse (OstreeRepo *repo,
ot_clear_gvariant (&archive_xattrs); ot_clear_gvariant (&archive_xattrs);
g_clear_object (&archive_file_info); g_clear_object (&archive_file_info);
g_clear_object (&input); g_clear_object (&input);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
if (content_temp_path)
{
(void) unlink (ot_gfile_get_path_cached (content_temp_path));
g_clear_object (&content_temp_path);
}
if (meta_temp_path)
{
(void) unlink (ot_gfile_get_path_cached (meta_temp_path));
g_clear_object (&meta_temp_path);
}
return ret; return ret;
} }
static gboolean static gboolean
store_commit_recurse (OstreeRepo *repo, fetch_and_store_commit_recurse (OstreeRepo *repo,
SoupSession *soup, SoupSession *soup,
SoupURI *base_uri, SoupURI *base_uri,
const char *rev, const char *rev,
GError **error) GError **error)
{ {
gboolean ret = FALSE; gboolean ret = FALSE;
GVariant *commit = NULL; GVariant *commit = NULL;
const char *tree_contents_checksum; const char *tree_contents_checksum;
const char *tree_meta_checksum; const char *tree_meta_checksum;
gboolean did_exist; gboolean is_pending;
if (!store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT, &did_exist, error)) if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT, &is_pending, &commit, error))
goto out; goto out;
if (did_exist) if (!is_pending)
log_verbose ("Already have commit %s", rev); log_verbose ("Already have commit %s", rev);
else else
{ {
if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, &commit, error))
goto out;
/* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */ /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
g_variant_get_child (commit, 6, "&s", &tree_contents_checksum); g_variant_get_child (commit, 6, "&s", &tree_contents_checksum);
g_variant_get_child (commit, 7, "&s", &tree_meta_checksum); g_variant_get_child (commit, 7, "&s", &tree_meta_checksum);
if (!store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, &did_exist, error)) if (!fetch_and_store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, NULL, NULL, error))
goto out; goto out;
if (!store_tree_recurse (repo, soup, base_uri, tree_contents_checksum, error)) if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_contents_checksum, error))
goto out; goto out;
} }
@ -450,7 +517,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_prepare_transaction (repo, NULL, error)) if (!ostree_repo_prepare_transaction (repo, NULL, error))
goto out; goto out;
if (!store_commit_recurse (repo, soup, base_uri, rev, error)) if (!fetch_and_store_commit_recurse (repo, soup, base_uri, rev, error))
goto out; goto out;
if (!ostree_repo_commit_transaction (repo, NULL, error)) if (!ostree_repo_commit_transaction (repo, NULL, error))