diff --git a/src/libostree/README.md b/src/libostree/README.md index 83d9f5fb..9a769279 100644 --- a/src/libostree/README.md +++ b/src/libostree/README.md @@ -44,6 +44,8 @@ MILESTONE 2 ----------- * Store checksums as ay * Drop version/metadata from tree/dirmeta objects +* Add index size to superindex, pack size to index + - So pull can calculate how much we need to download * Split pack files into metadata/data * Restructure repository so that links can be generated as a cache; i.e. objects/raw, pack files are now the canonical diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 8ea4bbda..04e9f974 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -72,8 +72,6 @@ struct _OstreeRepoPrivate { GHashTable *pack_index_mappings; GHashTable *pack_data_mappings; - - GHashTable *pending_transaction; }; static void @@ -93,7 +91,6 @@ ostree_repo_finalize (GObject *object) g_clear_object (&priv->config_file); g_hash_table_destroy (priv->pack_index_mappings); g_hash_table_destroy (priv->pack_data_mappings); - g_hash_table_destroy (priv->pending_transaction); if (priv->config) g_key_file_free (priv->config); @@ -202,9 +199,6 @@ ostree_repo_init (OstreeRepo *self) priv->pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_mapped_file_unref); - priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, - NULL); } OstreeRepo* @@ -768,22 +762,6 @@ ostree_repo_get_file_object_path (OstreeRepo *self, return ostree_repo_get_object_path (self, checksum, get_objtype_for_repo_file (self)); } -static GFile * -get_pending_object_path (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype) -{ - OstreeRepoPrivate *priv = GET_PRIVATE (self); - char *relpath; - GFile *ret; - - relpath = ostree_get_relative_object_path (checksum, objtype); - ret = g_file_resolve_relative_path (priv->pending_dir, relpath); - g_free (relpath); - - return ret; -} - static GFileInfo * dup_file_info_owned_by_me (GFileInfo *file_info) { @@ -807,52 +785,40 @@ stage_object_impl (OstreeRepo *self, GCancellable *cancellable, GError **error); -static void -insert_into_transaction (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype) -{ - OstreeRepoPrivate *priv = GET_PRIVATE (self); - char *key; - - key = ostree_object_to_string (checksum, objtype); - /* Takes ownership */ - g_hash_table_replace (priv->pending_transaction, key, NULL); -} - static gboolean -stage_tmpfile_trusted (OstreeRepo *self, - const char *checksum, - OstreeObjectType objtype, - GFile *tempfile_path, - GError **error) +commit_tmpfile_trusted (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + GFile *tempfile_path, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; - GFile *pending_path = NULL; + GFile *dest_file = NULL; GFile *checksum_dir = NULL; - pending_path = get_pending_object_path (self, checksum, objtype); - checksum_dir = g_file_get_parent (pending_path); + dest_file = ostree_repo_get_object_path (self, checksum, objtype); + checksum_dir = g_file_get_parent (dest_file); - if (!ot_gfile_ensure_directory (checksum_dir, TRUE, error)) + if (!ot_gfile_ensure_directory (checksum_dir, FALSE, error)) goto out; - - if (link (ot_gfile_get_path_cached (tempfile_path), ot_gfile_get_path_cached (pending_path)) < 0) + + if (link (ot_gfile_get_path_cached (tempfile_path), ot_gfile_get_path_cached (dest_file)) < 0) { if (errno != EEXIST) { ot_util_set_error_from_errno (error, errno); + g_prefix_error (error, "Storing file '%s': ", + ot_gfile_get_path_cached (dest_file)); goto out; } } - insert_into_transaction (self, checksum, objtype); - (void) unlink (ot_gfile_get_path_cached (tempfile_path)); - + ret = TRUE; out: - g_clear_object (&pending_path); + g_clear_object (&dest_file); g_clear_object (&checksum_dir); return ret; } @@ -933,12 +899,12 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, else actual_checksum = g_checksum_get_string (ret_checksum); - if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - content_temp_file, error)) + if (!commit_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, + content_temp_file, cancellable, error)) goto out; - if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - meta_temp_file, error)) + if (!commit_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, + meta_temp_file, cancellable, error)) goto out; ret = TRUE; @@ -972,7 +938,6 @@ stage_object_impl (OstreeRepo *self, GFileInfo *temp_info = NULL; GFile *temp_file = NULL; GFile *stored_path = NULL; - GFile *pending_path = NULL; char *pack_checksum = NULL; guint64 pack_offset; const char *actual_checksum; @@ -989,16 +954,14 @@ stage_object_impl (OstreeRepo *self, if (!store_if_packed) { if (!ostree_repo_find_object (self, objtype, expected_checksum, - &stored_path, &pending_path, - &pack_checksum, &pack_offset, + &stored_path, &pack_checksum, &pack_offset, cancellable, error)) goto out; } else { if (!ostree_repo_find_object (self, objtype, expected_checksum, - &stored_path, &pending_path, - NULL, NULL, + &stored_path, NULL, NULL, cancellable, error)) goto out; } @@ -1007,7 +970,7 @@ stage_object_impl (OstreeRepo *self, g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META); - if (stored_path == NULL && pending_path == NULL && pack_checksum == NULL) + if (stored_path == NULL && pack_checksum == NULL) { if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) { @@ -1055,17 +1018,12 @@ stage_object_impl (OstreeRepo *self, } } - if (!stage_tmpfile_trusted (self, actual_checksum, objtype, - temp_file, error)) + if (!commit_tmpfile_trusted (self, actual_checksum, objtype, + temp_file, cancellable, error)) goto out; g_clear_object (&temp_file); } } - else if (pending_path) - { - g_assert (expected_checksum); - insert_into_transaction (self, expected_checksum, objtype); - } else { g_assert (stored_path != NULL || pack_checksum != NULL); @@ -1080,50 +1038,11 @@ stage_object_impl (OstreeRepo *self, g_clear_object (&temp_file); g_clear_object (&temp_info); g_clear_object (&stored_path); - g_clear_object (&pending_path); g_free (pack_checksum); ot_clear_checksum (&ret_checksum); return ret; } -static gboolean -commit_staged_file (OstreeRepo *self, - GFile *file, - const char *checksum, - OstreeObjectType objtype, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - GFile *dest_file = NULL; - GFile *checksum_dir = NULL; - - dest_file = ostree_repo_get_object_path (self, checksum, objtype); - checksum_dir = g_file_get_parent (dest_file); - - if (!ot_gfile_ensure_directory (checksum_dir, FALSE, error)) - goto out; - - if (link (ot_gfile_get_path_cached (file), ot_gfile_get_path_cached (dest_file)) < 0) - { - if (errno != EEXIST) - { - ot_util_set_error_from_errno (error, errno); - g_prefix_error (error, "Storing file '%s': ", - ot_gfile_get_path_cached (file)); - goto out; - } - } - - (void) unlink (ot_gfile_get_path_cached (file)); - - ret = TRUE; - out: - g_clear_object (&dest_file); - g_clear_object (&checksum_dir); - return ret; -} - gboolean ostree_repo_prepare_transaction (OstreeRepo *self, GCancellable *cancellable, @@ -1148,34 +1067,13 @@ ostree_repo_commit_transaction (OstreeRepo *self, { gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); - GFile *f = NULL; - GHashTableIter iter; - gpointer key, value; - char *checksum = NULL; - OstreeObjectType objtype; g_return_val_if_fail (priv->in_transaction == TRUE, FALSE); - g_hash_table_iter_init (&iter, priv->pending_transaction); - while (g_hash_table_iter_next (&iter, &key, &value)) - { - g_free (checksum); - ostree_object_from_string ((char*)key, &checksum, &objtype); - - g_clear_object (&f); - f = get_pending_object_path (self, checksum, objtype); - - if (!commit_staged_file (self, f, checksum, objtype, cancellable, error)) - goto out; - } - ret = TRUE; - out: + /* out: */ priv->in_transaction = FALSE; - g_free (checksum); - g_hash_table_remove_all (priv->pending_transaction); - g_clear_object (&f); return ret; } @@ -1187,8 +1085,6 @@ ostree_repo_abort_transaction (OstreeRepo *self, gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); - /* For now, let's not delete pending files */ - g_hash_table_remove_all (priv->pending_transaction); priv->in_transaction = FALSE; ret = TRUE; @@ -3219,7 +3115,7 @@ ostree_repo_load_file (OstreeRepo *self, /* Blah, right now we need to look up the content too to get the file size */ if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - checksum, &content_loose_path, NULL, + checksum, &content_loose_path, &content_pack_checksum, &content_pack_offset, cancellable, error)) goto out; @@ -3475,7 +3371,6 @@ ostree_repo_find_object (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, GFile **out_stored_path, - GFile **out_pending_path, char **out_pack_checksum, guint64 *out_pack_offset, GCancellable *cancellable, @@ -3484,7 +3379,6 @@ ostree_repo_find_object (OstreeRepo *self, gboolean ret = FALSE; GFile *object_path = NULL; GFile *ret_stored_path = NULL; - GFile *ret_pending_path = NULL; char *ret_pack_checksum = NULL; guint64 ret_pack_offset = 0; struct stat stbuf; @@ -3499,15 +3393,6 @@ ostree_repo_find_object (OstreeRepo *self, else { g_clear_object (&object_path); - if (out_pending_path) - { - object_path = get_pending_object_path (self, checksum, objtype); - if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) - { - ret_pending_path = object_path; - object_path = NULL; - } - } } if (out_pack_checksum) @@ -3520,14 +3405,12 @@ ostree_repo_find_object (OstreeRepo *self, ret = TRUE; ot_transfer_out_value (out_stored_path, &ret_stored_path); - ot_transfer_out_value (out_pending_path, &ret_pending_path); ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum); if (out_pack_offset) *out_pack_offset = ret_pack_offset; out: g_clear_object (&object_path); g_clear_object (&ret_stored_path); - g_clear_object (&ret_pending_path); g_free (ret_pack_checksum); return ret; } @@ -3541,7 +3424,6 @@ ostree_repo_load_variant (OstreeRepo *self, { gboolean ret = FALSE; GFile *object_path = NULL; - GFile *pending_path = NULL; GVariant *packed_object = NULL; GVariant *ret_variant = NULL; char *pack_checksum = NULL; @@ -3552,16 +3434,15 @@ ostree_repo_load_variant (OstreeRepo *self, g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE); - if (!ostree_repo_find_object (self, objtype, sha256, &object_path, &pending_path, + if (!ostree_repo_find_object (self, objtype, sha256, &object_path, &pack_checksum, &object_offset, cancellable, error)) goto out; /* Prefer loose metadata for now */ - if (object_path != NULL || pending_path != NULL) + if (object_path != NULL) { - if (!ostree_map_metadata_file (object_path ? object_path : pending_path, - objtype, &ret_variant, error)) + if (!ostree_map_metadata_file (object_path, objtype, &ret_variant, error)) goto out; } else if (pack_checksum != NULL) diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 55c918e2..a4244962 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -96,7 +96,6 @@ gboolean ostree_repo_find_object (OstreeRepo *self, OstreeObjectType objtype, const char *checksum, GFile **out_stored_path, - GFile **out_pending_path, char **out_pack_checksum, guint64 *out_pack_offset, GCancellable *cancellable, diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index 1e543569..de3fa8f0 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -62,6 +62,8 @@ typedef struct { gboolean fetched_packs; GPtrArray *cached_pack_indexes; + + GHashTable *file_checksums_to_fetch; } OtPullData; static SoupURI * @@ -102,7 +104,6 @@ suburi_new (SoupURI *base, return ret; } - typedef struct { SoupSession *session; GOutputStream *stream; @@ -436,58 +437,6 @@ fetch_loose_object (OtPullData *pull_data, return ret; } -static gboolean -find_object (OtPullData *pull_data, - const char *checksum, - OstreeObjectType objtype, - gboolean *out_is_stored, - gboolean *out_is_pending, - char **out_remote_pack_checksum, - guint64 *out_offset, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gboolean ret_is_stored; - gboolean ret_is_pending; - GFile *stored_path = NULL; - GFile *pending_path = NULL; - char *local_pack_checksum = NULL; - char *ret_remote_pack_checksum = NULL; - guint64 offset; - - if (!ostree_repo_find_object (pull_data->repo, objtype, checksum, - &stored_path, &pending_path, - &local_pack_checksum, NULL, - cancellable, error)) - goto out; - - ret_is_stored = (stored_path != NULL || local_pack_checksum != NULL); - ret_is_pending = pending_path != NULL; - - if (!(ret_is_stored || ret_is_pending)) - { - if (!find_object_in_remote_packs (pull_data, checksum, objtype, - &ret_remote_pack_checksum, &offset, - cancellable, error)) - goto out; - } - - ret = TRUE; - if (out_is_stored) - *out_is_stored = ret_is_stored; - if (out_is_pending) - *out_is_pending = ret_is_pending; - ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum); - if (out_offset) - *out_offset = offset; - out: - g_free (local_pack_checksum); - g_free (ret_remote_pack_checksum); - g_clear_object (&stored_path); - return ret; -} - static void unlink_file_on_unref (GFile *f) { @@ -499,31 +448,39 @@ static gboolean fetch_object_if_not_stored (OtPullData *pull_data, const char *checksum, OstreeObjectType objtype, - gboolean *out_is_stored, - gboolean *out_is_pending, GInputStream **out_input, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - gboolean ret_is_stored = FALSE; - gboolean ret_is_pending = FALSE; GInputStream *ret_input = NULL; GFile *temp_path = NULL; + GFile *stored_path = NULL; GFile *pack_path = NULL; GMappedFile *pack_map = NULL; + char *local_pack_checksum = NULL; char *remote_pack_checksum = NULL; guint64 pack_offset = 0; GVariant *pack_entry = NULL; + gboolean is_stored; - if (!find_object (pull_data, checksum, objtype, &ret_is_stored, - &ret_is_pending, &remote_pack_checksum, - &pack_offset, cancellable, error)) + if (!ostree_repo_find_object (pull_data->repo, objtype, checksum, + &stored_path, &local_pack_checksum, NULL, + cancellable, error)) goto out; + + is_stored = (stored_path != NULL || local_pack_checksum != NULL); + if (!is_stored) + { + if (!find_object_in_remote_packs (pull_data, checksum, objtype, + &remote_pack_checksum, &pack_offset, + cancellable, error)) + goto out; + } if (remote_pack_checksum != NULL) { - g_assert (!(ret_is_stored || ret_is_pending)); + g_assert (!is_stored); if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path, cancellable, error)) @@ -539,12 +496,13 @@ fetch_object_if_not_stored (OtPullData *pull_data, cancellable, error)) goto out; + /* Kind of a hack... */ ret_input = ostree_read_pack_entry_as_stream (pack_entry); g_object_set_data_full ((GObject*)ret_input, "ostree-pull-pack-map", pack_map, (GDestroyNotify) g_mapped_file_unref); pack_map = NULL; /* Transfer ownership */ } - else if (!(ret_is_stored || ret_is_pending)) + else if (!is_stored) { if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error)) goto out; @@ -559,11 +517,9 @@ fetch_object_if_not_stored (OtPullData *pull_data, ret = TRUE; ot_transfer_out_value (out_input, &ret_input); - if (out_is_stored) - *out_is_stored = ret_is_stored; - if (out_is_pending) - *out_is_pending = ret_is_pending; out: + g_free (local_pack_checksum); + g_clear_object (&stored_path); g_clear_object (&temp_path); g_clear_object (&pack_path); if (pack_map) @@ -578,27 +534,20 @@ static gboolean fetch_and_store_object (OtPullData *pull_data, const char *checksum, OstreeObjectType objtype, - gboolean *out_was_stored, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; GFileInfo *file_info = NULL; GInputStream *input = NULL; - GFile *stored_path = NULL; - GFile *pending_path = NULL; - char *pack_checksum = NULL; - gboolean is_stored; - gboolean is_pending; g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); - if (!fetch_object_if_not_stored (pull_data, checksum, objtype, - &is_stored, &is_pending, &input, + if (!fetch_object_if_not_stored (pull_data, checksum, objtype, &input, cancellable, error)) goto out; - if (is_pending || input) + if (input) { if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, NULL, NULL, input, cancellable, error)) @@ -608,14 +557,9 @@ fetch_and_store_object (OtPullData *pull_data, } ret = TRUE; - if (out_was_stored) - *out_was_stored = is_stored; out: g_clear_object (&file_info); g_clear_object (&input); - g_clear_object (&stored_path); - g_clear_object (&pending_path); - g_free (pack_checksum); return ret; } @@ -623,17 +567,15 @@ static gboolean fetch_and_store_metadata (OtPullData *pull_data, const char *checksum, OstreeObjectType objtype, - gboolean *out_was_stored, GVariant **out_variant, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - gboolean ret_was_stored; GVariant *ret_variant = NULL; if (!fetch_and_store_object (pull_data, checksum, objtype, - &ret_was_stored, cancellable, error)) + cancellable, error)) goto out; if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum, @@ -642,8 +584,6 @@ fetch_and_store_metadata (OtPullData *pull_data, ret = TRUE; ot_transfer_out_value (out_variant, &ret_variant); - if (out_was_stored) - *out_was_stored = ret_was_stored; out: ot_clear_gvariant (&ret_variant); return ret; @@ -658,7 +598,6 @@ fetch_and_store_file (OtPullData *pull_data, gboolean ret = FALSE; GInputStream *input = NULL; GFile *stored_path = NULL; - GFile *pending_path = NULL; char *pack_checksum = NULL; GVariant *archive_metadata_container = NULL; GVariant *archive_metadata = NULL; @@ -672,23 +611,11 @@ fetch_and_store_file (OtPullData *pull_data, if (ostree_repo_get_mode (pull_data->repo) == OSTREE_REPO_MODE_BARE) { if (!ostree_repo_find_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, - checksum, &stored_path, &pending_path, &pack_checksum, + checksum, &stored_path, &pack_checksum, NULL, cancellable, error)) goto out; - if (stored_path || pack_checksum) - skip_archive_fetch = TRUE; - else if (pending_path != NULL) - { - skip_archive_fetch = TRUE; - if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, - checksum, NULL, NULL, NULL, cancellable, error)) - goto out; - } - else - skip_archive_fetch = FALSE; - - g_clear_object (&stored_path); + skip_archive_fetch = (stored_path || pack_checksum); } else { @@ -699,7 +626,7 @@ fetch_and_store_file (OtPullData *pull_data, { if (!fetch_object_if_not_stored (pull_data, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - NULL, NULL, &input, cancellable, error)) + &input, cancellable, error)) goto out; if (input != NULL) @@ -721,8 +648,7 @@ fetch_and_store_file (OtPullData *pull_data, { if (!fetch_object_if_not_stored (pull_data, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - NULL, NULL, &input, - cancellable, error)) + &input, cancellable, error)) goto out; } @@ -737,7 +663,6 @@ fetch_and_store_file (OtPullData *pull_data, out: g_free (pack_checksum); g_clear_object (&stored_path); - g_clear_object (&pending_path); g_clear_object (&input); ot_clear_gvariant (&archive_metadata_container); ot_clear_gvariant (&archive_metadata); @@ -747,74 +672,70 @@ fetch_and_store_file (OtPullData *pull_data, } static gboolean -fetch_and_store_tree_recurse (OtPullData *pull_data, - const char *rev, - GCancellable *cancellable, - GError **error) +fetch_and_store_tree_metadata_recurse (OtPullData *pull_data, + const char *rev, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; GVariant *tree = NULL; GVariant *files_variant = NULL; GVariant *dirs_variant = NULL; - gboolean was_stored; int i, n; GFile *stored_path = NULL; - GFile *pending_path = NULL; char *pack_checksum = NULL; if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE, - &was_stored, &tree, cancellable, error)) + &tree, cancellable, error)) goto out; - if (was_stored) - log_verbose ("Already have tree %s", rev); - else + /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ + files_variant = g_variant_get_child_value (tree, 2); + dirs_variant = g_variant_get_child_value (tree, 3); + + n = g_variant_n_children (files_variant); + for (i = 0; i < n; i++) { - /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ - files_variant = g_variant_get_child_value (tree, 2); - dirs_variant = g_variant_get_child_value (tree, 3); + const char *filename; + const char *checksum; + + g_variant_get_child (files_variant, i, "(&s&s)", &filename, &checksum); + + if (!ot_util_filename_validate (filename, error)) + goto out; + if (!ostree_validate_checksum_string (checksum, error)) + goto out; + + { + char *duped_key = g_strdup (checksum); + g_hash_table_replace (pull_data->file_checksums_to_fetch, + duped_key, duped_key); + } + } - n = g_variant_n_children (files_variant); - for (i = 0; i < n; i++) - { - const char *filename; - const char *checksum; + n = g_variant_n_children (dirs_variant); + for (i = 0; i < n; i++) + { + const char *dirname; + const char *tree_checksum; + const char *meta_checksum; - g_variant_get_child (files_variant, i, "(&s&s)", &filename, &checksum); + g_variant_get_child (dirs_variant, i, "(&s&s&s)", + &dirname, &tree_checksum, &meta_checksum); - if (!ot_util_filename_validate (filename, error)) - goto out; - if (!ostree_validate_checksum_string (checksum, error)) - goto out; + if (!ot_util_filename_validate (dirname, error)) + goto out; + if (!ostree_validate_checksum_string (tree_checksum, error)) + goto out; + if (!ostree_validate_checksum_string (meta_checksum, error)) + goto out; - if (!fetch_and_store_file (pull_data, checksum, cancellable, error)) - goto out; - } - - n = g_variant_n_children (dirs_variant); - for (i = 0; i < n; i++) - { - const char *dirname; - const char *tree_checksum; - const char *meta_checksum; + if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, + cancellable, error)) + goto out; - g_variant_get_child (dirs_variant, i, "(&s&s&s)", - &dirname, &tree_checksum, &meta_checksum); - - if (!ot_util_filename_validate (dirname, error)) - goto out; - if (!ostree_validate_checksum_string (tree_checksum, error)) - goto out; - if (!ostree_validate_checksum_string (meta_checksum, error)) - goto out; - - if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, - NULL, cancellable, error)) - goto out; - - if (!fetch_and_store_tree_recurse (pull_data, tree_checksum, cancellable, error)) - goto out; - } + if (!fetch_and_store_tree_metadata_recurse (pull_data, tree_checksum, cancellable, error)) + goto out; } ret = TRUE; @@ -823,7 +744,6 @@ fetch_and_store_tree_recurse (OtPullData *pull_data, ot_clear_gvariant (&files_variant); ot_clear_gvariant (&dirs_variant); g_clear_object (&stored_path); - g_clear_object (&pending_path); g_free (pack_checksum); return ret; } @@ -838,28 +758,22 @@ fetch_and_store_commit_recurse (OtPullData *pull_data, GVariant *commit = NULL; const char *tree_contents_checksum; const char *tree_meta_checksum; - gboolean was_stored; if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT, - &was_stored, &commit, cancellable, error)) + &commit, cancellable, error)) goto out; - if (was_stored) - log_verbose ("Already have commit %s", rev); - else - { - /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */ - g_variant_get_child (commit, 6, "&s", &tree_contents_checksum); - g_variant_get_child (commit, 7, "&s", &tree_meta_checksum); - - if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, - NULL, cancellable, error)) - goto out; - - if (!fetch_and_store_tree_recurse (pull_data, tree_contents_checksum, - cancellable, error)) - goto out; - } + /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */ + g_variant_get_child (commit, 6, "&s", &tree_contents_checksum); + g_variant_get_child (commit, 7, "&s", &tree_meta_checksum); + + if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, + cancellable, error)) + goto out; + + if (!fetch_and_store_tree_metadata_recurse (pull_data, tree_contents_checksum, + cancellable, error)) + goto out; ret = TRUE; out: @@ -897,6 +811,29 @@ fetch_ref_contents (OtPullData *pull_data, return ret; } +static gboolean +fetch_files (OtPullData *pull_data, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GHashTableIter hash_iter; + gpointer key, value; + + g_hash_table_iter_init (&hash_iter, pull_data->file_checksums_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *checksum = key; + + if (!fetch_and_store_file (pull_data, checksum, cancellable, error)) + goto out; + } + + ret = TRUE; + out: + return ret; +} + static gboolean pull_one_commit (OtPullData *pull_data, const char *branch, @@ -941,6 +878,9 @@ pull_one_commit (OtPullData *pull_data, if (!fetch_and_store_commit_recurse (pull_data, rev, cancellable, error)) goto out; + if (!fetch_files (pull_data, cancellable, error)) + goto out; + if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error)) goto out; @@ -1026,7 +966,6 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) char *path = NULL; char *baseurl = NULL; char *summary_data = NULL; - SoupURI *base_uri = NULL; SoupURI *summary_uri = NULL; GKeyFile *config = NULL; GCancellable *cancellable = NULL; @@ -1048,6 +987,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) memset (pull_data, 0, sizeof (*pull_data)); pull_data->repo = repo; + pull_data->file_checksums_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (argc < 2) { @@ -1092,7 +1032,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) } else { - summary_uri = soup_uri_copy (base_uri); + summary_uri = soup_uri_copy (pull_data->base_uri); path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL); soup_uri_set_path (summary_uri, path); @@ -1122,6 +1062,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) g_free (baseurl); g_free (summary_data); g_free (branch_rev); + ot_clear_hashtable (&(pull_data->file_checksums_to_fetch)); if (context) g_option_context_free (context); g_clear_object (&pull_data->session);