shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2024-12-22 17:35:55 +03:00
Code Issues Projects Releases Wiki Activity

pull: Only download summary if we need it for the pull operation

Browse Source 
If we have a commit id for all the refs we're pulling, and if we
don't need the summary to list all the refs when mirroring then the
only reason to download the summary is for the list of deltas.

With the new "indexed-deltas" property in the config file (and mirrored
to the summary file) we can detect when we don't need the summary for
deltas and completely avoid downloading it then.
This commit is contained in:
Alexander Larsson 2020-10-23 13:05:25 +02:00
parent 6c8e6539e2
commit 125ed2b199
2 changed files with 489 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/libostree/ostree-repo-pull-private.h
View File

@ -80,6 +80,7 @@ typedef struct {
GVariant *summary;
GHashTable *summary_deltas_checksums; /* Filled from summary and delta indexes */
gboolean summary_has_deltas; /* True if the summary existed and had a delta index */
gboolean has_indexed_deltas;
GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
GHashTable *verified_commits; /* Set<checksum> of commits that have been verified */
GHashTable *signapi_verified_commits; /* Map<checksum,verification> of commits that have been signapi verified */

932
src/libostree/ostree-repo-pull.c
View File

@ -3535,18 +3535,17 @@ initiate_request (OtPullData *pull_data,
return FALSE;
}
/* If we have a summary, we can use the newer logic */
if (pull_data->summary)
/* If we have a summary or delta index, we can use the newer logic.
* We prefer the index as it might have more deltas than the summary
* (i.e. leave some deltas out of summary to make it smaller). */
if (pull_data->has_indexed_deltas)
{
if (!pull_data->summary_has_deltas)
{
enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref);
}
else
{
if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error))
return FALSE;
}
enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref);
}
else if (pull_data->summary_has_deltas)
{
if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error))
return FALSE;
}
else if (ref != NULL)
{
@ -3590,6 +3589,20 @@ initiate_request (OtPullData *pull_data,
return TRUE;
}
static gboolean
all_requested_refs_have_commit (GHashTable *requested_refs /* (element-type OstreeCollectionRef utf8) */)
{
GLNX_HASH_TABLE_FOREACH_KV (requested_refs, const OstreeCollectionRef*, ref,
const char*, override_commitid)
{
/* Note: "" override means whatever is latest */
if (override_commitid == NULL || *override_commitid == 0)
return FALSE;
}
return TRUE;
}
/* ------------------------------------------------------------------------------------------
* Below is the libsoup-invariant API; these should match
* the stub functions in the #else clause
@ -3695,9 +3708,11 @@ ostree_repo_pull_with_options (OstreeRepo *self,
gboolean opt_ref_keyring_map_set = FALSE;
gboolean disable_sign_verify = FALSE;
gboolean disable_sign_verify_summary = FALSE;
gboolean need_summary = FALSE;
const char *main_collection_id = NULL;
const char *url_override = NULL;
gboolean inherit_transaction = FALSE;
gboolean require_summary_for_mirror = FALSE;
g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */
gsize i;
g_autofree char **opt_localcache_repos = NULL;
@ -3709,6 +3724,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
*/
const char *the_ref_to_fetch = NULL;
OstreeRepoTransactionStats tstats = { 0, };
gboolean remote_mode_loaded = FALSE;
/* Default */
pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE;
@ -4132,442 +4148,11 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->is_commit_only)
pull_data->disable_static_deltas = TRUE;
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
{
g_autoptr(GBytes) bytes_sig = NULL;
gboolean summary_sig_not_modified = FALSE;
g_autofree char *summary_sig_etag = NULL;
guint64 summary_sig_last_modified = 0;
gsize n;
g_autoptr(GVariant) refs = NULL;
g_autoptr(GVariant) deltas = NULL;
g_autoptr(GVariant) additional_metadata = NULL;
gboolean summary_from_cache = FALSE;
gboolean remote_mode_loaded = FALSE;
gboolean tombstone_commits = FALSE;
if (summary_sig_bytes_v)
{
/* Must both be specified */
g_assert (summary_bytes_v);
bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v);
bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v);
if (!bytes_sig || !bytes_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"summary-bytes or summary-sig-bytes set to invalid value");
goto out;
}
g_debug ("Loaded %s summary from options", remote_name_or_baseurl);
}
if (!bytes_sig)
{
g_autofree char *summary_sig_if_none_match = NULL;
guint64 summary_sig_if_modified_since = 0;
/* Load the summary.sig from the network, but send its ETag and
* Last-Modified from the on-disk cache (if it exists) to reduce the
* download size if nothing’s changed. */
_ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig",
&summary_sig_if_none_match, &summary_sig_if_modified_since);
g_clear_pointer (&summary_sig_etag, g_free);
summary_sig_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
summary_sig_if_none_match, summary_sig_if_modified_since,
pull_data->n_network_retries,
&bytes_sig,
&summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary.sig from the cache. Also load summary, since
* `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
if (summary_sig_not_modified)
{
g_clear_pointer (&bytes_sig, g_bytes_unref);
g_clear_pointer (&bytes_summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig",
&bytes_sig,
cancellable, error))
goto out;
if (!bytes_summary &&
!pull_data->remote_repo_local &&
!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
cancellable, error))
goto out;
}
}
if (bytes_sig &&
!bytes_summary &&
!pull_data->remote_repo_local &&
!_ostree_repo_load_cache_summary_if_same_sig (self,
remote_name_or_baseurl,
bytes_sig,
&bytes_summary,
cancellable,
error))
goto out;
if (bytes_summary && !summary_bytes_v)
{
g_debug ("Loaded %s summary from cache", remote_name_or_baseurl);
summary_from_cache = TRUE;
}
if (!pull_data->summary && !bytes_summary)
{
g_autofree char *summary_if_none_match = NULL;
guint64 summary_if_modified_since = 0;
_ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL,
&summary_if_none_match, &summary_if_modified_since);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
summary_if_none_match, summary_if_modified_since,
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary from the cache. */
if (summary_not_modified)
{
g_clear_pointer (&bytes_summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
cancellable, error))
goto out;
}
}
#ifndef OSTREE_DISABLE_GPGME
if (!bytes_summary && pull_data->gpg_verify_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
#endif /* OSTREE_DISABLE_GPGME */
if (!bytes_summary && pull_data->require_static_deltas)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Fetch configured to require static deltas, but no summary found");
goto out;
}
#ifndef OSTREE_DISABLE_GPGME
if (!bytes_sig && pull_data->gpg_verify_summary)
{
g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
"GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
{
g_autoptr(OstreeGpgVerifyResult) result = NULL;
g_autoptr(GError) temp_error = NULL;
result = ostree_repo_verify_summary (self, pull_data->remote_name,
bytes_summary, bytes_sig,
cancellable, &temp_error);
if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error))
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again.
* Don’t set the cache headers in the HTTP request, to force a
* full download. */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote %s cached summary invalid and "
"OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
pull_data->remote_name);
goto out;
}
else
g_debug ("Remote %s cached summary invalid, pulling new version",
pull_data->remote_name);
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0, /* no cache headers */
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
g_autoptr(OstreeGpgVerifyResult) retry =
ostree_repo_verify_summary (self, pull_data->remote_name,
bytes_summary, bytes_sig,
cancellable, error);
if (!ostree_gpg_verify_result_require_valid_signature (retry, error))
goto out;
}
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
}
}
}
#endif /* OSTREE_DISABLE_GPGME */
if (pull_data->signapi_summary_verifiers)
{
if (!bytes_sig && pull_data->signapi_summary_verifiers)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)");
goto out;
}
if (bytes_summary && bytes_sig)
{
g_autoptr(GVariant) signatures = NULL;
g_autoptr(GError) temp_error = NULL;
signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
bytes_sig, FALSE);
g_assert (pull_data->signapi_summary_verifiers);
if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error))
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again.
* Don’t set the cache headers in the HTTP request, to force a
* full download. */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote %s cached summary invalid and "
"OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
pull_data->remote_name);
goto out;
}
else
g_debug ("Remote %s cached summary invalid, pulling new version",
pull_data->remote_name);
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0, /* no cache headers */
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error))
goto out;
}
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
}
}
}
}
if (bytes_summary)
{
pull_data->summary_data = g_bytes_ref (bytes_summary);
pull_data->summary_etag = g_strdup (summary_etag);
pull_data->summary_last_modified = summary_last_modified;
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE);
if (!g_variant_is_normal_form (pull_data->summary))
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Not normal form");
goto out;
}
if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Doesn't match variant type '%s'",
(char *)OSTREE_SUMMARY_GVARIANT_FORMAT);
goto out;
}
if (bytes_sig)
{
pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
pull_data->summary_sig_etag = g_strdup (summary_sig_etag);
pull_data->summary_sig_last_modified = summary_sig_last_modified;
}
}
if (!summary_from_cache && bytes_summary && bytes_sig)
{
if (!pull_data->remote_repo_local &&
!_ostree_repo_cache_summary (self,
remote_name_or_baseurl,
bytes_summary,
summary_etag, summary_last_modified,
bytes_sig,
summary_sig_etag, summary_sig_last_modified,
cancellable,
error))
goto out;
}
if (pull_data->summary)
{
additional_metadata = g_variant_get_child_value (pull_data->summary, 1);
if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id))
main_collection_id = NULL;
else if (!ostree_validate_collection_id (main_collection_id, error))
goto out;
refs = g_variant_get_child_value (pull_data->summary, 0);
for (i = 0, n = g_variant_n_children (refs); i < n; i++)
{
const char *refname;
g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i);
g_variant_get_child (ref, 0, "&s", &refname);
if (!ostree_validate_rev (refname, error))
goto out;
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
{
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (main_collection_id, refname), NULL);
}
}
g_autoptr(GVariant) collection_map = NULL;
collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}"));
if (collection_map != NULL)
{
GVariantIter collection_map_iter;
const char *collection_id;
g_autoptr(GVariant) collection_refs = NULL;
g_variant_iter_init (&collection_map_iter, collection_map);
while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs))
{
if (!ostree_validate_collection_id (collection_id, error))
goto out;
for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++)
{
const char *refname;
g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i);
g_variant_get_child (ref, 0, "&s", &refname);
if (!ostree_validate_rev (refname, error))
goto out;
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
{
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (collection_id, refname), NULL);
}
}
}
}
deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0;
if (!collect_available_deltas_for_pull (pull_data, deltas, error))
goto out;
}
if (pull_data->summary &&
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) &&
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits))
{
if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
goto out;
pull_data->has_tombstone_commits = tombstone_commits;
remote_mode_loaded = TRUE;
}
else if (pull_data->remote_repo_local == NULL)
{
/* Fall-back path which loads the necessary config from the remote’s
* `config` file. Doing so is deprecated since it means an
* additional round trip to the remote for each pull. No need to do
* it for local pulls. */
if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
goto out;
if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
&remote_mode_str, error))
goto out;
if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
goto out;
if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "tombstone-commits", FALSE,
&pull_data->has_tombstone_commits, error))
goto out;
remote_mode_loaded = TRUE;
}
if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Can't pull from archives with mode \"%s\"",
remote_mode_str);
goto out;
}
}
/* Compute the set of collection-refs (and optional commit id) to fetch */
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches)
{
if (!bytes_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
goto out;
}
require_summary_for_mirror = TRUE;
}
else if (opt_collection_refs_set)
{
@ -4631,6 +4216,465 @@ ostree_repo_pull_with_options (OstreeRepo *self,
}
}
/* Deltas are necessary when mirroring or resolving a requested ref to a commit.
* We try to avoid loading the potentially large summary if it is not needed. */
need_summary = require_summary_for_mirror || !all_requested_refs_have_commit (requested_refs_to_fetch) || summary_sig_bytes_v != NULL;
/* If we don't have indexed deltas, we need the summary for deltas, so check
* the config file for support.
* NOTE: Avoid download if we don't need deltas */
if (!need_summary && !pull_data->disable_static_deltas)
{
if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
goto out;
/* Check if remote has delta indexes outside summary */
if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "indexed-deltas", FALSE,
&pull_data->has_indexed_deltas, error))
goto out;
if (!pull_data->has_indexed_deltas)
need_summary = TRUE;
}
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
if (need_summary)
{
g_autoptr(GBytes) bytes_sig = NULL;
gboolean summary_sig_not_modified = FALSE;
g_autofree char *summary_sig_etag = NULL;
guint64 summary_sig_last_modified = 0;
gsize n;
g_autoptr(GVariant) refs = NULL;
g_autoptr(GVariant) deltas = NULL;
g_autoptr(GVariant) additional_metadata = NULL;
gboolean summary_from_cache = FALSE;
gboolean tombstone_commits = FALSE;
if (summary_sig_bytes_v)
{
/* Must both be specified */
g_assert (summary_bytes_v);
bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v);
bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v);
if (!bytes_sig || !bytes_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"summary-bytes or summary-sig-bytes set to invalid value");
goto out;
}
g_debug ("Loaded %s summary from options", remote_name_or_baseurl);
}
if (!bytes_sig)
{
g_autofree char *summary_sig_if_none_match = NULL;
guint64 summary_sig_if_modified_since = 0;
/* Load the summary.sig from the network, but send its ETag and
* Last-Modified from the on-disk cache (if it exists) to reduce the
* download size if nothing’s changed. */
_ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig",
&summary_sig_if_none_match, &summary_sig_if_modified_since);
g_clear_pointer (&summary_sig_etag, g_free);
summary_sig_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
summary_sig_if_none_match, summary_sig_if_modified_since,
pull_data->n_network_retries,
&bytes_sig,
&summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary.sig from the cache. Also load summary, since
* `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
if (summary_sig_not_modified)
{
g_clear_pointer (&bytes_sig, g_bytes_unref);
g_clear_pointer (&bytes_summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig",
&bytes_sig,
cancellable, error))
goto out;
if (!bytes_summary &&
!pull_data->remote_repo_local &&
!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
cancellable, error))
goto out;
}
}
if (bytes_sig &&
!bytes_summary &&
!pull_data->remote_repo_local &&
!_ostree_repo_load_cache_summary_if_same_sig (self,
remote_name_or_baseurl,
bytes_sig,
&bytes_summary,
cancellable,
error))
goto out;
if (bytes_summary && !summary_bytes_v)
{
g_debug ("Loaded %s summary from cache", remote_name_or_baseurl);
summary_from_cache = TRUE;
}
if (!pull_data->summary && !bytes_summary)
{
g_autofree char *summary_if_none_match = NULL;
guint64 summary_if_modified_since = 0;
_ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL,
&summary_if_none_match, &summary_if_modified_since);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
summary_if_none_match, summary_if_modified_since,
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary from the cache. */
if (summary_not_modified)
{
g_clear_pointer (&bytes_summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
cancellable, error))
goto out;
}
}
#ifndef OSTREE_DISABLE_GPGME
if (!bytes_summary && pull_data->gpg_verify_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
#endif /* OSTREE_DISABLE_GPGME */
if (!bytes_summary && require_summary_for_mirror)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
goto out;
}
#ifndef OSTREE_DISABLE_GPGME
if (!bytes_sig && pull_data->gpg_verify_summary)
{
g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
"GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
{
g_autoptr(OstreeGpgVerifyResult) result = NULL;
g_autoptr(GError) temp_error = NULL;
result = ostree_repo_verify_summary (self, pull_data->remote_name,
bytes_summary, bytes_sig,
cancellable, &temp_error);
if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error))
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again.
* Don’t set the cache headers in the HTTP request, to force a
* full download. */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote %s cached summary invalid and "
"OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
pull_data->remote_name);
goto out;
}
else
g_debug ("Remote %s cached summary invalid, pulling new version",
pull_data->remote_name);
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0, /* no cache headers */
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
g_autoptr(OstreeGpgVerifyResult) retry =
ostree_repo_verify_summary (self, pull_data->remote_name,
bytes_summary, bytes_sig,
cancellable, error);
if (!ostree_gpg_verify_result_require_valid_signature (retry, error))
goto out;
}
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
}
}
}
#endif /* OSTREE_DISABLE_GPGME */
if (pull_data->signapi_summary_verifiers)
{
if (!bytes_sig && pull_data->signapi_summary_verifiers)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)");
goto out;
}
if (bytes_summary && bytes_sig)
{
g_autoptr(GVariant) signatures = NULL;
g_autoptr(GError) temp_error = NULL;
signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
bytes_sig, FALSE);
g_assert (pull_data->signapi_summary_verifiers);
if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error))
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again.
* Don’t set the cache headers in the HTTP request, to force a
* full download. */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote %s cached summary invalid and "
"OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
pull_data->remote_name);
goto out;
}
else
g_debug ("Remote %s cached summary invalid, pulling new version",
pull_data->remote_name);
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0, /* no cache headers */
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error))
goto out;
}
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
}
}
}
}
if (bytes_summary)
{
pull_data->summary_data = g_bytes_ref (bytes_summary);
pull_data->summary_etag = g_strdup (summary_etag);
pull_data->summary_last_modified = summary_last_modified;
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE);
if (!g_variant_is_normal_form (pull_data->summary))
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Not normal form");
goto out;
}
if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Doesn't match variant type '%s'",
(char *)OSTREE_SUMMARY_GVARIANT_FORMAT);
goto out;
}
if (bytes_sig)
{
pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
pull_data->summary_sig_etag = g_strdup (summary_sig_etag);
pull_data->summary_sig_last_modified = summary_sig_last_modified;
}
}
if (!summary_from_cache && bytes_summary && bytes_sig)
{
if (!pull_data->remote_repo_local &&
!_ostree_repo_cache_summary (self,
remote_name_or_baseurl,
bytes_summary,
summary_etag, summary_last_modified,
bytes_sig,
summary_sig_etag, summary_sig_last_modified,
cancellable,
error))
goto out;
}
if (pull_data->summary)
{
additional_metadata = g_variant_get_child_value (pull_data->summary, 1);
if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id))
main_collection_id = NULL;
else if (!ostree_validate_collection_id (main_collection_id, error))
goto out;
refs = g_variant_get_child_value (pull_data->summary, 0);
for (i = 0, n = g_variant_n_children (refs); i < n; i++)
{
const char *refname;
g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i);
g_variant_get_child (ref, 0, "&s", &refname);
if (!ostree_validate_rev (refname, error))
goto out;
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
{
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (main_collection_id, refname), NULL);
}
}
g_autoptr(GVariant) collection_map = NULL;
collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}"));
if (collection_map != NULL)
{
GVariantIter collection_map_iter;
const char *collection_id;
g_autoptr(GVariant) collection_refs = NULL;
g_variant_iter_init (&collection_map_iter, collection_map);
while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs))
{
if (!ostree_validate_collection_id (collection_id, error))
goto out;
for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++)
{
const char *refname;
g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i);
g_variant_get_child (ref, 0, "&s", &refname);
if (!ostree_validate_rev (refname, error))
goto out;
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
{
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (collection_id, refname), NULL);
}
}
}
}
deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0;
if (!collect_available_deltas_for_pull (pull_data, deltas, error))
goto out;
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_INDEXED_DELTAS, "b", &pull_data->has_indexed_deltas);
}
if (pull_data->summary &&
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) &&
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits))
{
if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
goto out;
pull_data->has_tombstone_commits = tombstone_commits;
remote_mode_loaded = TRUE;
}
}
if (pull_data->require_static_deltas && !pull_data->has_indexed_deltas && !pull_data->summary_has_deltas)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Fetch configured to require static deltas, but no summary deltas or delta index found");
goto out;
}
if (remote_mode_loaded && pull_data->remote_repo_local == NULL)
{
/* Fall-back path which loads the necessary config from the remote’s
* `config` file (unless we already read it above). Doing so is deprecated since it means an
* additional round trip to the remote for each pull. No need to do
* it for local pulls. */
if (remote_config == NULL &&
!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
goto out;
if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
&remote_mode_str, error))
goto out;
if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
goto out;
if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "tombstone-commits", FALSE,
&pull_data->has_tombstone_commits, error))
goto out;
remote_mode_loaded = TRUE;
}
if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Can't pull from archives with mode \"%s\"",
remote_mode_str);
goto out;
}
/* Resolve the checksum for each ref. This has to be done into a new hash table,
* since we can’t modify the keys of @requested_refs_to_fetch while iterating
* over it, and we need to ensure the collection IDs are resolved too. */