fetcher: Define an abstraction over SoupURI

This is preparatory work for a potential libcurl backend.

Closes: #616
Approved by: jlebon
This commit is contained in:
Colin Walters 2016-12-06 11:34:05 -05:00 committed by Atomic Bot
parent 099576ee4a
commit 7519457382
5 changed files with 186 additions and 103 deletions

View File

@ -24,6 +24,10 @@
#include <gio/gfiledescriptorbased.h>
#include <gio/gunixoutputstream.h>
#define LIBSOUP_USE_UNSTABLE_REQUEST_API
#include <libsoup/soup.h>
#include <libsoup/soup-requester.h>
#include <libsoup/soup-request-http.h>
#include "libglnx.h"
#include "ostree-fetcher.h"
@ -418,25 +422,20 @@ static void
create_pending_soup_request (OstreeFetcherPendingURI *pending,
GError **error)
{
g_autofree char *uristr = NULL;
SoupURI *next_mirror = NULL;
SoupURI *uri = NULL;
OstreeFetcherURI *next_mirror = NULL;
g_autoptr(OstreeFetcherURI) uri = NULL;
g_assert (pending->mirrorlist);
g_assert (pending->mirrorlist_idx < pending->mirrorlist->len);
next_mirror = g_ptr_array_index (pending->mirrorlist,
pending->mirrorlist_idx);
uristr = g_build_filename (soup_uri_get_path (next_mirror),
pending->filename /* may be NULL */, NULL);
uri = soup_uri_copy (next_mirror);
soup_uri_set_path (uri, uristr);
next_mirror = g_ptr_array_index (pending->mirrorlist, pending->mirrorlist_idx);
if (pending->filename)
uri = _ostree_fetcher_uri_new_subpath (next_mirror, pending->filename);
g_clear_object (&pending->request);
pending->request = soup_session_request_uri (pending->thread_closure->session,
uri, error);
soup_uri_free (uri);
(SoupURI*)(uri ? uri : next_mirror), error);
}
static void
@ -1404,7 +1403,7 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
/* Helper for callers who just want to fetch single one-off URIs */
gboolean
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
SoupURI *uri,
OstreeFetcherURI *uri,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,
@ -1419,3 +1418,84 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
out_contents, max_size,
cancellable, error);
}
void
_ostree_fetcher_uri_free (OstreeFetcherURI *uri)
{
if (uri)
soup_uri_free ((SoupURI*)uri);
}
OstreeFetcherURI *
_ostree_fetcher_uri_parse (const char *str,
GError **error)
{
SoupURI *soupuri = soup_uri_new (str);
if (soupuri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse uri: %s", str);
return NULL;
}
return (OstreeFetcherURI*)soupuri;
}
static OstreeFetcherURI *
_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri,
gboolean extend,
const char *path)
{
SoupURI *newuri = soup_uri_copy ((SoupURI*)uri);
if (path)
{
if (extend)
{
const char *origpath = soup_uri_get_path ((SoupURI*)uri);
g_autofree char *newpath = g_build_filename (origpath, path, NULL);
soup_uri_set_path (newuri, newpath);
}
else
{
soup_uri_set_path (newuri, path);
}
}
return (OstreeFetcherURI*)newuri;
}
OstreeFetcherURI *
_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri,
const char *path)
{
return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path);
}
OstreeFetcherURI *
_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri,
const char *subpath)
{
return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath);
}
OstreeFetcherURI *
_ostree_fetcher_uri_clone (OstreeFetcherURI *uri)
{
return _ostree_fetcher_uri_new_subpath (uri, NULL);
}
char *
_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri)
{
return g_strdup (soup_uri_get_scheme ((SoupURI*)uri));
}
char *
_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri)
{
return g_strdup (soup_uri_get_path ((SoupURI*)uri));
}
char *
_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri)
{
return soup_uri_to_string ((SoupURI*)uri, FALSE);
}

View File

@ -22,10 +22,7 @@
#ifndef __GI_SCANNER__
#define LIBSOUP_USE_UNSTABLE_REQUEST_API
#include <libsoup/soup.h>
#include <libsoup/soup-requester.h>
#include <libsoup/soup-request-http.h>
#include "libglnx.h"
G_BEGIN_DECLS
@ -39,6 +36,8 @@ G_BEGIN_DECLS
/* Lower values have higher priority */
#define OSTREE_FETCHER_DEFAULT_PRIORITY 0
typedef struct OstreeFetcherURI OstreeFetcherURI;
typedef struct OstreeFetcherClass OstreeFetcherClass;
typedef struct OstreeFetcher OstreeFetcher;
@ -52,6 +51,34 @@ typedef enum {
OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0)
} OstreeFetcherConfigFlags;
void
_ostree_fetcher_uri_free (OstreeFetcherURI *uri);
G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeFetcherURI, _ostree_fetcher_uri_free)
OstreeFetcherURI *
_ostree_fetcher_uri_parse (const char *str,
GError **error);
OstreeFetcherURI *
_ostree_fetcher_uri_clone (OstreeFetcherURI *uri);
OstreeFetcherURI *
_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri,
const char *subpath);
OstreeFetcherURI *
_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri,
const char *subpath);
char *
_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri);
char *
_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri);
char *
_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri);
GType _ostree_fetcher_get_type (void) G_GNUC_CONST;
OstreeFetcher *_ostree_fetcher_new (int tmpdir_dfd,
@ -100,7 +127,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GError **error);
gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
SoupURI *uri,
OstreeFetcherURI *uri,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,

View File

@ -43,7 +43,7 @@ struct OstreeMetalink
{
GObject parent_instance;
SoupURI *uri;
OstreeFetcherURI *uri;
OstreeFetcher *fetcher;
char *requested_file;
@ -357,7 +357,7 @@ metalink_parser_text (GMarkupParseContext *context,
case OSTREE_METALINK_STATE_URL:
{
g_autofree char *uri_text = g_strndup (text, text_len);
SoupURI *uri = soup_uri_new (uri_text);
OstreeFetcherURI *uri = _ostree_fetcher_uri_parse (uri_text, NULL);
if (uri != NULL)
g_ptr_array_add (self->urls, uri);
}
@ -377,7 +377,7 @@ _ostree_metalink_finalize (GObject *object)
g_object_unref (self->fetcher);
g_free (self->requested_file);
soup_uri_free (self->uri);
_ostree_fetcher_uri_free (self->uri);
G_OBJECT_CLASS (_ostree_metalink_parent_class)->finalize (object);
}
@ -399,14 +399,14 @@ OstreeMetalink *
_ostree_metalink_new (OstreeFetcher *fetcher,
const char *requested_file,
guint64 max_size,
SoupURI *uri)
OstreeFetcherURI *uri)
{
OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL);
self->fetcher = g_object_ref (fetcher);
self->requested_file = g_strdup (requested_file);
self->max_size = max_size;
self->uri = soup_uri_copy (uri);
self->uri = _ostree_fetcher_uri_clone (uri);
return self;
}
@ -421,7 +421,7 @@ valid_hex_checksum (const char *s, gsize expected_len)
static gboolean
try_one_url (OstreeMetalinkRequest *self,
SoupURI *uri,
OstreeFetcherURI *uri,
GBytes **out_data,
GError **error)
{
@ -486,12 +486,12 @@ try_one_url (OstreeMetalinkRequest *self,
static gboolean
try_metalink_targets (OstreeMetalinkRequest *self,
SoupURI **out_target_uri,
OstreeFetcherURI **out_target_uri,
GBytes **out_data,
GError **error)
{
gboolean ret = FALSE;
SoupURI *target_uri = NULL;
OstreeFetcherURI *target_uri = NULL;
g_autoptr(GBytes) ret_data = NULL;
if (!self->found_a_file_element)
@ -568,7 +568,7 @@ try_metalink_targets (OstreeMetalinkRequest *self,
ret = TRUE;
if (out_target_uri)
*out_target_uri = soup_uri_copy (target_uri);
*out_target_uri = _ostree_fetcher_uri_clone (target_uri);
if (out_data)
*out_data = g_steal_pointer (&ret_data);
out:
@ -585,7 +585,7 @@ static const GMarkupParser metalink_parser = {
typedef struct
{
SoupURI **out_target_uri;
OstreeFetcherURI **out_target_uri;
GBytes **out_data;
gboolean success;
GError **error;
@ -594,7 +594,7 @@ typedef struct
gboolean
_ostree_metalink_request_sync (OstreeMetalink *self,
SoupURI **out_target_uri,
OstreeFetcherURI **out_target_uri,
GBytes **out_data,
GCancellable *cancellable,
GError **error)
@ -610,7 +610,7 @@ _ostree_metalink_request_sync (OstreeMetalink *self,
g_main_context_push_thread_default (mainctx);
request.metalink = g_object_ref (self);
request.urls = g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
request.urls = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL);
if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher,

View File

@ -46,10 +46,10 @@ GType _ostree_metalink_get_type (void) G_GNUC_CONST;
OstreeMetalink *_ostree_metalink_new (OstreeFetcher *fetcher,
const char *requested_file,
guint64 max_size,
SoupURI *uri);
OstreeFetcherURI *uri);
gboolean _ostree_metalink_request_sync (OstreeMetalink *self,
SoupURI **out_target_uri,
OstreeFetcherURI **out_target_uri,
GBytes **out_data,
GCancellable *cancellable,
GError **error);

View File

@ -353,7 +353,7 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
static gboolean
fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
SoupURI *uri,
OstreeFetcherURI *uri,
char **out_contents,
GCancellable *cancellable,
GError **error)
@ -2055,17 +2055,13 @@ fetch_mirrorlist (OstreeFetcher *fetcher,
gboolean ret = FALSE;
g_auto(GStrv) lines = NULL;
g_autofree char *contents = NULL;
SoupURI *mirrorlist = NULL;
g_autoptr(OstreeFetcherURI) mirrorlist = NULL;
g_autoptr(GPtrArray) ret_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
mirrorlist = soup_uri_new (mirrorlist_url);
if (mirrorlist == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse mirrorlist URL '%s'", mirrorlist_url);
mirrorlist = _ostree_fetcher_uri_parse (mirrorlist_url, error);
if (!mirrorlist)
goto out;
}
if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents,
cancellable, error))
@ -2083,27 +2079,28 @@ fetch_mirrorlist (OstreeFetcher *fetcher,
for (char **iter = lines; iter && *iter; iter++)
{
const char *mirror_uri_str = *iter;
SoupURI *mirror_uri = NULL;
g_autoptr(OstreeFetcherURI) mirror_uri = NULL;
g_autofree char *scheme = NULL;
/* let's be nice and support empty lines and comments */
if (*mirror_uri_str == '\0' || *mirror_uri_str == '#')
continue;
mirror_uri = soup_uri_new (mirror_uri_str);
if (mirror_uri == NULL)
mirror_uri = _ostree_fetcher_uri_parse (mirror_uri_str, NULL);
if (!mirror_uri)
{
g_debug ("Can't parse mirrorlist line '%s'", mirror_uri_str);
continue;
}
else if ((strcmp (soup_uri_get_scheme (mirror_uri), "http") != 0) &&
(strcmp (soup_uri_get_scheme (mirror_uri), "https") != 0))
scheme = _ostree_fetcher_uri_get_scheme (mirror_uri);
if (!(g_str_equal (scheme, "http") || (g_str_equal (scheme, "https"))))
{
/* let's not support mirrorlists that contain non-http based URIs for
* now (e.g. local URIs) -- we need to think about if and how we want
* to support this since we set up things differently depending on
* whether we're pulling locally or not */
g_debug ("Ignoring non-http/s mirrorlist entry '%s'", mirror_uri_str);
soup_uri_free (mirror_uri);
continue;
}
@ -2114,9 +2111,7 @@ fetch_mirrorlist (OstreeFetcher *fetcher,
if (ret_mirrorlist->len == 0)
{
GError *local_error = NULL;
g_autofree char *config_uri_str = g_build_filename (mirror_uri_str,
"config", NULL);
SoupURI *config_uri = soup_uri_new (config_uri_str);
g_autoptr(OstreeFetcherURI) config_uri = _ostree_fetcher_uri_new_subpath (mirror_uri, "config");
if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL,
cancellable, &local_error))
@ -2127,16 +2122,11 @@ fetch_mirrorlist (OstreeFetcher *fetcher,
mirror_uri_str, local_error->message);
g_clear_error (&local_error);
}
soup_uri_free (config_uri);
}
else
{
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
}
if (mirror_uri != NULL)
soup_uri_free (mirror_uri);
}
if (ret_mirrorlist->len == 0)
@ -2151,8 +2141,6 @@ fetch_mirrorlist (OstreeFetcher *fetcher,
ret = TRUE;
out:
if (mirrorlist != NULL)
soup_uri_free (mirrorlist);
return ret;
}
@ -2201,18 +2189,14 @@ repo_remote_fetch_summary (OstreeRepo *self,
}
else
{
SoupURI *uri = soup_uri_new (url_string);
g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (url_string, error);
if (uri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", url_string);
if (!uri)
goto out;
}
mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (mirrorlist, uri /* transfer ownership */ );
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (mirrorlist, g_steal_pointer (&uri));
}
}
@ -2522,36 +2506,27 @@ ostree_repo_pull_with_options (OstreeRepo *self,
}
else
{
SoupURI *baseuri = soup_uri_new (baseurl);
g_autoptr(OstreeFetcherURI) baseuri = _ostree_fetcher_uri_parse (baseurl, error);
if (baseuri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", baseurl);
if (!baseuri)
goto out;
}
pull_data->meta_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, baseuri /* transfer */);
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, g_steal_pointer (&baseuri));
}
}
else
{
g_autoptr(GBytes) summary_bytes = NULL;
SoupURI *metalink_uri = soup_uri_new (metalink_url_str);
SoupURI *target_uri = NULL;
g_autoptr(OstreeFetcherURI) metalink_uri = _ostree_fetcher_uri_parse (metalink_url_str, error);
g_autoptr(OstreeFetcherURI) target_uri = NULL;
if (!metalink_uri)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid metalink URL: %s", metalink_url_str);
goto out;
}
metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
OSTREE_MAX_METADATA_SIZE, metalink_uri);
soup_uri_free (metalink_uri);
if (! _ostree_metalink_request_sync (metalink,
&target_uri,
@ -2564,12 +2539,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
* mirrors here since we use it as such anyway (rather than the "usual"
* use case of metalink, which is only for a single target filename) */
{
/* reuse target_uri and take ownership */
g_autofree char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
soup_uri_set_path (target_uri, repo_base);
g_autofree char *path = _ostree_fetcher_uri_get_path (target_uri);
g_autofree char *basepath = g_path_get_dirname (path);
g_autoptr(OstreeFetcherURI) new_target_uri = _ostree_fetcher_uri_new_path (target_uri, basepath);
pull_data->meta_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, target_uri);
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, g_steal_pointer (&new_target_uri));
}
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
@ -2603,19 +2578,15 @@ ostree_repo_pull_with_options (OstreeRepo *self,
}
else
{
SoupURI *contenturi = soup_uri_new (contenturl);
g_autoptr(OstreeFetcherURI) contenturi = _ostree_fetcher_uri_parse (contenturl, error);
if (contenturi == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse contenturl '%s'", contenturl);
if (!contenturi)
goto out;
}
pull_data->content_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (pull_data->content_mirrorlist,
contenturi /* transfer */);
g_steal_pointer (&contenturi));
}
}
}
@ -2625,12 +2596,16 @@ ostree_repo_pull_with_options (OstreeRepo *self,
&configured_branches, error))
goto out;
/* TODO reindent later */
{ OstreeFetcherURI *first_uri = pull_data->meta_mirrorlist->pdata[0];
g_autofree char *first_scheme = _ostree_fetcher_uri_get_scheme (first_uri);
/* NB: we don't support local mirrors in mirrorlists, so if this passes, it
* means that we're not using mirrorlists (see also fetch_mirrorlist()) */
if (strcmp (soup_uri_get_scheme (pull_data->meta_mirrorlist->pdata[0]), "file") == 0)
if (g_str_equal (first_scheme, "file"))
{
g_autoptr(GFile) remote_repo_path =
g_file_new_for_path (soup_uri_get_path (pull_data->meta_mirrorlist->pdata[0]));
g_autofree char *path = _ostree_fetcher_uri_get_path (first_uri);
g_autoptr(GFile) remote_repo_path = g_file_new_for_path (path);
pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
goto out;
@ -2659,6 +2634,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
goto out;
}
}
}
/* For local pulls, default to disabling static deltas so that the
* exact object files are copied.