pull: Add support for http-headers option

Some deployments may want to gate access to content based on things
like OAuth.  In this model, the client system would normally compute a
token and pass it to the server via an API.

We could theoretically support this in the remote config too, but
that'd be a bit weird for OAuth as the information is dynamic.
Therefore this cleans up the code a little bit to more clearly handle
the case that the fetcher is initialized from both remote config
data plus pull options.

Closes: #574
Approved by: giuseppe

Makefile-tests.am

@@ -46,6 +46,7 @@ dist_test_scripts = \
 	tests/test-archivez.sh \
 	tests/test-remote-add.sh \
 	tests/test-remote-cookies.sh \
+	tests/test-remote-headers.sh \
 	tests/test-remote-gpg-import.sh \
 	tests/test-commit-sign.sh \
 	tests/test-export.sh \

src/libostree/ostree-fetcher.c

@@ -53,6 +53,7 @@ typedef struct {
   GLnxLockFile tmpdir_lock;
   int base_tmpdir_dfd;
 
+  GVariant *extra_headers;
   int max_outstanding;
 
   /* Queue for libsoup, see bgo#708591 */
@@ -148,6 +149,8 @@ thread_closure_unref (ThreadClosure *thread_closure)
 
       g_clear_pointer (&thread_closure->main_context, g_main_context_unref);
 
+      g_clear_pointer (&thread_closure->extra_headers, (GDestroyNotify)g_variant_unref);
+
       if (thread_closure->tmpdir_dfd != -1)
         close (thread_closure->tmpdir_dfd);
 
@@ -336,6 +339,16 @@ session_thread_set_cookie_jar_cb (ThreadClosure *thread_closure,
                             SOUP_SESSION_FEATURE (jar));
 }
 
+static void
+session_thread_set_headers_cb (ThreadClosure *thread_closure,
+                               gpointer data)
+{
+  GVariant *headers = data;
+
+  g_clear_pointer (&thread_closure->extra_headers, (GDestroyNotify)g_variant_unref);
+  thread_closure->extra_headers = g_variant_ref (headers);
+}
+
 #ifdef HAVE_LIBSOUP_CLIENT_CERTS
 static void
 session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure,
@@ -448,6 +461,17 @@ session_thread_request_uri (ThreadClosure *thread_closure,
       return;
     }
 
+  if (SOUP_IS_REQUEST_HTTP (pending->request) && thread_closure->extra_headers)
+    {
+      glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
+      g_autoptr(GVariantIter) viter = g_variant_iter_new (thread_closure->extra_headers);
+      const char *key;
+      const char *value;
+
+      while (g_variant_iter_next (viter, "(&s&s)", &key, &value))
+        soup_message_headers_append (msg->request_headers, key, value);
+    }
+
   if (pending->is_stream)
     {
       soup_request_send_async (pending->request,
@@ -812,6 +836,16 @@ _ostree_fetcher_set_tls_database (OstreeFetcher *self,
     }
 }
 
+void
+_ostree_fetcher_set_extra_headers (OstreeFetcher *self,
+                                   GVariant      *extra_headers)
+{
+  session_thread_idle_add (self->thread_closure,
+                           session_thread_set_headers_cb,
+                           g_variant_ref (extra_headers),
+                           (GDestroyNotify) g_variant_unref);
+}
+
 static gboolean
 finish_stream (OstreeFetcherPendingURI *pending,
                GCancellable            *cancellable,

src/libostree/ostree-fetcher.h

@@ -71,6 +71,9 @@ void _ostree_fetcher_set_client_cert (OstreeFetcher *fetcher,
 void _ostree_fetcher_set_tls_database (OstreeFetcher *self,
                                        GTlsDatabase *db);
 
+void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
+                                        GVariant      *extra_headers);
+
 guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
 void _ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher         *self,

src/libostree/ostree-repo-pull.c

@@ -54,6 +54,8 @@ typedef struct {
   GCancellable *cancellable;
   OstreeAsyncProgress *progress;
 
+  GVariant         *extra_headers;
+
   gboolean      dry_run;
   gboolean      dry_run_emitted_progress;
   gboolean      legacy_transaction_resuming;
@@ -2276,6 +2278,23 @@ repo_remote_fetch_summary (OstreeRepo    *self,
   return ret;
 }
 
+/* Create the fetcher by unioning options from the remote config, plus
+ * any options specific to this pull (such as extra headers).
+ */
+static gboolean
+reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **error)
+{
+  g_clear_object (&pull_data->fetcher);
+  pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, error);
+  if (pull_data->fetcher == NULL)
+    return FALSE;
+
+  if (pull_data->extra_headers)
+    _ostree_fetcher_set_extra_headers (pull_data->fetcher, pull_data->extra_headers);
+
+  return TRUE;
+}
+
 /* ------------------------------------------------------------------------------------------
  * Below is the libsoup-invariant API; these should match
  * the stub functions in the #else clause
@@ -2308,6 +2327,7 @@ repo_remote_fetch_summary (OstreeRepo    *self,
  *   * dry-run (b): Only print information on what will be downloaded (requires static deltas)
  *   * override-url (s): Fetch objects from this URL if remote specifies no metalink in options
  *   * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do mutliple pulls in one transaction.
+ *   * http-headers (a(ss)): Additional headers to add to all HTTP requests
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -2368,6 +2388,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
       (void) g_variant_lookup (options, "override-url", "&s", &url_override);
       (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction);
+      (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers);
     }
 
   g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -2467,8 +2488,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
   pull_data->phase = OSTREE_PULL_PHASE_FETCHING_REFS;
 
-  pull_data->fetcher = _ostree_repo_remote_new_fetcher (self, remote_name_or_baseurl, error);
-  if (pull_data->fetcher == NULL)
+  if (!reinitialize_fetcher (pull_data, remote_name_or_baseurl, error))
     goto out;
 
   pull_data->tmpdir_dfd = pull_data->repo->tmp_dir_fd;
@@ -2906,9 +2926,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   /* Now discard the previous fetcher, as it was bound to a temporary main context
    * for synchronous requests.
    */
-  g_clear_object (&pull_data->fetcher);
-  pull_data->fetcher = _ostree_repo_remote_new_fetcher (self, remote_name_or_baseurl, error);
-  if (pull_data->fetcher == NULL)
+  if (!reinitialize_fetcher (pull_data, remote_name_or_baseurl, error))
     goto out;
 
   pull_data->legacy_transaction_resuming = FALSE;
@@ -3120,6 +3138,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_source_destroy (update_timeout);
   g_strfreev (configured_branches);
   g_clear_object (&pull_data->fetcher);
+  g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref);
   g_clear_object (&pull_data->cancellable);
   g_clear_object (&pull_data->remote_repo_local);
   g_free (pull_data->remote_name);

src/ostree/ot-builtin-pull.c

@@ -35,6 +35,7 @@ static gboolean opt_disable_static_deltas;
 static gboolean opt_require_static_deltas;
 static gboolean opt_untrusted;
 static char** opt_subpaths;
+static char** opt_http_headers;
 static char* opt_cache_dir;
 static int opt_depth = 0;
 static char* opt_url;
@@ -51,6 +52,7 @@ static GOptionEntry options[] = {
    { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL },
    { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" },
    { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL },
+   { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" },
    { NULL }
  };
 
@@ -249,6 +251,29 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
       g_variant_builder_add (&builder, "{s@v}", "override-commit-ids",
                              g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len)));
 
+    if (opt_http_headers)
+      {
+        GVariantBuilder hdr_builder;
+        g_variant_builder_init (&hdr_builder, G_VARIANT_TYPE ("a(ss)"));
+
+        for (char **iter = opt_http_headers; iter && *iter; iter++)
+          {
+            const char *kv = *iter;
+            const char *eq = strchr (kv, '=');
+            g_autofree char *key = NULL;
+            if (!eq)
+              {
+                g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                             "Missing '=' in --http-header");
+                goto out;
+              }
+            key = g_strndup (kv, eq - kv);
+            g_variant_builder_add (&hdr_builder, "(ss)", key, eq + 1);
+          }
+        g_variant_builder_add (&builder, "{s@v}", "http-headers",
+                               g_variant_new_variant (g_variant_builder_end (&hdr_builder)));
+      }
+
     if (!opt_dry_run)
       {
         if (console.is_tty)

src/ostree/ot-builtin-trivial-httpd.c

@@ -43,8 +43,8 @@ static int opt_random_500s_percentage;
  * cases involving repeated random 500s. */
 static int opt_random_500s_max = 100;
 static gint opt_port = 0;
-
 static gchar **opt_expected_cookies;
+static gchar **opt_expected_headers;
 
 static guint emitted_random_500s_count = 0;
 
@@ -64,6 +64,7 @@ static GOptionEntry options[] = {
   { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" },
   { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" },
   { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" },
+  { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" },
   { NULL }
 };
 
@@ -238,6 +239,36 @@ do_get (OtTrivialHttpd    *self,
       soup_cookies_free (cookies);
     }
 
+  if (opt_expected_headers)
+    {
+      for (int i = 0 ; opt_expected_headers[i] != NULL; i++)
+        {
+          const gchar *kv = opt_expected_headers[i];
+          const gchar *eq = strchr (kv, '=');
+
+          g_assert (eq);
+
+          {
+            g_autofree char *k = g_strndup (kv, eq - kv);
+            const gchar *expected_v = eq + 1;
+            const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k);
+
+            if (!found_v)
+              {
+                httpd_log (self, "Expected header not found %s\n", k);
+                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+                goto out;
+              }
+            if (strcmp (found_v, expected_v) != 0)
+              {
+                httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v);
+                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+                goto out;
+              }
+          }
+        }
+    }
+
   if (strstr (path, "../") != NULL)
     {
       soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);

tests/test-remote-headers.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+echo '1..2'
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive" "" \
+  "--expected-header foo=bar --expected-header baz=badger"
+
+assert_fail (){
+  set +e
+  $@
+  if [ $? = 0 ] ; then
+    echo 1>&2 "$@ did not fail"; exit 1
+  fi
+  set -euo pipefail
+}
+
+cd ${test_tmpdir}
+rm repo -rf
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+# Sanity check the setup, without headers the pull should fail
+assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main
+
+echo "ok, setup done"
+
+# Now pull should succeed now
+${CMD_PREFIX} ostree --repo=repo pull --http-header foo=bar --http-header baz=badger origin main
+
+echo "ok, pull succeeded"