Add repository "summary" file and metalink support

For Fedora and potentially other distributions which use globally
distributed mirrors, metalink is a popular solution to redirect
clients to a dynamic set of mirrors.

In order to make metalink work though, it needs *one* file which can
be checksummed.  (Well, potentially we could explode all refs into the
metalink.xml, but that would be a lot more invasive, and a bit weird
as we'd end up checksumming the checksum file).

This commit adds a new command:

$ ostree summary -u

To regenerate the summary file.  Can only be run by one process at a
time.

After that's done, the metalink can be generated based on it, and the
client fetch code will parse and load it.

https://bugzilla.gnome.org/show_bug.cgi?id=729585

Makefile-libostree.am

@@ -112,6 +112,8 @@ if USE_LIBSOUP
 libostree_1_la_SOURCES += \
 	src/libostree/ostree-fetcher.h \
 	src/libostree/ostree-fetcher.c \
+	src/libostree/ostree-metalink.h \
+	src/libostree/ostree-metalink.c \
 	src/libostree/ostree-repo-pull.c \
 	$(NULL)
 libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS)

Makefile-ostree.am

@@ -40,6 +40,7 @@ ostree_SOURCES = src/ostree/main.c \
 	src/ostree/ot-builtin-remote.c \
 	src/ostree/ot-builtin-reset.c \
 	src/ostree/ot-builtin-rev-parse.c \
+	src/ostree/ot-builtin-summary.c \
 	src/ostree/ot-builtin-show.c \
 	src/ostree/ot-builtin-static-delta.c \
 	src/ostree/ot-main.h \

Makefile-tests.am

@@ -31,6 +31,7 @@ testfiles = test-basic \
 	test-pull-archive-z \
 	test-pull-corruption \
 	test-pull-large-metadata \
+	test-pull-metalink \
 	test-pull-resume \
 	test-gpg-signed-commit \
 	test-admin-deploy-syslinux \

src/libostree/ostree-core.h

@@ -117,6 +117,15 @@ typedef enum {
 #define OSTREE_COMMIT_GVARIANT_STRING "(a{sv}aya(say)sstayay)"
 #define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_COMMIT_GVARIANT_STRING)
 
+/**
+ * OSTREE_SUMMARY_GVARIANT_FORMAT:
+ *
+ * refs: a(s(taya{sv})) - Map of ref name -> (latest commit size, latest commit checksum, additional metadata), sorted by ref name
+ * extensions: a{sv} - Additional metadata, none defined at the current time
+ */
+#define OSTREE_SUMMARY_GVARIANT_STRING "(a(s(taya{sv}))a{sv})"
+#define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING)
+
 /**
  * OstreeRepoMode:
  * @OSTREE_REPO_MODE_BARE: Files are stored as themselves; can only be written as root

src/libostree/ostree-metalink.c

@@ -0,0 +1,710 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2014 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ostree-metalink.h"
+
+#include "otutil.h"
+#include "libgsystem.h"
+
+typedef enum {
+  OSTREE_METALINK_STATE_INITIAL,
+  OSTREE_METALINK_STATE_METALINK,
+  OSTREE_METALINK_STATE_FILES,
+  OSTREE_METALINK_STATE_FILE,
+  OSTREE_METALINK_STATE_SIZE,
+  OSTREE_METALINK_STATE_VERIFICATION,
+  OSTREE_METALINK_STATE_HASH,
+  OSTREE_METALINK_STATE_RESOURCES,
+  OSTREE_METALINK_STATE_URL,
+
+  OSTREE_METALINK_STATE_PASSTHROUGH /* Ignoring unknown elements */
+} OstreeMetalinkState;
+
+struct OstreeMetalink
+{
+  GObject parent_instance;
+
+  SoupURI *uri;
+
+  OstreeFetcher *fetcher;
+  char *requested_file;
+  guint64 max_size;
+};
+
+G_DEFINE_TYPE (OstreeMetalink, _ostree_metalink, G_TYPE_OBJECT)
+
+typedef struct
+{
+  OstreeMetalink *metalink;
+
+  GTask *task;
+  GMarkupParseContext *parser;
+
+  guint passthrough_depth;
+  OstreeMetalinkState passthrough_previous;
+  
+  guint found_a_file_element : 1;
+  guint found_our_file_element : 1;
+  guint verification_known : 1;
+
+  GChecksumType in_verification_type;
+
+  guint64 size;
+  char *verification_sha256;
+  char *verification_sha512;
+
+  GFile *result;
+
+  char *last_metalink_error;
+  guint current_url_index;
+  GPtrArray *urls;
+
+  OstreeMetalinkState state;
+} OstreeMetalinkRequest;
+
+static void
+state_transition (OstreeMetalinkRequest  *self,
+                  OstreeMetalinkState     new_state)
+{
+  g_assert (self->state != new_state);
+  self->state = new_state;
+}
+
+static void
+unknown_element (OstreeMetalinkRequest         *self,
+                 const char                    *element_name,
+                 GError                       **error)
+{
+  state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+  g_assert (self->passthrough_depth == 0);
+}
+
+static void
+metalink_parser_start (GMarkupParseContext  *context,
+                       const gchar          *element_name,
+                       const gchar         **attribute_names,
+                       const gchar         **attribute_values,
+                       gpointer              user_data,
+                       GError              **error)
+{
+  GTask *task = user_data;
+  OstreeMetalinkRequest *self = g_task_get_task_data (task);
+
+  switch (self->state)
+    {
+    case OSTREE_METALINK_STATE_INITIAL:
+      if (strcmp (element_name, "metalink") == 0)
+        state_transition (self, OSTREE_METALINK_STATE_METALINK);
+      else
+        unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_METALINK:
+      if (strcmp (element_name, "files") == 0)
+        state_transition (self, OSTREE_METALINK_STATE_FILES);
+      else
+        unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_FILES:
+      /* If we've already processed a <file> element we're OK with, just
+       * ignore the others.
+       */
+      if (self->urls->len > 0)
+        {
+          state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+        }
+      else if (strcmp (element_name, "file") == 0)
+        {
+          const char *file_name;
+
+          if (!g_markup_collect_attributes (element_name,
+                                            attribute_names,
+                                            attribute_values,
+                                            error,
+                                            G_MARKUP_COLLECT_STRING,
+                                            "name",
+                                            &file_name,
+                                            G_MARKUP_COLLECT_INVALID))
+            goto out;
+
+          self->found_a_file_element = TRUE;
+
+          if (strcmp (file_name, self->metalink->requested_file) != 0)
+            {
+              state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+              g_assert (self->passthrough_depth == 0);
+            }
+          else
+            {
+              self->found_our_file_element = TRUE;
+              state_transition (self, OSTREE_METALINK_STATE_FILE);
+            }
+        }
+      else
+        unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_FILE:
+      if (strcmp (element_name, "size") == 0)
+        state_transition (self, OSTREE_METALINK_STATE_SIZE);
+      else if (strcmp (element_name, "verification") == 0)
+        state_transition (self, OSTREE_METALINK_STATE_VERIFICATION);
+      else if (strcmp (element_name, "resources") == 0)
+        state_transition (self, OSTREE_METALINK_STATE_RESOURCES);
+      else
+        unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_SIZE:
+      unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_VERIFICATION:
+      if (strcmp (element_name, "hash") == 0)
+        {
+           char *verification_type_str = NULL;
+
+          state_transition (self, OSTREE_METALINK_STATE_HASH);
+          if (!g_markup_collect_attributes (element_name,
+                                            attribute_names,
+                                            attribute_values,
+                                            error,
+                                            G_MARKUP_COLLECT_STRING,
+                                            "type",
+                                            &verification_type_str,
+                                            G_MARKUP_COLLECT_INVALID))
+            goto out;
+
+          /* Only accept sha256/sha512 */
+          self->verification_known = TRUE;
+          if (strcmp (verification_type_str, "sha256") == 0)
+            self->in_verification_type = G_CHECKSUM_SHA256;
+          else if (strcmp (verification_type_str, "sha512") == 0)
+            self->in_verification_type = G_CHECKSUM_SHA512;
+          else
+            self->verification_known = FALSE;
+        }
+      else
+        unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_HASH:
+      unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_RESOURCES:
+      if (self->size == 0)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "No <size> element found or it is zero");
+          goto out;
+        }
+      if (!self->verification_known)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "No <verification> element with known <hash type=> found");
+          goto out;
+        }
+
+      if (strcmp (element_name, "url") == 0)
+        {
+          const char *protocol;
+
+          if (!g_markup_collect_attributes (element_name,
+                                            attribute_names,
+                                            attribute_values,
+                                            error,
+                                            G_MARKUP_COLLECT_STRING,
+                                            "protocol",
+                                            &protocol,
+                                            G_MARKUP_COLLECT_STRING,
+                                            "type",
+                                            NULL,
+                                            G_MARKUP_COLLECT_STRING,
+                                            "location",
+                                            NULL,
+                                            G_MARKUP_COLLECT_STRING,
+                                            "preference",
+                                            NULL,
+                                            G_MARKUP_COLLECT_INVALID))
+            goto out;
+
+          /* Ignore non-HTTP resources */
+          if (!(strcmp (protocol, "http") == 0 || strcmp (protocol, "https") == 0))
+            state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+          else
+            state_transition (self, OSTREE_METALINK_STATE_URL);
+        }
+      else
+        unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_URL:
+      unknown_element (self, element_name, error);
+      break;
+    case OSTREE_METALINK_STATE_PASSTHROUGH:
+      self->passthrough_depth++;
+      break;
+    }
+
+ out:
+  return;
+}
+
+static void
+metalink_parser_end (GMarkupParseContext  *context,
+                     const gchar          *element_name,
+                     gpointer              user_data,
+                     GError              **error)
+{
+  GTask *task = user_data;
+  OstreeMetalinkRequest *self = g_task_get_task_data (task);
+
+  switch (self->state)
+    {
+    case OSTREE_METALINK_STATE_INITIAL:
+      break;
+    case OSTREE_METALINK_STATE_METALINK:
+      state_transition (self, OSTREE_METALINK_STATE_INITIAL);
+      break;
+    case OSTREE_METALINK_STATE_FILES:
+      state_transition (self, OSTREE_METALINK_STATE_METALINK);
+      break;
+    case OSTREE_METALINK_STATE_FILE:
+      state_transition (self, OSTREE_METALINK_STATE_FILES);
+      break;
+    case OSTREE_METALINK_STATE_SIZE:
+    case OSTREE_METALINK_STATE_VERIFICATION:
+    case OSTREE_METALINK_STATE_RESOURCES:
+      state_transition (self, OSTREE_METALINK_STATE_FILE);
+      break;
+    case OSTREE_METALINK_STATE_HASH:
+      state_transition (self, OSTREE_METALINK_STATE_VERIFICATION);
+      break;
+    case OSTREE_METALINK_STATE_URL:
+      state_transition (self, OSTREE_METALINK_STATE_RESOURCES);
+      break;
+    case OSTREE_METALINK_STATE_PASSTHROUGH:
+      g_assert_cmpint (self->passthrough_depth, >, 0);
+      self->passthrough_depth--;
+      if (self->passthrough_depth == 0)
+        state_transition (self, self->passthrough_previous);
+      break;
+    }
+}
+
+static void
+metalink_parser_text (GMarkupParseContext *context,
+                      const gchar         *text,
+                      gsize                text_len,
+                      gpointer             user_data,
+                      GError             **error)
+{
+  GTask *task = user_data;
+  OstreeMetalinkRequest *self = g_task_get_task_data (task);
+
+  switch (self->state)
+    {
+    case OSTREE_METALINK_STATE_INITIAL:
+      break;
+    case OSTREE_METALINK_STATE_METALINK:
+      break;
+    case OSTREE_METALINK_STATE_FILES:
+      break;
+    case OSTREE_METALINK_STATE_FILE:
+      break;
+    case OSTREE_METALINK_STATE_SIZE:
+      {
+        gs_free char *duped = g_strndup (text, text_len);
+        self->size = g_ascii_strtoull (duped, NULL, 10);
+      }
+      break;
+    case OSTREE_METALINK_STATE_VERIFICATION:
+      break;
+    case OSTREE_METALINK_STATE_HASH:
+      if (self->verification_known)
+        {
+          switch (self->in_verification_type)
+            {
+            case G_CHECKSUM_SHA256:
+              g_free (self->verification_sha256);
+              self->verification_sha256 = g_strndup (text, text_len);
+              break;
+            case G_CHECKSUM_SHA512:
+              g_free (self->verification_sha512);
+              self->verification_sha512 = g_strndup (text, text_len);
+              break;
+            default:
+              g_assert_not_reached ();
+            }
+        }
+      break;
+    case OSTREE_METALINK_STATE_RESOURCES:
+      break;
+    case OSTREE_METALINK_STATE_URL:
+      {
+        gs_free char *uri_text = g_strndup (text, text_len);
+        SoupURI *uri = soup_uri_new (uri_text);
+        if (uri != NULL)
+          g_ptr_array_add (self->urls, uri);
+      }
+      break;
+    case OSTREE_METALINK_STATE_PASSTHROUGH:
+      break;
+    }
+
+}
+
+static void
+_ostree_metalink_finalize (GObject *object)
+{
+  OstreeMetalink *self;
+
+  self = OSTREE_METALINK (object);
+
+  g_object_unref (self->fetcher);
+  g_free (self->requested_file);
+  soup_uri_free (self->uri);
+
+  G_OBJECT_CLASS (_ostree_metalink_parent_class)->finalize (object);
+}
+
+static void
+_ostree_metalink_class_init (OstreeMetalinkClass *klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+
+  gobject_class->finalize = _ostree_metalink_finalize;
+}
+
+static void
+_ostree_metalink_init (OstreeMetalink *self)
+{
+}
+
+OstreeMetalink *
+_ostree_metalink_new (OstreeFetcher  *fetcher,
+                      const char     *requested_file,
+                      guint64         max_size,
+                      SoupURI        *uri)
+{
+  OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL);
+
+  self->fetcher = g_object_ref (fetcher);
+  self->requested_file = g_strdup (requested_file);
+  self->max_size = max_size;
+  self->uri = soup_uri_copy (uri);
+ 
+  return self;
+}
+
+static void
+try_next_url (OstreeMetalinkRequest          *self);
+
+static gboolean
+valid_hex_checksum (const char *s, gsize expected_len)
+{
+  gsize len = strspn (s, "01234567890abcdef");
+
+  return len == expected_len && s[len] == '\0';
+}
+
+static void
+on_fetched_url (GObject              *src,
+                GAsyncResult         *res,
+                gpointer              user_data)
+{
+  GTask *task = user_data;
+  OstreeMetalinkRequest *self = g_task_get_task_data (task);
+  GError *local_error = NULL;
+  gs_unref_object GFile *result = NULL;
+  gs_unref_object GFileInfo *finfo = NULL;
+
+  result = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)src, res, &local_error);
+  if (!result)
+    goto out;
+  
+  finfo = g_file_query_info (result, OSTREE_GIO_FAST_QUERYINFO, 0,
+                             g_task_get_cancellable (task), &local_error);
+  if (!finfo)
+    goto out;
+
+  if (g_file_info_get_size (finfo) != self->size)
+    {
+      g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Expected size is %" G_GUINT64_FORMAT " bytes but content is %" G_GUINT64_FORMAT " bytes",
+                   self->size, g_file_info_get_size (finfo));
+      goto out;
+    }
+  
+  if (self->verification_sha512)
+    {
+      gs_free char *actual = ot_checksum_file (result, G_CHECKSUM_SHA512,
+                                               g_task_get_cancellable (task),
+                                               &local_error);
+      
+      if (!actual)
+        goto out;
+
+      if (strcmp (self->verification_sha512, actual) != 0)
+        {
+          g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Expected checksum is %s but actual is %s",
+                       self->verification_sha512, actual);
+          goto out;
+        }
+    }
+
+  if (self->verification_sha256)
+    {
+      gs_free char *actual = ot_checksum_file (result, G_CHECKSUM_SHA256,
+                                               g_task_get_cancellable (task),
+                                               &local_error);
+      
+      if (!actual)
+        goto out;
+
+      if (strcmp (self->verification_sha256, actual) != 0)
+        {
+          g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Expected checksum is %s but actual is %s",
+                       self->verification_sha256, actual);
+          goto out;
+        }
+    }
+
+ out:
+  if (local_error)
+    {
+      g_free (self->last_metalink_error);
+      self->last_metalink_error = g_strdup (local_error->message);
+      g_clear_error (&local_error);
+
+      /* And here we iterate on the next one if we hit an error */
+      self->current_url_index++;
+      try_next_url (self);
+    }
+  else
+    {
+      self->result = g_object_ref (result);
+      g_task_return_boolean (self->task, TRUE);
+    }
+}
+
+static void
+try_next_url (OstreeMetalinkRequest          *self)
+{
+  if (self->current_url_index >= self->urls->len)
+    {
+      g_task_return_new_error (self->task, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Exhausted %u metalink targets, last error: %s",
+                               self->urls->len, self->last_metalink_error);
+    }
+  else
+    {
+      SoupURI *next = self->urls->pdata[self->current_url_index];
+      
+      _ostree_fetcher_request_uri_with_partial_async (self->metalink->fetcher, next,
+                                                      self->metalink->max_size,
+                                                      g_task_get_cancellable (self->task),
+                                                      on_fetched_url, self->task);
+    }
+}
+
+static gboolean
+start_target_request_phase (OstreeMetalinkRequest      *self,
+                            GError                    **error)
+{
+  gboolean ret = FALSE;
+
+  if (!self->found_a_file_element)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No <file> element found");
+      goto out;
+    }
+
+  if (!self->found_our_file_element)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No <file name='%s'> found", self->metalink->requested_file);
+      goto out;
+    }
+
+  if (!(self->verification_sha256 || self->verification_sha512))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No <verification> hash for sha256 or sha512 found");
+      goto out;
+    }
+
+  if (self->verification_sha256 && !valid_hex_checksum (self->verification_sha256, 64))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid hash digest for sha256");
+      goto out;
+    }
+
+  if (self->verification_sha512 && !valid_hex_checksum (self->verification_sha512, 128))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid hash digest for sha512");
+      goto out;
+    }
+
+  if (self->urls->len == 0)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No <url method='http'> elements found");
+      goto out;
+    }
+
+  try_next_url (self);
+  
+  ret = TRUE;
+ out:
+  return ret;
+}
+
+static void
+on_metalink_bytes_read (GObject           *src,
+                        GAsyncResult      *result,
+                        gpointer           user_data)
+{
+  GError *local_error = NULL;
+  GTask *task = user_data;
+  OstreeMetalinkRequest *self = g_task_get_task_data (task);
+  gs_unref_bytes GBytes *bytes = NULL;
+  gsize len;
+  const guint8 *data;
+
+  bytes = g_input_stream_read_bytes_finish ((GInputStream*)src,
+                                            result, &local_error);
+  if (!bytes)
+    goto out;
+  
+  data = g_bytes_get_data (bytes, &len);
+
+  if (len == 0)
+    {
+      if (!start_target_request_phase (self, &local_error))
+        goto out;
+    }
+  else
+    {
+      if (!g_markup_parse_context_parse (self->parser, (const char*)data, len, &local_error))
+        goto out;
+
+      g_input_stream_read_bytes_async ((GInputStream*)src, 8192, G_PRIORITY_DEFAULT,
+                                       g_task_get_cancellable (task),
+                                       on_metalink_bytes_read, task);
+    }
+
+ out:
+  if (local_error)
+    g_task_return_error (task, local_error);
+}
+
+static void
+on_retrieved_metalink (GObject           *src,
+                       GAsyncResult      *result,
+                       gpointer           user_data)
+{
+  GError *local_error = NULL;
+  GTask *task = user_data;
+  gs_unref_object GInputStream *metalink_stream = NULL;
+
+  metalink_stream = _ostree_fetcher_stream_uri_finish ((OstreeFetcher*)src, result, &local_error);
+  if (!metalink_stream)
+    goto out;
+
+  g_input_stream_read_bytes_async (metalink_stream, 8192, G_PRIORITY_DEFAULT,
+                                   g_task_get_cancellable (task),
+                                   on_metalink_bytes_read, task);
+
+ out:
+  if (local_error)
+    g_task_return_error (task, local_error);
+}
+
+static void
+ostree_metalink_request_unref (gpointer data)
+{
+  OstreeMetalinkRequest  *request = data;
+  g_object_unref (request->metalink);
+  g_clear_object (&request->result);
+  g_free (request->last_metalink_error);
+  g_ptr_array_unref (request->urls);
+  g_free (request);
+}
+                               
+static const GMarkupParser metalink_parser = {
+  metalink_parser_start,
+  metalink_parser_end,
+  metalink_parser_text,
+  NULL,
+  NULL
+};
+
+void
+_ostree_metalink_request_async (OstreeMetalink         *self,
+                                GCancellable           *cancellable,
+                                GAsyncReadyCallback     callback,
+                                gpointer                user_data)
+{
+  GTask *task = g_task_new (self, cancellable, callback, user_data);
+  OstreeMetalinkRequest *request = g_new0 (OstreeMetalinkRequest, 1);
+
+  request->metalink = g_object_ref (self);
+  request->urls = g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+  request->task = task; /* Unowned */
+
+  request->parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, task, NULL);
+  
+  g_task_set_task_data (task, request, ostree_metalink_request_unref);
+  _ostree_fetcher_stream_uri_async (self->fetcher, self->uri,
+                                    self->max_size, cancellable,
+                                    on_retrieved_metalink, task);
+}
+
+gboolean
+_ostree_metalink_request_finish (OstreeMetalink         *self,
+                                 GAsyncResult           *result,
+                                 SoupURI               **out_target_uri,
+                                 GFile                 **out_data,
+                                 GError                **error)
+{
+  OstreeMetalinkRequest *request;
+
+  g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
+
+  request = g_task_get_task_data ((GTask*)result);
+
+  if (g_task_propagate_boolean ((GTask*)result, error))
+    {
+      g_assert_cmpint (request->current_url_index, <, request->urls->len);
+      *out_target_uri = request->urls->pdata[request->current_url_index];
+      *out_data = g_object_ref (request->result);
+      return TRUE;
+    }
+  else
+    return FALSE;
+}
+
+SoupURI *
+_ostree_metalink_get_uri (OstreeMetalink        *self)
+{
+  return self->uri;
+}

src/libostree/ostree-metalink.h

@@ -0,0 +1,66 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2014 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#ifndef __GI_SCANNER__
+
+#include "ostree-fetcher.h"
+
+G_BEGIN_DECLS
+
+#define OSTREE_TYPE_METALINK         (_ostree_metalink_get_type ())
+#define OSTREE_METALINK(o)           (G_TYPE_CHECK_INSTANCE_CAST ((o), OSTREE_TYPE_METALINK, OstreeMetalink))
+#define OSTREE_METALINK_CLASS(k)     (G_TYPE_CHECK_CLASS_CAST((k), OSTREE_TYPE_METALINK, OstreeMetalinkClass))
+#define OSTREE_IS_METALINK(o)        (G_TYPE_CHECK_INSTANCE_TYPE ((o), OSTREE_TYPE_METALINK))
+#define OSTREE_IS_METALINK_CLASS(k)  (G_TYPE_CHECK_CLASS_TYPE ((k), OSTREE_TYPE_METALINK))
+#define OSTREE_METALINK_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), OSTREE_TYPE_METALINK, OstreeMetalinkClass))
+
+typedef struct OstreeMetalinkClass   OstreeMetalinkClass;
+typedef struct OstreeMetalink   OstreeMetalink;
+
+struct OstreeMetalinkClass
+{
+  GObjectClass parent_class;
+};
+
+GType   _ostree_metalink_get_type (void) G_GNUC_CONST;
+
+OstreeMetalink *_ostree_metalink_new (OstreeFetcher  *fetcher,
+                                      const char     *requested_file,
+                                      guint64         max_size,
+                                      SoupURI        *uri);
+
+SoupURI *_ostree_metalink_get_uri (OstreeMetalink         *self);
+
+void _ostree_metalink_request_async (OstreeMetalink         *self,
+                                     GCancellable          *cancellable,
+                                     GAsyncReadyCallback    callback,
+                                     gpointer               user_data);
+
+gboolean _ostree_metalink_request_finish (OstreeMetalink         *self,
+                                          GAsyncResult           *result,
+                                          SoupURI               **out_target_uri,
+                                          GFile                 **out_data,
+                                          GError                **error);
+
+G_END_DECLS
+
+#endif

src/libostree/ostree-repo-pull.c

@@ -26,7 +26,7 @@
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
 #include "ostree-repo-static-delta-private.h"
-#include "ostree-fetcher.h"
+#include "ostree-metalink.h"
 #include "otutil.h"
 
 typedef struct {
@@ -52,7 +52,9 @@ typedef struct {
   
   gboolean          gpg_verify;
 
+  GVariant         *summary;
   GPtrArray        *static_delta_metas;
+  GHashTable       *expected_commit_sizes; /* Maps commit checksum to known size */
   GHashTable       *scanned_metadata; /* Maps object name to itself */
   GHashTable       *requested_metadata; /* Maps object name to itself */
   GHashTable       *requested_content; /* Maps object name to itself */
@@ -368,6 +370,49 @@ fetch_uri_contents_utf8_sync (OtPullData  *pull_data,
   return ret;
 }
 
+typedef struct
+{
+  OtPullData             *pull_data;
+  SoupURI               **out_target_uri;
+  GFile                 **out_data;
+  gboolean                success;
+} FetchMetalinkSyncData;
+
+static void
+on_metalink_fetched (GObject          *src,
+                     GAsyncResult     *result,
+                     gpointer          user_data)
+{
+  FetchMetalinkSyncData *data = user_data;
+
+  data->success = _ostree_metalink_request_finish ((OstreeMetalink*)src, result,
+                                                   data->out_target_uri, data->out_data,
+                                                   data->pull_data->async_error);
+  g_main_loop_quit (data->pull_data->loop);
+}
+
+static gboolean
+request_metalink_sync (OtPullData             *pull_data,
+                       OstreeMetalink         *metalink,
+                       SoupURI               **out_target_uri,
+                       GFile                 **out_data,
+                       GCancellable           *cancellable,
+                       GError                **error)
+{
+  FetchMetalinkSyncData data = { 0, };
+
+  data.pull_data = pull_data;
+  data.out_target_uri = out_target_uri;
+  data.out_data = out_data;
+
+  pull_data->fetching_sync_uri = _ostree_metalink_get_uri (metalink);
+  _ostree_metalink_request_async (metalink, cancellable, on_metalink_fetched, &data);
+  
+  run_mainloop_monitor_fetcher (pull_data);
+
+  return data.success;
+}
+
 static void
 enqueue_one_object_request (OtPullData        *pull_data,
                             const char        *checksum,
@@ -519,6 +564,45 @@ fetch_ref_contents (OtPullData    *pull_data,
   return ret;
 }
 
+static gboolean
+lookup_commit_checksum_from_summary (OtPullData    *pull_data,
+                                     const char    *ref,
+                                     char         **out_checksum,
+                                     gsize         *out_size,
+                                     GError       **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_variant GVariant *refs = g_variant_get_child_value (pull_data->summary, 0);
+  gs_unref_variant GVariant *refdata = NULL;
+  gs_unref_variant GVariant *reftargetdata = NULL;
+  gs_unref_variant GVariant *commit_data = NULL;
+  guint64 commit_size;
+  gs_unref_variant GVariant *commit_csum_v = NULL;
+  gs_unref_bytes GBytes *commit_bytes = NULL;
+  int i;
+  
+  if (!ot_variant_bsearch_str (refs, ref, &i))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No such branch '%s' in repository summary",
+                   ref);
+      goto out;
+    }
+      
+  refdata = g_variant_get_child_value (refs, i);
+  reftargetdata = g_variant_get_child_value (refdata, 1);
+  g_variant_get (reftargetdata, "(t@ay@a{sv})", &commit_size, &commit_csum_v, NULL);
+
+  if (!ostree_validate_structureof_csum_v (commit_csum_v, error))
+    goto out;
+
+  ret = TRUE;
+  *out_checksum = ostree_checksum_from_bytes_v (commit_csum_v);
+  *out_size = commit_size;
+ out:
+  return ret;
+}
+
 static void
 content_fetch_on_write_complete (GObject        *object,
                                  GAsyncResult   *result,
@@ -690,7 +774,8 @@ meta_fetch_on_complete (GObject           *object,
   GError **error = &local_error;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
-  g_debug ("fetch of %s complete", ostree_object_to_string (checksum, objtype));
+  g_debug ("fetch of %s%s complete", ostree_object_to_string (checksum, objtype),
+           fetch_data->is_detached_meta ? " (detached)" : "");
 
   temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
   if (!temp_path)
@@ -926,9 +1011,12 @@ enqueue_one_object_request (OtPullData        *pull_data,
   gboolean is_meta;
   FetchObjectData *fetch_data;
   gs_free char *objpath = NULL;
+  guint64 *expected_max_size_p;
+  guint64 expected_max_size;
 
-  g_debug ("queuing fetch of %s.%s", checksum,
+  g_debug ("queuing fetch of %s.%s%s", checksum,
-           ostree_object_type_to_string (objtype));
+           ostree_object_type_to_string (objtype),
+           is_detached_meta ? " (detached)" : "");
 
   if (is_detached_meta)
     {
@@ -958,8 +1046,17 @@ enqueue_one_object_request (OtPullData        *pull_data,
   fetch_data->pull_data = pull_data;
   fetch_data->object = ostree_object_name_serialize (checksum, objtype);
   fetch_data->is_detached_meta = is_detached_meta;
+
+  expected_max_size_p = g_hash_table_lookup (pull_data->expected_commit_sizes, checksum);
+  if (expected_max_size_p)
+    expected_max_size = *expected_max_size_p;
+  else if (is_meta)
+    expected_max_size = OSTREE_MAX_METADATA_SIZE;
+  else
+    expected_max_size = 0;
+
   _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri,
-                                                 is_meta ? OSTREE_MAX_METADATA_SIZE : 0,
+                                                  expected_max_size,
                                                   pull_data->cancellable,
                                                   is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
   soup_uri_free (obj_uri);
@@ -1124,9 +1221,11 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
   gs_free char *remote_key = NULL;
   gs_free char *path = NULL;
   gs_free char *baseurl = NULL;
+  gs_free char *metalink_url_str = NULL;
   gs_unref_hashtable GHashTable *requested_refs_to_fetch = NULL;
   gs_unref_hashtable GHashTable *commits_to_fetch = NULL;
   gs_free char *remote_mode_str = NULL;
+  gs_unref_object OstreeMetalink *metalink = NULL;
   OtPullData pull_data_real = { 0, };
   OtPullData *pull_data = &pull_data_real;
   GKeyFile *config = NULL;
@@ -1146,6 +1245,9 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
   pull_data->repo = self;
   pull_data->progress = progress;
 
+  pull_data->expected_commit_sizes = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                                            (GDestroyNotify)g_free,
+                                                            (GDestroyNotify)g_free);
   pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
                                                        (GDestroyNotify)g_variant_unref, NULL);
   pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
@@ -1167,10 +1269,6 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
                    remote_key);
       goto out;
     }
-  if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error))
-    goto out;
-
-  pull_data->base_uri = soup_uri_new (baseurl);
 
 #ifdef HAVE_GPGME
   if (!ot_keyfile_get_boolean_with_default (config, remote_key, "gpg-verify",
@@ -1256,12 +1354,55 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
       _ostree_fetcher_set_proxy (pull_data->fetcher, http_proxy);
   }
 
+  if (!ot_keyfile_get_value_with_default (config, remote_key, "metalink",
+                                          NULL, &metalink_url_str, error))
+    goto out;
+
+  if (!metalink_url_str)
+    {
+      if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error))
+        goto out;
+
+      pull_data->base_uri = soup_uri_new (baseurl);
+
       if (!pull_data->base_uri)
         {
           g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                        "Failed to parse url '%s'", baseurl);
           goto out;
         }
+    }
+  else
+    {
+      gs_unref_object GFile *metalink_data = NULL;
+      SoupURI *metalink_uri = soup_uri_new (metalink_url_str);
+      SoupURI *target_uri = NULL;
+      
+      if (!metalink_uri)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Invalid metalink URL: %s", metalink_url_str);
+          goto out;
+        }
+      
+      metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
+                                       OSTREE_MAX_METADATA_SIZE, metalink_uri);
+      soup_uri_free (metalink_uri);
+
+      if (!request_metalink_sync (pull_data, metalink, &target_uri, &metalink_data,
+                                  cancellable, error))
+        goto out;
+
+      {
+        gs_free char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
+        pull_data->base_uri = soup_uri_copy (target_uri);
+        soup_uri_set_path (pull_data->base_uri, repo_base);
+      }
+
+      if (!ot_util_variant_map (metalink_data, OSTREE_SUMMARY_GVARIANT_FORMAT, FALSE,
+                                &pull_data->summary, error))
+        goto out;
+    }
 
   if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
     goto out;
@@ -1292,7 +1433,6 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
       for (strviter = refs_to_fetch; *strviter; strviter++)
         {
           const char *branch = *strviter;
-          char *contents;
 
           if (ostree_validate_checksum_string (branch, NULL))
             {
@@ -1301,11 +1441,7 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
             }
           else
             {
-              if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
+              g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
-                goto out;
-
-              /* Transfer ownership of contents */
-              g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
             }
         }
     }
@@ -1325,14 +1461,37 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
       for (;branches_iter && *branches_iter; branches_iter++)
         {
           const char *branch = *branches_iter;
+              
+          g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
+        }
+    }
+
+  g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
+  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+    {
+      const char *branch = key;
       char *contents;
 
-          if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
+      if (pull_data->summary)
+        {
+          guint64 commit_size;
+          guint64 *malloced_size;
+
+          if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error))
             goto out;
 
-          /* Transfer ownership of contents */
+          malloced_size = g_new0 (guint64, 1);
-          g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
+          *malloced_size = commit_size;
+          g_hash_table_insert (pull_data->expected_commit_sizes, contents, malloced_size);
         }
+      else
+        {
+          if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
+            goto out;
+        }
+      
+      /* Transfer ownership of contents */
+      g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents);
     }
 
   /* Create the state directory here - it's new with the commitpartial code,
@@ -1460,7 +1619,9 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
   g_free (pull_data->remote_name);
   if (pull_data->base_uri)
     soup_uri_free (pull_data->base_uri);
+  g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
   g_clear_pointer (&pull_data->static_delta_metas, (GDestroyNotify) g_ptr_array_unref);
+  g_clear_pointer (&pull_data->expected_commit_sizes, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);

src/libostree/ostree-repo.c

@@ -381,7 +381,7 @@ GS_DEFINE_CLEANUP_FUNCTION0(GKeyFile*, local_keyfile_unref, g_key_file_unref)
  * ostree_repo_remote_add:
  * @self: Repo
  * @name: Name of remote
- * @url: URL for remote
+ * @url: URL for remote (if URL begins with metalink=, it will be used as such)
  * @options: (allow-none): GVariant of type a{sv}
  * @cancellable: Cancellable
  * @error: Error
@@ -446,7 +446,11 @@ ostree_repo_remote_add (OstreeRepo     *self,
       target_keyfile = ostree_repo_copy_config (self);
     }
 
+  if (g_str_has_prefix (url, "metalink="))
+    g_key_file_set_string (target_keyfile, section, "metalink", url + strlen ("metalink="));
+  else
     g_key_file_set_string (target_keyfile, section, "url", url);
+
   if (options)
     keyfile_set_from_vardict (target_keyfile, section, options);
 
@@ -2264,3 +2268,81 @@ out:
     (void) gs_file_unlink (commit_tmp_path, NULL, NULL);
   return ret;
 }
+
+/**
+ * ostree_repo_regenerate_summary:
+ * @self: Repo
+ * @additional_metadata: (allow-none): A GVariant of type a{sv}, or %NULL
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * An OSTree repository can contain a high level "summary" file that
+ * describes the available branches and other metadata.
+ *
+ * It is not regenerated automatically when commits are created; this
+ * API is available to atomically regenerate the summary after
+ * multiple commits.  It should only be invoked by one process at a
+ * time.
+ */
+gboolean
+ostree_repo_regenerate_summary (OstreeRepo     *self,
+                                GVariant       *additional_metadata,
+                                GCancellable   *cancellable,
+                                GError        **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_object GFile *summary_path = NULL;
+  gs_unref_hashtable GHashTable *refs = NULL;
+  gs_unref_variant_builder GVariantBuilder *refs_builder = NULL;
+  gs_unref_variant GVariant *summary = NULL;
+  GList *ordered_keys = NULL;
+  GList *iter = NULL;
+
+  if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error))
+    goto out;
+
+  refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))"));
+
+  ordered_keys = g_hash_table_get_keys (refs);
+  ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp);
+  
+  for (iter = ordered_keys; iter; iter = iter->next)
+    {
+      const char *ref = iter->data;
+      const char *commit = g_hash_table_lookup (refs, ref);
+      gs_unref_variant GVariant *commit_obj = NULL;
+
+      g_assert (commit);
+
+      if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error))
+        goto out;
+
+      g_variant_builder_add_value (refs_builder, 
+                                   g_variant_new ("(s(t@ay@a{sv}))", ref,
+                                                  g_variant_get_size (commit_obj),
+                                                  ostree_checksum_to_bytes_v (commit),
+                                                  ot_gvariant_new_empty_string_dict ()));
+    }
+
+  {
+    gs_unref_variant_builder GVariantBuilder *summary_builder =
+      g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT);
+
+    g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder));
+    g_variant_builder_add_value (summary_builder, additional_metadata ? additional_metadata : ot_gvariant_new_empty_string_dict ());
+    summary = g_variant_builder_end (summary_builder);
+    g_variant_ref_sink (summary);
+  }
+
+  summary_path = g_file_get_child (self->repodir, "summary");
+
+  if (!ot_util_variant_save (summary_path, summary, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  if (ordered_keys)
+    g_list_free (ordered_keys);
+  return ret;
+}
+

src/libostree/ostree-repo.h

@@ -568,5 +568,11 @@ gboolean ostree_repo_verify_commit (OstreeRepo   *self,
                                     GCancellable *cancellable,
                                     GError      **error);
 
+gboolean ostree_repo_regenerate_summary (OstreeRepo     *self,
+                                         GVariant       *additional_metadata,
+                                         GCancellable   *cancellable,
+                                         GError        **error);
+
+
 G_END_DECLS
 

src/libotutil/ot-checksum-utils.c

@@ -139,6 +139,32 @@ ot_gio_checksum_stream (GInputStream   *in,
   return ot_gio_splice_get_checksum (NULL, in, out_csum, cancellable, error);
 }
 
+char *
+ot_checksum_file (GFile          *file,
+                  GChecksumType   checksum_type,
+                  GCancellable   *cancellable,
+                  GError        **error)
+{
+  GChecksum *checksum = NULL;
+  char *ret = NULL;
+  gs_unref_object GInputStream *in = NULL;
+
+  in = (GInputStream*)g_file_read (file, cancellable, error);
+  if (!in)
+    goto out;
+
+  checksum = g_checksum_new (checksum_type);
+
+  if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+    goto out;
+
+  ret = g_strdup (g_checksum_get_string (checksum));
+ out:
+  g_clear_pointer (&checksum, (GDestroyNotify) g_checksum_free);
+  return ret;
+
+}
+
 static void
 checksum_stream_thread (GSimpleAsyncResult   *result,
                         GObject              *object,

src/libotutil/ot-checksum-utils.h

@@ -53,6 +53,11 @@ gboolean ot_gio_checksum_stream (GInputStream   *in,
                                  GCancellable   *cancellable,
                                  GError        **error);
 
+char * ot_checksum_file (GFile          *file,
+                         GChecksumType   checksum_type,
+                         GCancellable   *cancellable,
+                         GError        **error);
+
 void ot_gio_checksum_stream_async (GInputStream         *in,
                                    int                   io_priority,
                                    GCancellable         *cancellable,

src/libotutil/ot-variant-utils.c

@@ -30,6 +30,12 @@
 
 #include "otutil.h"
 
+GVariant *
+ot_gvariant_new_empty_string_dict (void)
+{
+  return g_variant_builder_end (g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")));
+}
+
 GVariant *
 ot_gvariant_new_bytearray (const guchar   *data,
                            gsize           len)
@@ -282,3 +288,61 @@ ot_variant_new_from_bytes (const GVariantType  *type,
                                   (GDestroyNotify)g_bytes_unref, bytes);
 #endif
 }
+
+/**
+ * ot_variant_bsearch_str:
+ * @array: A GVariant array whose first element must be a string
+ * @str: Search for this string
+ * @out_pos: Output position
+ *
+ *
+ * Binary search in a GVariant array, which must be of the form 'a(s...)',
+ * where '...' may be anything.  The array elements must be sorted.
+ *
+ * Returns: %TRUE if found, %FALSE otherwise
+ */
+gboolean
+ot_variant_bsearch_str (GVariant   *array,
+                        const char *str,
+                        int        *out_pos)
+{
+  gsize imax, imin;
+  gsize imid;
+  gsize n;
+
+  n = g_variant_n_children (array);
+  if (n == 0)
+    return FALSE;
+
+  imax = n - 1;
+  imin = 0;
+  while (imax >= imin)
+    {
+      gs_unref_variant GVariant *child = NULL;
+      const char *cur;
+      int cmp;
+
+      imid = (imin + imax) / 2;
+
+      child = g_variant_get_child_value (array, imid);
+      g_variant_get_child (child, 0, "&s", &cur, NULL);      
+
+      cmp = strcmp (cur, str);
+      if (cmp < 0)
+        imin = imid + 1;
+      else if (cmp > 0)
+        {
+          if (imid == 0)
+            break;
+          imax = imid - 1;
+        }
+      else
+        {
+          *out_pos = imid;
+          return TRUE;
+        }
+    }
+
+  *out_pos = imid;
+  return FALSE;
+}

src/libotutil/ot-variant-utils.h

@@ -31,6 +31,8 @@ GVariant *ot_gvariant_new_bytearray (const guchar   *data,
 
 GVariant *ot_gvariant_new_ay_bytes (GBytes *bytes);
 
+GVariant *ot_gvariant_new_empty_string_dict (void);
+
 GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant);
 
 GVariant * ot_util_variant_take_ref (GVariant *variant);
@@ -70,5 +72,10 @@ ot_variant_new_from_bytes (const GVariantType  *type,
                            GBytes        *bytes,
                            gboolean       trusted);
 
+gboolean
+ot_variant_bsearch_str (GVariant   *array,
+                        const char *str,
+                        int        *out_pos);
+
 G_END_DECLS
 

src/ostree/main.c

@@ -55,6 +55,7 @@ static OstreeCommand commands[] = {
   { "rev-parse", ostree_builtin_rev_parse, 0 },
   { "show", ostree_builtin_show, 0 },
   { "static-delta", ostree_builtin_static_delta, 0 },
+  { "summary", ostree_builtin_summary, 0 },
 #ifdef HAVE_LIBSOUP 
   { "trivial-httpd", ostree_builtin_trivial_httpd, OSTREE_BUILTIN_FLAG_NO_REPO },
 #endif

src/ostree/ot-builtin-summary.c

@@ -0,0 +1,63 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2014 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ot-builtins.h"
+#include "ostree.h"
+#include "otutil.h"
+
+static gboolean opt_update;
+
+static GOptionEntry options[] = {
+  { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL },
+  { NULL }
+};
+
+gboolean
+ostree_builtin_summary (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError **error)
+{
+  gboolean ret = FALSE;
+  GOptionContext *context;
+
+  context = g_option_context_new ("Manage summary metadata");
+  g_option_context_add_main_entries (context, options, NULL);
+
+  if (!g_option_context_parse (context, &argc, &argv, error))
+    goto out;
+
+  if (opt_update)
+    {
+      if (!ostree_repo_regenerate_summary (repo, NULL, cancellable, error))
+        goto out;
+    }
+  else
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No option specified; use -u to update summary");
+      goto out;
+    }
+
+  ret = TRUE;
+ out:
+  if (context)
+    g_option_context_free (context);
+  return ret;
+}

src/ostree/ot-builtins.h

@@ -46,6 +46,7 @@ BUILTINPROTO(reset);
 BUILTINPROTO(fsck);
 BUILTINPROTO(show);
 BUILTINPROTO(static_delta);
+BUILTINPROTO(summary);
 BUILTINPROTO(rev_parse);
 BUILTINPROTO(remote);
 BUILTINPROTO(write_refs);

tests/test-pull-metalink.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# Copyright (C) 2014 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+
+# And another web server acting as the metalink server
+cd ${test_tmpdir}
+mkdir metalink-data
+cd metalink-data
+ostree trivial-httpd --daemonize -p ${test_tmpdir}/metalink-httpd-port
+metalink_port=$(cat ${test_tmpdir}/metalink-httpd-port)
+echo "http://127.0.0.1:${metalink_port}" > ${test_tmpdir}/metalink-httpd-address
+
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u
+
+summary_path=${test_tmpdir}/ostree-srv/gnomerepo/summary
+
+echo -n broken > ${summary_path}.bad
+
+echo '1..1'
+cd ${test_tmpdir}
+
+cat > ${test_tmpdir}/metalink-data/metalink.xml <<EOF
+<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <files>
+    <file name="summary">
+      <size>$(stat -c '%s' ${summary_path})</size>
+      <verification>
+        <hash type="md5">$(md5sum ${summary_path} | cut -f 1 -d ' ')</hash>
+        <hash type="sha256">$(sha256sum ${summary_path} | cut -f 1 -d ' ')</hash>
+        <hash type="sha512">$(sha512sum ${summary_path} | cut -f 1 -d ' ')</hash>
+      </verification>
+      <resources maxconnections="1">
+        <url protocol="http" type="http" location="US" preference="100" >$(cat httpd-address)/ostree/gnomerepo/summary.bad</url>
+        <url protocol="http" type="http" location="US" preference="99" >$(cat httpd-address)/ostree/gnomerepo/nosuchfile</url>
+        <url protocol="http" type="http" location="US" preference="98" >$(cat httpd-address)/ostree/gnomerepo/summary</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+EOF
+
+cd ${test_tmpdir}
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat metalink-httpd-address)/metalink.xml
+${CMD_PREFIX} ostree --repo=repo pull origin:main
+${CMD_PREFIX} ostree --repo=repo rev-parse origin:main
+${CMD_PREFIX} ostree --repo=repo fsck
+echo "ok pull via metalink"
+
+cp metalink-data/metalink.xml{,.orig}
+cp ostree-srv/gnomerepo/summary{,.orig}
+
+test_metalink_pull_error() {
+    msg=$1
+    rm repo -rf
+    mkdir repo
+    ${CMD_PREFIX} ostree --repo=repo init
+    ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat metalink-httpd-address)/metalink.xml
+    if ${CMD_PREFIX} ostree --repo=repo pull origin:main 2>err.txt; then
+	assert_not_reached "pull unexpectedly succeeded"
+    fi
+    cat err.txt
+    assert_file_has_content err.txt "${msg}"
+}
+
+cd ${test_tmpdir}
+sed -e 's,<hash type="sha512">.*</hash>,<hash type="sha512">bacon</hash>,' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "Invalid hash digest for sha512"
+echo "ok metalink err hash format"
+
+cd ${test_tmpdir}
+sed -e 's,<hash type="sha512">.*</hash>,<hash type="sha512">'$( (echo -n dummy; cat ${summary_path}) | sha512sum | cut -f 1 -d ' ')'</hash>,' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "Expected checksum is .* but actual is"
+echo "ok metalink err hash sha512"
+
+cd ${test_tmpdir}
+cp metalink-data/metalink.xml.orig metalink-data/metalink.xml
+echo -n moo > ostree-srv/gnomerepo/summary
+test_metalink_pull_error "Expected size is .* bytes but content is 3 bytes"
+echo "ok metalink err size"
+cp ostree-srv/gnomerepo/summary{.orig,}
+
+cd ${test_tmpdir}
+grep -v sha256 < metalink-data/metalink.xml.orig |grep -v sha512 > metalink-data/metalink.xml
+test_metalink_pull_error "No.*verification.*with known.*hash"
+echo "ok metalink err no verify"
+
+cd ${test_tmpdir}
+grep -v '<url protocol' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "No.*url.*method.*elements"
+echo "ok metalink err no url"
+
+cd ${test_tmpdir}
+echo bacon > metalink-data/metalink.xml
+test_metalink_pull_error "Document must begin with an element"
+echo "ok metalink err malformed"
+