pull: Add support for resuming downloads via range requests

Use a consistent temporary filename to download uri's.
Check for downloaded files before fetching from uri.
Download to hash.part file, then copy/move to hash.done when complete.
Add argument support to setup_fake_remote_repo1 function.
Add test for pull resume.
To implement this, pass --force-range-requests into the trivial-httpd,
which will only serve half of the objects to clients at a time.

https://bugzilla.gnome.org/show_bug.cgi?id=706344

Makefile-tests.am

@@ -27,6 +27,7 @@ testfiles = test-basic \
 	test-libarchive \
 	test-pull-archive-z \
 	test-pull-corruption \
+	test-pull-resume \
 	test-admin-deploy-1 \
 	test-admin-deploy-2 \
 	test-setuid \

src/libostree/ostree-fetcher.c

@@ -43,6 +43,7 @@ typedef struct {
   SoupRequest *request;
 
   GFile *tmpfile;
+  gchar *filename; /* Hash of the SoupURI used to request the file */
   GInputStream *request_body;
   GOutputStream *out_stream;
 
@@ -63,6 +64,7 @@ pending_uri_free (OstreeFetcherPendingURI *pending)
   soup_uri_free (pending->uri);
   g_clear_object (&pending->self);
   g_clear_object (&pending->tmpfile);
+  g_free (pending->filename);
   g_clear_object (&pending->request);
   g_clear_object (&pending->request_body);
   g_clear_object (&pending->out_stream);
@@ -87,6 +89,9 @@ struct OstreeFetcher
   guint total_requests;
 };
 
+static const gchar *partsuffix = ".part";
+static const gchar *donesuffix = ".done";
+
 G_DEFINE_TYPE (OstreeFetcher, ostree_fetcher, G_TYPE_OBJECT)
 
 static void
@@ -167,6 +172,37 @@ ostree_fetcher_new (GFile                    *tmpdir,
   return self;
 }
 
+static gboolean
+rename_part_file (OstreeFetcherPendingURI *pending,
+                  GCancellable  *cancellable,
+                  GError       **error)
+{
+  gboolean ret = FALSE;
+  GError *local_error = NULL;
+  const gchar *tempfilename = gs_file_get_path_cached (pending->tmpfile);
+
+  // Only rename files that end in .part
+  if (g_str_has_suffix (tempfilename, partsuffix))
+    {
+      gs_free gchar *finalname = g_strconcat (pending->filename, donesuffix, NULL);
+      gs_unref_object GFile *donefile = g_file_get_child (pending->self->tmpdir, finalname);
+      // Copy the .part file to .done file
+      if (!g_file_move (pending->tmpfile, 
+                        donefile, 
+                        G_FILE_COPY_OVERWRITE, 
+                        NULL, 
+                        NULL, 
+                        NULL, 
+                        &local_error))
+          goto out;
+      g_object_unref (pending->tmpfile);
+      pending->tmpfile = g_object_ref (donefile);
+    }
+  ret = TRUE;
+out:
+  return ret;
+}
+
 static void
 on_splice_complete (GObject        *object,
                     GAsyncResult   *result,
@@ -174,13 +210,27 @@ on_splice_complete (GObject        *object,
 {
   OstreeFetcherPendingURI *pending = user_data;
   gs_unref_object GFileInfo *file_info = NULL;
+  GError *local_error = NULL;
 
   pending->state = OSTREE_FETCHER_STATE_COMPLETE;
   file_info = g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
                                  G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
                                  NULL, NULL);
   if (file_info)
-    pending->self->total_downloaded += g_file_info_get_size (file_info);
+    {
+      goffset filesize = g_file_info_get_size (file_info);
+      if (filesize < pending->content_length)
+        {
+          g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete");
+          g_simple_async_result_take_error (pending->result, local_error);
+        }
+      else
+        {
+          pending->self->total_downloaded += g_file_info_get_size (file_info);
+          if (!rename_part_file (pending, NULL, &local_error))
+            g_simple_async_result_take_error (pending->result, local_error);
+        }
+    }
 
   (void) g_input_stream_close (pending->request_body, NULL, NULL);
 
@@ -212,7 +262,20 @@ on_request_sent (GObject        *object,
   if (SOUP_IS_REQUEST_HTTP (object))
     {
       msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
-      if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
+      if (msg->status_code == SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE)
+        {
+          // We already have the whole file, so just use it.
+          pending->state = OSTREE_FETCHER_STATE_COMPLETE;
+          if (!rename_part_file (pending, NULL, &local_error))
+            {
+              g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, "Rename failed");
+              g_simple_async_result_take_error (pending->result, local_error);
+            }
+          g_simple_async_result_complete (pending->result);
+          g_object_unref (pending->result);
+          return;
+        }
+      else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
         {
           GIOErrorEnum code;
           switch (msg->status_code)
@@ -237,18 +300,143 @@ on_request_sent (GObject        *object,
   
   pending->content_length = soup_request_get_content_length (pending->request);
   
-  /* TODO - make this async */
-  if (!gs_file_open_in_tmpdir (pending->self->tmpdir, 0644,
-                               &pending->tmpfile, &pending->out_stream,
-                               NULL, &local_error))
+  g_output_stream_splice_async (pending->out_stream, pending->request_body, flags, G_PRIORITY_DEFAULT,
+                                pending->cancellable, on_splice_complete, pending);
+}
+
+static OstreeFetcherPendingURI *
+ostree_fetcher_request_uri_internal (OstreeFetcher *self,
+                                     SoupURI       *uri,
+                                     GCancellable  *cancellable)
+{
+  OstreeFetcherPendingURI *pending;
+  GError *local_error = NULL;
+  gs_free char *uristring = soup_uri_to_string (uri, FALSE);
+
+  pending = g_new0 (OstreeFetcherPendingURI, 1);
+  pending->refcount = 1;
+  pending->self = g_object_ref (self);
+  pending->uri = soup_uri_copy (uri);
+  pending->filename = g_compute_checksum_for_string (G_CHECKSUM_SHA256, uristring, strlen (uristring));
+  pending->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
+  pending->request = soup_requester_request_uri (self->requester, uri, &local_error);
+  
+  g_assert_no_error (local_error);
+  
+  pending->refcount++;
+
+  return pending;
+}
+
+static void
+ostree_fetcher_request_uri_use_existing_file (OstreeFetcher           *self,
+                                              OstreeFetcherPendingURI *pending,
+                                              GAsyncReadyCallback      callback,
+                                              gpointer                 user_data,
+                                              gpointer                 source_tag)
+{
+  pending->state = OSTREE_FETCHER_STATE_COMPLETE;
+  pending->result = g_simple_async_result_new ((GObject*) self,
+                                               callback,
+                                               user_data,
+                                               source_tag);
+  g_simple_async_result_set_op_res_gpointer (pending->result,
+                                             pending,
+                                             (GDestroyNotify) pending_uri_free);
+  g_simple_async_result_complete (pending->result);
+  g_object_unref (pending->result);
+}
+
+void
+ostree_fetcher_request_uri_with_partial_async (OstreeFetcher         *self,
+                                               SoupURI               *uri,
+                                               GCancellable          *cancellable,
+                                               GAsyncReadyCallback    callback,
+                                               gpointer               user_data)
+{
+  OstreeFetcherPendingURI *pending;
+  GError *local_error = NULL;
+  gs_free char *finalname = NULL;
+  gs_free char *downloadname = NULL;
+
+  self->total_requests++;
+
+  pending = ostree_fetcher_request_uri_internal (self, uri, cancellable);
+
+  finalname = g_strconcat (pending->filename, donesuffix, NULL);
+  downloadname = g_strconcat (pending->filename, partsuffix, NULL);
+  
+  /* First check if the finalname file exists */
+  pending->tmpfile = g_file_get_child (pending->self->tmpdir, finalname);
+  if (g_file_query_exists (pending->tmpfile, NULL) )
+    {
+      // We already have the complete file, so just use it.
+      ostree_fetcher_request_uri_use_existing_file (self,
+                                                    pending,
+                                                    callback,
+                                                    user_data,
+                                                    ostree_fetcher_request_uri_with_partial_async);
+    }
+  else
+    {
+      g_object_unref (pending->tmpfile);
+      pending->tmpfile = g_file_get_child (pending->self->tmpdir, downloadname);
+      pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->tmpfile, G_FILE_CREATE_NONE, NULL, &local_error));
+      if (!pending->out_stream)
+        goto out;
+
+      if (SOUP_IS_REQUEST_HTTP (pending->request))
+        {
+          SoupMessage *msg;
+          gs_unref_object GFileInfo *file_info = 
+            g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
+                               G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                               NULL, &local_error);
+          if (!file_info)
+            goto out;
+
+          msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
+          if (g_file_info_get_size (file_info) > 0)
+            soup_message_headers_set_range (msg->request_headers, g_file_info_get_size (file_info), -1);
+          g_hash_table_insert (self->message_to_request,
+                               soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
+                               pending);
+        }
+
+      pending->result = g_simple_async_result_new ((GObject*) self,
+                        callback, user_data,
+                        ostree_fetcher_request_uri_with_partial_async);
+      g_simple_async_result_set_op_res_gpointer (pending->result, pending,
+          (GDestroyNotify) pending_uri_free);
+
+      soup_request_send_async (pending->request, cancellable,
+                               on_request_sent, pending);
+    }
+
+ out:
+  if (local_error != NULL)
     {
       g_simple_async_result_take_error (pending->result, local_error);
       g_simple_async_result_complete (pending->result);
-      return;
     }
-  
-  g_output_stream_splice_async (pending->out_stream, pending->request_body, flags, G_PRIORITY_DEFAULT,
-                                pending->cancellable, on_splice_complete, pending);
+}
+
+GFile *
+ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher         *self,
+                                   GAsyncResult          *result,
+                                   GError               **error)
+{
+  GSimpleAsyncResult *simple;
+  OstreeFetcherPendingURI *pending;
+
+  g_return_val_if_fail (g_simple_async_result_is_valid (result, (GObject*)self, ostree_fetcher_request_uri_with_partial_async), FALSE);
+
+  simple = G_SIMPLE_ASYNC_RESULT (result);
+  if (g_simple_async_result_propagate_error (simple, error))
+    return NULL;
+  pending = g_simple_async_result_get_op_res_gpointer (simple);
+
+  return g_object_ref (pending->tmpfile);
 }
 
 void
@@ -260,33 +448,51 @@ ostree_fetcher_request_uri_async (OstreeFetcher         *self,
 {
   OstreeFetcherPendingURI *pending;
   GError *local_error = NULL;
+  gs_free char *finalname = NULL;
 
   self->total_requests++;
 
-  pending = g_new0 (OstreeFetcherPendingURI, 1);
-  pending->refcount = 1;
-  pending->self = g_object_ref (self);
-  pending->uri = soup_uri_copy (uri);
-  pending->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
-  pending->request = soup_requester_request_uri (self->requester, uri, &local_error);
-  g_assert_no_error (local_error);
+  pending = ostree_fetcher_request_uri_internal (self, uri, cancellable);
 
-  pending->refcount++;
-  if (SOUP_IS_REQUEST_HTTP (pending->request))
+  finalname = g_strconcat (pending->filename, donesuffix, NULL);
+
+  /* TODO - make this async */
+  pending->tmpfile = g_file_get_child (pending->self->tmpdir, finalname);
+  if (g_file_query_exists (pending->tmpfile, NULL) )
     {
-      g_hash_table_insert (self->message_to_request,
-                           soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
-                           pending);
+      // We already have the complete file, so just use it.
+      ostree_fetcher_request_uri_use_existing_file (self,
+                                                    pending,
+                                                    callback,
+                                                    user_data,
+                                                    ostree_fetcher_request_uri_async);
     }
+  else
+    {
+      pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->tmpfile, G_FILE_CREATE_NONE, NULL, &local_error));
+      if (local_error)
+        {
+          g_simple_async_result_take_error (pending->result, local_error);
+          g_simple_async_result_complete (pending->result);
+          return;
+        }
 
-  pending->result = g_simple_async_result_new ((GObject*) self,
-                                               callback, user_data,
-                                               ostree_fetcher_request_uri_async);
-  g_simple_async_result_set_op_res_gpointer (pending->result, pending,
-                                             (GDestroyNotify) pending_uri_free);
+      if (SOUP_IS_REQUEST_HTTP (pending->request))
+        {
+          g_hash_table_insert (self->message_to_request,
+                               soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
+                               pending);
+        }
 
-  soup_request_send_async (pending->request, cancellable,
-                           on_request_sent, pending);
+      pending->result = g_simple_async_result_new ((GObject*) self,
+                                                   callback, user_data,
+                                                   ostree_fetcher_request_uri_async);
+      g_simple_async_result_set_op_res_gpointer (pending->result, pending,
+                                                 (GDestroyNotify) pending_uri_free);
+
+      soup_request_send_async (pending->request, cancellable,
+                               on_request_sent, pending);
+    }
 }
 
 GFile *

src/libostree/ostree-fetcher.h

@@ -60,6 +60,16 @@ guint64 ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
 guint ostree_fetcher_get_n_requests (OstreeFetcher       *self);
 
+void ostree_fetcher_request_uri_with_partial_async (OstreeFetcher         *self,
+                                                    SoupURI               *uri,
+                                                    GCancellable          *cancellable,
+                                                    GAsyncReadyCallback    callback,
+                                                    gpointer               user_data);
+
+GFile *ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
+                                                       GAsyncResult  *result,
+                                                       GError       **error);
+
 void ostree_fetcher_request_uri_async (OstreeFetcher         *self,
                                        SoupURI               *uri,
                                        GCancellable          *cancellable,

src/libostree/ostree-repo-pull.c

@@ -402,8 +402,6 @@ fetch_uri_contents_utf8 (OtPullData  *pull_data,
   ret = TRUE;
   ot_transfer_out_value (out_contents, &ret_contents);
  out:
-  if (tmpf)
-    (void) unlink (gs_file_get_path_cached (tmpf));
   return ret;
 }
 
@@ -589,7 +587,7 @@ content_fetch_on_complete (GObject        *object,
   const char *checksum;
   OstreeObjectType objtype;
 
-  fetch_data->temp_path = ostree_fetcher_request_uri_finish ((OstreeFetcher*)object, result, error);
+  fetch_data->temp_path = ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
   if (!fetch_data->temp_path)
     goto out;
 
@@ -679,7 +677,7 @@ meta_fetch_on_complete (GObject           *object,
   GError *local_error = NULL;
   GError **error = &local_error;
 
-  fetch_data->temp_path = ostree_fetcher_request_uri_finish ((OstreeFetcher*)object, result, error);
+  fetch_data->temp_path = ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
   if (!fetch_data->temp_path)
     goto out;
 
@@ -990,7 +988,7 @@ on_metadata_objects_to_fetch_ready (gint         fd,
       fetch_data = g_new (FetchObjectData, 1);
       fetch_data->pull_data = pull_data;
       fetch_data->object = g_variant_ref (msg->d.item);
-      ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
+      ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
                                         is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
       soup_uri_free (obj_uri);
       g_variant_unref (msg->d.item);

tests/libtest.sh

@@ -120,6 +120,7 @@ setup_test_repository () {
 
 setup_fake_remote_repo1() {
     mode=$1
+    args=$2
     shift
     oldpwd=`pwd`
     mkdir ostree-srv
@@ -146,7 +147,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ostree trivial-httpd --daemonize -p ${test_tmpdir}/httpd-port
+    ostree trivial-httpd --daemonize -p ${test_tmpdir}/httpd-port $args
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 

tests/test-pull-resume.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Jeremy Whiting <jeremy.whiting@collabora.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2" "--force-range-requests"
+
+echo '1..1'
+
+repopath=${test_tmpdir}/ostree-srv/gnomerepo
+cp -a ${repopath} ${repopath}.orig
+
+cd ${test_tmpdir}
+rm repo -rf
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
+
+maxtries=`find ${repopath}/objects | wc -l`
+maxtries=`expr $maxtries \* 2`
+
+for ((i = 0; i < $maxtries; i=i+1))
+do
+if ${CMD_PREFIX} ostree --repo=repo pull origin main; then
+    break;
+fi
+done
+if ${CMD_PREFIX} ostree --repo=repo fsck; then
+    echo "ok, pull succeeded!"
+else
+    assert_not_reached "pull failed!"
+fi
+rm -rf ${repopath}
+cp -a ${repopath}.orig ${repopath}