shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-01-11 09:18:20 +03:00
Code Issues Projects Releases Wiki Activity

static deltas: Add support for inline-parts

Browse Source 
In this mode the parts are stored in the metadata of the main delta
superblock file.  This can be useful if you want a single-file delta
for easy transport, or for http in the case the delta is very small.
This commit is contained in:
Alexander Larsson 2015-09-10 20:52:54 +02:00
parent 30768a979f
commit 11a79220e2
3 changed files with 119 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/libostree/ostree-repo-pull.c
View File

@ -1489,11 +1489,13 @@ process_one_static_delta (OtPullData *pull_data,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) fallback_objects = NULL;
guint i, n;
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
metadata = g_variant_get_child_value (delta_superblock, 0);
headers = g_variant_get_child_value (delta_superblock, 6);
fallback_objects = g_variant_get_child_value (delta_superblock, 7);
@ -1557,6 +1559,8 @@ process_one_static_delta (OtPullData *pull_data,
FetchStaticDeltaData *fetch_data;
g_autoptr(GVariant) csum_v = NULL;
g_autoptr(GVariant) objects = NULL;
g_autoptr(GVariant) part_data = NULL;
g_autoptr(GBytes) delta_data = NULL;
guint64 size, usize;
guint32 version;
@ -1574,6 +1578,29 @@ process_one_static_delta (OtPullData *pull_data,
if (!csum)
goto out;
deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)"));
if (part_data)
{
g_autofree char *actual_checksum = NULL;
g_autofree char *expected_checksum = ostree_checksum_from_bytes_v (csum_v);
delta_data = g_variant_get_data_as_bytes (part_data);
/* For inline parts we are relying on per-commit GPG, so this isn't strictly necessary for security.
* See https://github.com/GNOME/ostree/pull/139
*/
actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, delta_data);
if (strcmp (actual_checksum, expected_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted static delta part; checksum expected='%s' actual='%s'",
expected_checksum, actual_checksum);
goto out;
}
}
pull_data->total_deltapart_size += size;
if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo,
@ -1596,16 +1623,27 @@ process_one_static_delta (OtPullData *pull_data,
fetch_data->objects = g_variant_ref (objects);
fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch_data);
pull_data->n_outstanding_deltapart_fetches++;
soup_uri_free (target_uri);
if (delta_data != NULL)
{
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
delta_data,
pull_data->cancellable,
on_static_delta_written,
fetch_data);
pull_data->n_outstanding_deltapart_write_requests++;
}
else
{
target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch_data);
pull_data->n_outstanding_deltapart_fetches++;
soup_uri_free (target_uri);
}
}
ret = TRUE;

48
src/libostree/ostree-repo-static-delta-compilation.c
View File

@ -1226,6 +1226,7 @@ get_fallback_headers (OstreeRepo *self,
* for input files
* - compression: y: Compression type: 0=none, x=lzma, g=gzip
* - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE.
* - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE.
* - verbose: b: Print diagnostic messages. Default FALSE.
*/
gboolean
@ -1244,7 +1245,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
guint min_fallback_size;
guint max_bsdiff_size;
guint max_chunk_size;
GVariant *metadata_source;
GVariantBuilder metadata_builder;
DeltaOpts delta_opts = DELTAOPT_FLAG_NONE;
guint64 total_compressed_size = 0;
guint64 total_uncompressed_size = 0;
@ -1257,6 +1258,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
g_autoptr(GFile) descriptor_dir = NULL;
g_autoptr(GVariant) tmp_metadata = NULL;
g_autoptr(GVariant) fallback_headers = NULL;
gboolean inline_parts;
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@ -1286,6 +1288,9 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
delta_opts |= DELTAOPT_FLAG_VERBOSE;
}
if (!g_variant_lookup (params, "inline-parts", "b", &inline_parts))
inline_parts = FALSE;
if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, to,
&to_commit, error))
goto out;
@ -1295,6 +1300,20 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
cancellable, error))
goto out;
g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
if (metadata != NULL)
{
GVariantIter iter;
GVariant *item;
g_variant_iter_init (&iter, metadata);
while ((item = g_variant_iter_next_value (&iter)))
{
g_variant_builder_add (&metadata_builder, "@{sv}", item);
g_variant_unref (item);
}
}
part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT));
part_tempfiles = g_ptr_array_new_with_free_func (g_object_unref);
for (i = 0; i < builder.parts->len; i++)
@ -1359,10 +1378,16 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
delta_part = g_variant_new ("(y@ay)",
compression_type_char,
ot_gvariant_new_ay_bytes (g_memory_output_stream_steal_as_bytes (part_payload_out)));
g_variant_ref_sink (delta_part);
if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
&part_tempfile, &part_temp_outstream,
cancellable, error))
if (inline_parts)
{
g_autofree char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
g_variant_builder_add (&metadata_builder, "{sv}", part_relpath, delta_part);
}
else if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
&part_tempfile, &part_temp_outstream,
cancellable, error))
goto out;
part_in = ot_variant_read (delta_part);
if (!ot_gio_splice_get_checksum (part_temp_outstream, part_in,
@ -1378,8 +1403,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
(guint64) g_variant_get_size (delta_part),
part_builder->uncompressed_size,
ot_gvariant_new_ay_bytes (objtype_checksum_array));
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
if (part_tempfile)
g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
total_compressed_size += g_variant_get_size (delta_part);
total_uncompressed_size += part_builder->uncompressed_size;
@ -1400,7 +1427,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
if (!gs_file_ensure_directory (descriptor_dir, TRUE, cancellable, error))
goto out;
for (i = 0; i < builder.parts->len; i++)
for (i = 0; i < part_tempfiles->len; i++)
{
GFile *tempfile = part_tempfiles->pdata[i];
g_autofree char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
@ -1410,13 +1437,6 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
goto out;
}
if (metadata != NULL)
metadata_source = metadata;
else
{
metadata_source = ot_gvariant_new_empty_string_dict ();
}
if (!get_fallback_headers (self, &builder, &fallback_headers,
cancellable, error))
goto out;
@ -1432,7 +1452,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
delta_descriptor = g_variant_new ("(@a{sv}t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
"a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
"@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
metadata_source,
g_variant_builder_end (&metadata_builder),
GUINT64_TO_BE (g_date_time_to_unix (now)),
from_csum_v,
to_csum_v,

45
src/libostree/ostree-repo-static-delta-core.c
View File

@ -20,6 +20,7 @@
#include "config.h"
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
#include "ostree-repo-static-delta-private.h"
#include "otutil.h"
@ -227,7 +228,10 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
g_autoptr(GFile) meta_file = g_file_get_child (dir, "superblock");
g_autoptr(GVariant) meta = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) fallback = NULL;
g_autofree char *to_checksum = NULL;
g_autofree char *from_checksum = NULL;
if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
FALSE, &meta, error))
@ -238,7 +242,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
/* Write the to-commit object */
{
g_autoptr(GVariant) to_csum_v = NULL;
g_autofree char *to_checksum = NULL;
g_autoptr(GVariant) from_csum_v = NULL;
g_autoptr(GVariant) to_commit = NULL;
gboolean have_to_commit;
@ -247,6 +251,14 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
goto out;
to_checksum = ostree_checksum_from_bytes_v (to_csum_v);
from_csum_v = g_variant_get_child_value (meta, 2);
if (g_variant_n_children (from_csum_v) > 0)
{
if (!ostree_validate_structureof_csum_v (from_csum_v, error))
goto out;
from_checksum = ostree_checksum_from_bytes_v (from_csum_v);
}
if (!ostree_repo_has_object (self, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
&have_to_commit, cancellable, error))
goto out;
@ -270,6 +282,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
}
headers = g_variant_get_child_value (meta, 6);
metadata = g_variant_get_child_value (meta, 0);
n = g_variant_n_children (headers);
for (i = 0; i < n; i++)
{
@ -278,12 +291,15 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
guint64 usize;
const guchar *csum;
gboolean have_all;
g_autoptr(GBytes) delta_data = NULL;
g_autoptr(GVariant) part_data = NULL;
g_autoptr(GVariant) header = NULL;
g_autoptr(GVariant) csum_v = NULL;
g_autoptr(GVariant) objects = NULL;
g_autoptr(GFile) part_path = NULL;
g_autoptr(GInputStream) raw_in = NULL;
g_autoptr(GInputStream) in = NULL;
g_autofree char *deltapart_path = NULL;
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
@ -309,9 +325,17 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
if (!csum)
goto out;
part_path = ot_gfile_resolve_path_printf (dir, "%u", i);
deltapart_path =
_ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i);
in = (GInputStream*)g_file_read (part_path, cancellable, error);
part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)"));
if (part_data)
in = ot_variant_read (part_data);
else
{
part_path = ot_gfile_resolve_path_printf (dir, "%u", i);
in = (GInputStream*)g_file_read (part_path, cancellable, error);
}
if (!in)
goto out;
@ -325,14 +349,19 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
}
{
GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
g_autoptr(GBytes) bytes = NULL;
if (!mfile)
goto out;
if (part_data)
bytes = g_variant_get_data_as_bytes (part_data);
else
{
GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
if (!mfile)
goto out;
bytes = g_mapped_file_get_bytes (mfile);
g_mapped_file_unref (mfile);
bytes = g_mapped_file_get_bytes (mfile);
g_mapped_file_unref (mfile);
}
if (!_ostree_static_delta_part_execute (self, objects, bytes,
cancellable, error))