deltas: Use endianness marker when parsing

Extend the `static-delta show` and `pull` commands to use the
endianness information (if available).
This commit is contained in:
Colin Walters 2016-02-24 14:29:56 -05:00
parent 277220aaa6
commit 04d77da005
7 changed files with 201 additions and 36 deletions

View File

@ -1443,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
static gboolean
process_one_static_delta_fallback (OtPullData *pull_data,
gboolean delta_byteswap,
GVariant *fallback_object,
GCancellable *cancellable,
GError **error)
@ -1462,6 +1463,9 @@ process_one_static_delta_fallback (OtPullData *pull_data,
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
pull_data->total_deltapart_size += compressed_size;
pull_data->total_deltapart_usize += uncompressed_size;
@ -1518,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data,
GError **error)
{
gboolean ret = FALSE;
gboolean delta_byteswap;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) fallback_objects = NULL;
guint i, n;
delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock);
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
metadata = g_variant_get_child_value (delta_superblock, 0);
headers = g_variant_get_child_value (delta_superblock, 6);
@ -1535,7 +1542,7 @@ process_one_static_delta (OtPullData *pull_data,
g_autoptr(GVariant) fallback_object =
g_variant_get_child_value (fallback_objects, i);
if (!process_one_static_delta_fallback (pull_data,
if (!process_one_static_delta_fallback (pull_data, delta_byteswap,
fallback_object,
cancellable, error))
goto out;
@ -1609,6 +1616,10 @@ process_one_static_delta (OtPullData *pull_data,
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
version = maybe_swap_endian_u32 (delta_byteswap, version);
size = maybe_swap_endian_u64 (delta_byteswap, size);
usize = maybe_swap_endian_u64 (delta_byteswap, usize);
if (version > OSTREE_DELTAPART_VERSION)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,

View File

@ -59,6 +59,7 @@ typedef struct {
guint n_rollsum;
guint n_bsdiff;
guint n_fallback;
gboolean swap_endian;
} OstreeStaticDeltaBuilder;
typedef enum {
@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self,
g_variant_new ("(y@aytt)",
objtype,
ostree_checksum_to_bytes_v (checksum),
compressed_size, uncompressed_size));
maybe_swap_endian_u64 (builder->swap_endian, compressed_size),
maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size)));
}
ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self,
* - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE.
* - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE.
* - verbose: b: Print diagnostic messages. Default FALSE.
* - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN)
* - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository.
*/
gboolean
@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
g_autoptr(GVariant) fallback_headers = NULL;
g_autoptr(GVariant) detached = NULL;
gboolean inline_parts;
guint endianness = G_BYTE_ORDER;
g_autoptr(GFile) tmp_dir = NULL;
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
max_chunk_size = 32;
builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000;
(void) g_variant_lookup (params, "endianness", "u", &endianness);
g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE);
builder.swap_endian = endianness != G_BYTE_ORDER;
{ gboolean use_bsdiff;
if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff))
use_bsdiff = TRUE;
@ -1325,7 +1334,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
}
{ guint8 endianness_char;
switch (G_BYTE_ORDER)
switch (endianness)
{
case G_LITTLE_ENDIAN:
endianness_char = 'l';
@ -1433,10 +1443,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
delta_part_header = g_variant_new ("(u@aytt@ay)",
OSTREE_DELTAPART_VERSION,
maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION),
ot_gvariant_new_ay_bytes (checksum_bytes),
(guint64) g_variant_get_size (delta_part),
part_builder->uncompressed_size,
maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)),
maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size),
ot_gvariant_new_ay_bytes (objtype_checksum_array));
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));

View File

@ -589,6 +589,7 @@ _ostree_static_delta_part_open (GInputStream *part_in,
static gboolean
show_one_part (OstreeRepo *self,
gboolean swap_endian,
const char *from,
const char *to,
GVariant *meta_entries,
@ -608,6 +609,8 @@ show_one_part (OstreeRepo *self,
gint part_fd = -1;
g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
*total_size_ref += size;
*total_usize_ref += usize;
g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n",
@ -666,6 +669,45 @@ show_one_part (OstreeRepo *self,
return ret;
}
OstreeDeltaEndianness
_ostree_delta_get_endianness (GVariant *superblock)
{
guint8 endianness_char;
g_autoptr(GVariant) delta_meta = NULL;
g_autoptr(GVariantDict) delta_metadict = NULL;
delta_meta = g_variant_get_child_value (superblock, 0);
delta_metadict = g_variant_dict_new (delta_meta);
if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
{
switch (endianness_char)
{
case 'l':
return OSTREE_DELTA_ENDIAN_LITTLE;
case 'B':
return OSTREE_DELTA_ENDIAN_BIG;
default:
return OSTREE_DELTA_ENDIAN_INVALID;
}
}
return OSTREE_DELTA_ENDIAN_UNKNOWN;
}
gboolean
_ostree_delta_needs_byteswap (GVariant *superblock)
{
switch (_ostree_delta_get_endianness (superblock))
{
case OSTREE_DELTA_ENDIAN_BIG:
return G_BYTE_ORDER == G_LITTLE_ENDIAN;
case OSTREE_DELTA_ENDIAN_LITTLE:
return G_BYTE_ORDER == G_BIG_ENDIAN;
default:
return FALSE;
}
}
gboolean
_ostree_repo_static_delta_dump (OstreeRepo *self,
const char *delta_id,
@ -678,11 +720,11 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
g_autofree char *superblock_path = NULL;
glnx_fd_close int superblock_fd = -1;
g_autoptr(GVariant) delta_superblock = NULL;
g_autoptr(GVariant) delta_meta = NULL;
g_autoptr(GVariantDict) delta_metadict = NULL;
guint64 total_size = 0, total_usize = 0;
guint64 total_fallback_size = 0, total_fallback_usize = 0;
guint i;
OstreeDeltaEndianness endianness;
gboolean swap_endian = FALSE;
_ostree_parse_delta_name (delta_id, &from, &to);
superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
@ -692,30 +734,34 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
TRUE, &delta_superblock, error))
goto out;
delta_meta = g_variant_get_child_value (delta_superblock, 0);
delta_metadict = g_variant_dict_new (delta_meta);
g_print ("%s\n", g_variant_print (delta_superblock, 1));
g_print ("Delta: %s\n", delta_id);
{ guint8 endianness_char;
const char *endianness_description;
{ const char *endianness_description;
if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
endianness = _ostree_delta_get_endianness (delta_superblock);
switch (endianness)
{
switch (endianness_char)
{
case 'l':
endianness_description = "little";
break;
case 'B':
endianness_description = "big";
break;
default:
endianness_description = "invalid";
break;
}
case OSTREE_DELTA_ENDIAN_BIG:
endianness_description = "big";
if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
swap_endian = TRUE;
break;
case OSTREE_DELTA_ENDIAN_LITTLE:
endianness_description = "little";
if (G_BYTE_ORDER == G_BIG_ENDIAN)
swap_endian = TRUE;
break;
case OSTREE_DELTA_ENDIAN_UNKNOWN:
endianness_description = "unknown";
break;
case OSTREE_DELTA_ENDIAN_INVALID:
endianness_description = "invalid";
break;
default:
g_assert_not_reached ();
}
else
endianness_description = "unknown";
g_print ("Endianness: %s\n", endianness_description);
}
@ -739,6 +785,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
{
guint64 size, usize;
g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
total_fallback_size += size;
total_fallback_usize += usize;
}
@ -757,7 +805,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
for (i = 0; i < n_parts; i++)
{
if (!show_one_part (self, from, to, meta_entries, i,
if (!show_one_part (self, swap_endian, from, to, meta_entries, i,
&total_size, &total_usize,
cancellable, error))
goto out;

View File

@ -46,10 +46,10 @@ G_BEGIN_DECLS
/**
* OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
*
* u: version
* u: version (non-canonical endian)
* ay checksum
* guint64 size: Total size of delta (sum of parts)
* guint64 usize: Uncompressed size of resulting objects on disk
* guint64 size: Total size of delta (sum of parts) (non-canonical endian)
* guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian)
* ARRAY[(guint8 objtype, csum object)]
*
* The checksum is of the delta payload, and each entry in the array
@ -64,8 +64,8 @@ G_BEGIN_DECLS
*
* y: objtype
* ay: checksum
* t: compressed size
* t: uncompressed size
* t: compressed size (non-canonical endian)
* t: uncompressed size (non-canonical endian)
*
* Object to fetch invididually; includes compressed/uncompressed size.
*/
@ -79,7 +79,7 @@ G_BEGIN_DECLS
*
* delta-descriptor:
* metadata: a{sv}
* t: timestamp
* t: timestamp (big endian)
* from: ay checksum
* to: ay checksum
* commit: new commit object
@ -196,4 +196,38 @@ _ostree_repo_static_delta_dump (OstreeRepo *repo,
GCancellable *cancellable,
GError **error);
/* Used for static deltas which due to a historical mistake are
* inconsistent endian.
*
* https://bugzilla.gnome.org/show_bug.cgi?id=762515
*/
static inline guint32
maybe_swap_endian_u32 (gboolean swap,
guint32 v)
{
if (!swap)
return v;
return GUINT32_SWAP_LE_BE (v);
}
static inline guint64
maybe_swap_endian_u64 (gboolean swap,
guint64 v)
{
if (!swap)
return v;
return GUINT64_SWAP_LE_BE (v);
}
typedef enum {
OSTREE_DELTA_ENDIAN_BIG,
OSTREE_DELTA_ENDIAN_LITTLE,
OSTREE_DELTA_ENDIAN_UNKNOWN,
OSTREE_DELTA_ENDIAN_INVALID
} OstreeDeltaEndianness;
OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock);
gboolean _ostree_delta_needs_byteswap (GVariant *superblock);
G_END_DECLS

View File

@ -32,7 +32,9 @@ static char *opt_to_rev;
static char *opt_min_fallback_size;
static char *opt_max_bsdiff_size;
static char *opt_max_chunk_size;
static char *opt_endianness;
static gboolean opt_empty;
static gboolean opt_swap_endianness;
static gboolean opt_inline;
static gboolean opt_disable_bsdiff;
@ -60,6 +62,8 @@ static GOptionEntry generate_options[] = {
{ "inline", 0, 0, G_OPTION_ARG_NONE, &opt_inline, "Inline delta parts into main delta", NULL },
{ "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" },
{ "disable-bsdiff", 0, 0, G_OPTION_ARG_NONE, &opt_disable_bsdiff, "Disable use of bsdiff", NULL },
{ "set-endianness", 0, 0, G_OPTION_ARG_STRING, &opt_endianness, "Choose metadata endianness ('l' or 'B')", "ENDIAN" },
{ "swap-endianness", 0, 0, G_OPTION_ARG_NONE, &opt_swap_endianness, "Swap metadata endianness from host order", NULL },
{ "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL},
{ "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL},
{ "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL},
@ -194,6 +198,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
g_autofree char *to_resolved = NULL;
g_autofree char *from_parent_str = NULL;
g_autoptr(GVariantBuilder) parambuilder = NULL;
int endianness;
g_assert (opt_to_rev);
@ -224,6 +229,37 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
}
if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error))
goto out;
if (opt_endianness)
{
if (strcmp (opt_endianness, "l") == 0)
endianness = G_LITTLE_ENDIAN;
else if (strcmp (opt_endianness, "B") == 0)
endianness = G_BIG_ENDIAN;
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid endianness '%s'", opt_endianness);
goto out;
}
}
else
endianness = G_BYTE_ORDER;
if (opt_swap_endianness)
{
switch (endianness)
{
case G_LITTLE_ENDIAN:
endianness = G_BIG_ENDIAN;
break;
case G_BIG_ENDIAN:
endianness = G_LITTLE_ENDIAN;
break;
default:
g_assert_not_reached ();
}
}
parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
if (opt_min_fallback_size)
@ -243,6 +279,8 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
"inline-parts", g_variant_new_boolean (TRUE));
g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE));
if (opt_endianness || opt_swap_endianness)
g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness));
g_print ("Generating static delta:\n");
g_print (" From: %s\n", from_resolved ? from_resolved : "empty");

View File

@ -113,8 +113,8 @@ ostree --repo=ostree-srv/gnomerepo summary -u
cd ${test_tmpdir}
repo_init
${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt
assert_file_has_content out.txt 'Delta update: 0/1 parts'
${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt
assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts'
rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
assert_streq "${prev_rev}" "${rev}"
${CMD_PREFIX} ostree --repo=repo fsck
@ -142,6 +142,21 @@ assert_not_has_file baz/saucer
echo "ok static delta"
cd ${test_tmpdir}
${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main
${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
repo_init
${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt
${CMD_PREFIX} ostree --repo=repo fsck
if ! diff -u dry-run-pull.txt byteswapped-dry-run-pull.txt; then
assert_not_reached "byteswapped delta differs in size"
fi
echo "ok pull byteswapped delta"
cd ${test_tmpdir}
rm ostree-srv/gnomerepo/deltas -rf
ostree --repo=ostree-srv/gnomerepo summary -u

View File

@ -119,6 +119,15 @@ assert_file_has_content show.txt 'Endianness: \(little\|big\)'
echo 'ok show'
${CMD_PREFIX} ostree --repo=repo static-delta generate --swap-endianness --from=${origrev} --to=${newrev}
${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show-swapped.txt
totalsize_orig=$(grep 'Total Size:' show.txt)
totalsize_swapped=$(grep 'Total Size:' show-swapped.txt)
assert_not_streq "${totalsize_orig}" ""
assert_streq "${totalsize_orig}" "${totalsize_swapped}"
echo 'ok generate + show endian swapped'
mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2
${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
${CMD_PREFIX} ostree --repo=repo2 fsck