diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index efa424f6..0be1b380 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1443,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data, static gboolean process_one_static_delta_fallback (OtPullData *pull_data, + gboolean delta_byteswap, GVariant *fallback_object, GCancellable *cancellable, GError **error) @@ -1462,6 +1463,9 @@ process_one_static_delta_fallback (OtPullData *pull_data, if (!ostree_validate_structureof_csum_v (csum_v, error)) goto out; + compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size); + uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size); + pull_data->total_deltapart_size += compressed_size; pull_data->total_deltapart_usize += uncompressed_size; @@ -1518,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data, GError **error) { gboolean ret = FALSE; + gboolean delta_byteswap; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) headers = NULL; g_autoptr(GVariant) fallback_objects = NULL; guint i, n; + delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock); + /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ metadata = g_variant_get_child_value (delta_superblock, 0); headers = g_variant_get_child_value (delta_superblock, 6); @@ -1535,7 +1542,7 @@ process_one_static_delta (OtPullData *pull_data, g_autoptr(GVariant) fallback_object = g_variant_get_child_value (fallback_objects, i); - if (!process_one_static_delta_fallback (pull_data, + if (!process_one_static_delta_fallback (pull_data, delta_byteswap, fallback_object, cancellable, error)) goto out; @@ -1609,6 +1616,10 @@ process_one_static_delta (OtPullData *pull_data, header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); + version = maybe_swap_endian_u32 (delta_byteswap, version); + size = maybe_swap_endian_u64 (delta_byteswap, size); + usize = maybe_swap_endian_u64 (delta_byteswap, usize); + if (version > OSTREE_DELTAPART_VERSION) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index e579e910..2d02e6a6 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -59,6 +59,7 @@ typedef struct { guint n_rollsum; guint n_bsdiff; guint n_fallback; + gboolean swap_endian; } OstreeStaticDeltaBuilder; typedef enum { @@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self, g_variant_new ("(y@aytt)", objtype, ostree_checksum_to_bytes_v (checksum), - compressed_size, uncompressed_size)); + maybe_swap_endian_u64 (builder->swap_endian, compressed_size), + maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size))); } ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder)); @@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self, * - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE. * - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE. * - verbose: b: Print diagnostic messages. Default FALSE. + * - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN) * - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository. */ gboolean @@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GVariant) fallback_headers = NULL; g_autoptr(GVariant) detached = NULL; gboolean inline_parts; + guint endianness = G_BYTE_ORDER; g_autoptr(GFile) tmp_dir = NULL; builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref); builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); @@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self, max_chunk_size = 32; builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000; + (void) g_variant_lookup (params, "endianness", "u", &endianness); + g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE); + + builder.swap_endian = endianness != G_BYTE_ORDER; + { gboolean use_bsdiff; if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff)) use_bsdiff = TRUE; @@ -1325,7 +1334,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } { guint8 endianness_char; - switch (G_BYTE_ORDER) + + switch (endianness) { case G_LITTLE_ENDIAN: endianness_char = 'l'; @@ -1433,10 +1443,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self, checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN); objtype_checksum_array = objtype_checksum_array_new (part_builder->objects); delta_part_header = g_variant_new ("(u@aytt@ay)", - OSTREE_DELTAPART_VERSION, + maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION), ot_gvariant_new_ay_bytes (checksum_bytes), - (guint64) g_variant_get_size (delta_part), - part_builder->uncompressed_size, + maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)), + maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size), ot_gvariant_new_ay_bytes (objtype_checksum_array)); g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header)); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 6b1dc571..97cdb8c5 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -589,6 +589,7 @@ _ostree_static_delta_part_open (GInputStream *part_in, static gboolean show_one_part (OstreeRepo *self, + gboolean swap_endian, const char *from, const char *to, GVariant *meta_entries, @@ -608,6 +609,8 @@ show_one_part (OstreeRepo *self, gint part_fd = -1; g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); *total_size_ref += size; *total_usize_ref += usize; g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", @@ -666,6 +669,45 @@ show_one_part (OstreeRepo *self, return ret; } +OstreeDeltaEndianness +_ostree_delta_get_endianness (GVariant *superblock) +{ + guint8 endianness_char; + g_autoptr(GVariant) delta_meta = NULL; + g_autoptr(GVariantDict) delta_metadict = NULL; + + delta_meta = g_variant_get_child_value (superblock, 0); + delta_metadict = g_variant_dict_new (delta_meta); + + if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) + { + switch (endianness_char) + { + case 'l': + return OSTREE_DELTA_ENDIAN_LITTLE; + case 'B': + return OSTREE_DELTA_ENDIAN_BIG; + default: + return OSTREE_DELTA_ENDIAN_INVALID; + } + } + return OSTREE_DELTA_ENDIAN_UNKNOWN; +} + +gboolean +_ostree_delta_needs_byteswap (GVariant *superblock) +{ + switch (_ostree_delta_get_endianness (superblock)) + { + case OSTREE_DELTA_ENDIAN_BIG: + return G_BYTE_ORDER == G_LITTLE_ENDIAN; + case OSTREE_DELTA_ENDIAN_LITTLE: + return G_BYTE_ORDER == G_BIG_ENDIAN; + default: + return FALSE; + } +} + gboolean _ostree_repo_static_delta_dump (OstreeRepo *self, const char *delta_id, @@ -678,11 +720,11 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_autofree char *superblock_path = NULL; glnx_fd_close int superblock_fd = -1; g_autoptr(GVariant) delta_superblock = NULL; - g_autoptr(GVariant) delta_meta = NULL; - g_autoptr(GVariantDict) delta_metadict = NULL; guint64 total_size = 0, total_usize = 0; guint64 total_fallback_size = 0, total_fallback_usize = 0; guint i; + OstreeDeltaEndianness endianness; + gboolean swap_endian = FALSE; _ostree_parse_delta_name (delta_id, &from, &to); superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); @@ -692,30 +734,34 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, TRUE, &delta_superblock, error)) goto out; - delta_meta = g_variant_get_child_value (delta_superblock, 0); - delta_metadict = g_variant_dict_new (delta_meta); + g_print ("%s\n", g_variant_print (delta_superblock, 1)); g_print ("Delta: %s\n", delta_id); - { guint8 endianness_char; - const char *endianness_description; + { const char *endianness_description; - if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) + endianness = _ostree_delta_get_endianness (delta_superblock); + + switch (endianness) { - switch (endianness_char) - { - case 'l': - endianness_description = "little"; - break; - case 'B': - endianness_description = "big"; - break; - default: - endianness_description = "invalid"; - break; - } + case OSTREE_DELTA_ENDIAN_BIG: + endianness_description = "big"; + if (G_BYTE_ORDER == G_LITTLE_ENDIAN) + swap_endian = TRUE; + break; + case OSTREE_DELTA_ENDIAN_LITTLE: + endianness_description = "little"; + if (G_BYTE_ORDER == G_BIG_ENDIAN) + swap_endian = TRUE; + break; + case OSTREE_DELTA_ENDIAN_UNKNOWN: + endianness_description = "unknown"; + break; + case OSTREE_DELTA_ENDIAN_INVALID: + endianness_description = "invalid"; + break; + default: + g_assert_not_reached (); } - else - endianness_description = "unknown"; g_print ("Endianness: %s\n", endianness_description); } @@ -739,6 +785,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, { guint64 size, usize; g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); total_fallback_size += size; total_fallback_usize += usize; } @@ -757,7 +805,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, for (i = 0; i < n_parts; i++) { - if (!show_one_part (self, from, to, meta_entries, i, + if (!show_one_part (self, swap_endian, from, to, meta_entries, i, &total_size, &total_usize, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 2da000d6..d9e5c456 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -46,10 +46,10 @@ G_BEGIN_DECLS /** * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT: * - * u: version + * u: version (non-canonical endian) * ay checksum - * guint64 size: Total size of delta (sum of parts) - * guint64 usize: Uncompressed size of resulting objects on disk + * guint64 size: Total size of delta (sum of parts) (non-canonical endian) + * guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian) * ARRAY[(guint8 objtype, csum object)] * * The checksum is of the delta payload, and each entry in the array @@ -64,8 +64,8 @@ G_BEGIN_DECLS * * y: objtype * ay: checksum - * t: compressed size - * t: uncompressed size + * t: compressed size (non-canonical endian) + * t: uncompressed size (non-canonical endian) * * Object to fetch invididually; includes compressed/uncompressed size. */ @@ -79,7 +79,7 @@ G_BEGIN_DECLS * * delta-descriptor: * metadata: a{sv} - * t: timestamp + * t: timestamp (big endian) * from: ay checksum * to: ay checksum * commit: new commit object @@ -196,4 +196,38 @@ _ostree_repo_static_delta_dump (OstreeRepo *repo, GCancellable *cancellable, GError **error); +/* Used for static deltas which due to a historical mistake are + * inconsistent endian. + * + * https://bugzilla.gnome.org/show_bug.cgi?id=762515 + */ +static inline guint32 +maybe_swap_endian_u32 (gboolean swap, + guint32 v) +{ + if (!swap) + return v; + return GUINT32_SWAP_LE_BE (v); +} + +static inline guint64 +maybe_swap_endian_u64 (gboolean swap, + guint64 v) +{ + if (!swap) + return v; + return GUINT64_SWAP_LE_BE (v); +} + +typedef enum { + OSTREE_DELTA_ENDIAN_BIG, + OSTREE_DELTA_ENDIAN_LITTLE, + OSTREE_DELTA_ENDIAN_UNKNOWN, + OSTREE_DELTA_ENDIAN_INVALID +} OstreeDeltaEndianness; + +OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock); + +gboolean _ostree_delta_needs_byteswap (GVariant *superblock); + G_END_DECLS diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index c3c99ba0..36fb63fa 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -32,7 +32,9 @@ static char *opt_to_rev; static char *opt_min_fallback_size; static char *opt_max_bsdiff_size; static char *opt_max_chunk_size; +static char *opt_endianness; static gboolean opt_empty; +static gboolean opt_swap_endianness; static gboolean opt_inline; static gboolean opt_disable_bsdiff; @@ -60,6 +62,8 @@ static GOptionEntry generate_options[] = { { "inline", 0, 0, G_OPTION_ARG_NONE, &opt_inline, "Inline delta parts into main delta", NULL }, { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" }, { "disable-bsdiff", 0, 0, G_OPTION_ARG_NONE, &opt_disable_bsdiff, "Disable use of bsdiff", NULL }, + { "set-endianness", 0, 0, G_OPTION_ARG_STRING, &opt_endianness, "Choose metadata endianness ('l' or 'B')", "ENDIAN" }, + { "swap-endianness", 0, 0, G_OPTION_ARG_NONE, &opt_swap_endianness, "Swap metadata endianness from host order", NULL }, { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL}, { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL}, { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL}, @@ -194,6 +198,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab g_autofree char *to_resolved = NULL; g_autofree char *from_parent_str = NULL; g_autoptr(GVariantBuilder) parambuilder = NULL; + int endianness; g_assert (opt_to_rev); @@ -224,6 +229,37 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab } if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error)) goto out; + + if (opt_endianness) + { + if (strcmp (opt_endianness, "l") == 0) + endianness = G_LITTLE_ENDIAN; + else if (strcmp (opt_endianness, "B") == 0) + endianness = G_BIG_ENDIAN; + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid endianness '%s'", opt_endianness); + goto out; + } + } + else + endianness = G_BYTE_ORDER; + + if (opt_swap_endianness) + { + switch (endianness) + { + case G_LITTLE_ENDIAN: + endianness = G_BIG_ENDIAN; + break; + case G_BIG_ENDIAN: + endianness = G_LITTLE_ENDIAN; + break; + default: + g_assert_not_reached (); + } + } parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); if (opt_min_fallback_size) @@ -243,6 +279,8 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab "inline-parts", g_variant_new_boolean (TRUE)); g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE)); + if (opt_endianness || opt_swap_endianness) + g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness)); g_print ("Generating static delta:\n"); g_print (" From: %s\n", from_resolved ? from_resolved : "empty"); diff --git a/tests/pull-test.sh b/tests/pull-test.sh index f73b6818..9c8b41fa 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -113,8 +113,8 @@ ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} -${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt -assert_file_has_content out.txt 'Delta update: 0/1 parts' +${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt +assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts' rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) assert_streq "${prev_rev}" "${rev}" ${CMD_PREFIX} ostree --repo=repo fsck @@ -142,6 +142,21 @@ assert_not_has_file baz/saucer echo "ok static delta" +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u + +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt +${CMD_PREFIX} ostree --repo=repo fsck + +if ! diff -u dry-run-pull.txt byteswapped-dry-run-pull.txt; then + assert_not_reached "byteswapped delta differs in size" +fi + +echo "ok pull byteswapped delta" + cd ${test_tmpdir} rm ostree-srv/gnomerepo/deltas -rf ostree --repo=ostree-srv/gnomerepo summary -u diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 84ce8a7c..4679ab8e 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -119,6 +119,15 @@ assert_file_has_content show.txt 'Endianness: \(little\|big\)' echo 'ok show' +${CMD_PREFIX} ostree --repo=repo static-delta generate --swap-endianness --from=${origrev} --to=${newrev} +${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show-swapped.txt +totalsize_orig=$(grep 'Total Size:' show.txt) +totalsize_swapped=$(grep 'Total Size:' show-swapped.txt) +assert_not_streq "${totalsize_orig}" "" +assert_streq "${totalsize_orig}" "${totalsize_swapped}" + +echo 'ok generate + show endian swapped' + mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 fsck