From 5a82b141aed8396a4153ffb9d7e98927328a08b0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 9 Apr 2012 22:46:08 -0400 Subject: [PATCH] core: Split pack files into data/metadata This will allow us to download metadata first (separately), and in the future we can have an improved metadata pack format that compresses heavily. --- src/libostree/ostree-core.c | 113 ++++-- src/libostree/ostree-core.h | 41 ++- src/libostree/ostree-repo.c | 605 ++++++++++++++++++++------------- src/libostree/ostree-repo.h | 26 +- src/ostree/ostree-pull.c | 90 +++-- src/ostree/ot-builtin-fsck.c | 170 +++++---- src/ostree/ot-builtin-pack.c | 387 ++++++++++++--------- src/ostree/ot-builtin-unpack.c | 45 ++- 8 files changed, 922 insertions(+), 555 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index f7b2272a..c3290a59 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -710,6 +710,58 @@ ostree_get_relative_object_path (const char *checksum, return g_string_free (path, FALSE); } +static char * +get_pack_name (gboolean is_meta, + gboolean is_index, + const char *prefix, + const char *checksum) +{ + GString *path; + + g_assert (strlen (checksum) == 64); + + path = g_string_new (prefix); + if (is_meta) + g_string_append (path, "ostmetapack-"); + else + g_string_append (path, "ostdatapack-"); + g_string_append (path, checksum); + if (is_index) + g_string_append (path, ".index"); + else + g_string_append (path, ".data"); + + return g_string_free (path, FALSE); +} + +char * +ostree_get_pack_index_name (gboolean is_meta, + const char *checksum) +{ + return get_pack_name (is_meta, TRUE, "", checksum); +} + +char * +ostree_get_pack_data_name (gboolean is_meta, + const char *checksum) +{ + return get_pack_name (is_meta, FALSE, "", checksum); +} + +char * +ostree_get_relative_pack_index_path (gboolean is_meta, + const char *checksum) +{ + return get_pack_name (is_meta, TRUE, "objects/pack/", checksum); +} + +char * +ostree_get_relative_pack_data_path (gboolean is_meta, + const char *checksum) +{ + return get_pack_name (is_meta, FALSE, "objects/pack/", checksum); +} + GVariant * ostree_create_archive_file_metadata (GFileInfo *finfo, GVariant *xattrs) @@ -1169,6 +1221,7 @@ ostree_read_pack_entry_raw (guchar *pack_data, guint64 pack_len, guint64 offset, gboolean trusted, + gboolean is_meta, GVariant **out_entry, GCancellable *cancellable, GError **error) @@ -1215,7 +1268,8 @@ ostree_read_pack_entry_raw (guchar *pack_data, goto out; } - ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT, + ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT : + OSTREE_PACK_DATA_FILE_VARIANT_FORMAT, pack_data+entry_start, entry_len, trusted, NULL, NULL); g_variant_ref_sink (ret_entry); @@ -1274,32 +1328,13 @@ ostree_read_pack_entry_variant (GVariant *pack_entry, GError **error) { gboolean ret = FALSE; - guint32 actual_type; - ot_lobj GInputStream *stream = NULL; - ot_lvariant GVariant *container_variant = NULL; ot_lvariant GVariant *ret_variant = NULL; - stream = ostree_read_pack_entry_as_stream (pack_entry); - - if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT, - trusted, &container_variant, cancellable, error)) - goto out; - - g_variant_get (container_variant, "(uv)", - &actual_type, &ret_variant); - actual_type = GUINT32_FROM_BE (actual_type); - - if (actual_type != expected_objtype) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted metadata object in pack file; found type %u, expected %u", - actual_type, (guint32)expected_objtype); - goto out; - } + g_variant_get_child (pack_entry, 2, "v", &ret_variant); ret = TRUE; ot_transfer_out_value (out_variant, &ret_variant); - out: + /* out: */ return ret; } @@ -1332,16 +1367,15 @@ ostree_pack_index_search (GVariant *index, while (imax >= imin) { GVariant *cur_csum_bytes; - guint32 cur_objtype; + guchar cur_objtype; guint64 cur_offset; gsize imid; int c; imid = (imin + imax) / 2; - g_variant_get_child (index_contents, imid, "(u@ayt)", &cur_objtype, + g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype, &cur_csum_bytes, &cur_offset); - cur_objtype = GUINT32_FROM_BE (cur_objtype); c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum); if (c == 0) @@ -1375,12 +1409,12 @@ ostree_pack_index_search (GVariant *index, } gboolean -ostree_validate_structureof_objtype (guint32 objtype, +ostree_validate_structureof_objtype (guchar objtype, GError **error) { - objtype = GUINT32_FROM_BE (objtype); - if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE - || objtype > OSTREE_OBJECT_TYPE_COMMIT) + OstreeObjectType objtype_v = (OstreeObjectType) objtype; + if (objtype_v < OSTREE_OBJECT_TYPE_RAW_FILE + || objtype_v > OSTREE_OBJECT_TYPE_COMMIT) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid object type '%u'", objtype); @@ -1542,7 +1576,8 @@ validate_stat_mode_perms (guint32 mode, GError **error) { gboolean ret = FALSE; - guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO); + guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO & + ~S_ISUID & ~S_ISGID & ~S_ISVTX); if (mode & otherbits) { @@ -1615,7 +1650,7 @@ ostree_validate_structureof_pack_index (GVariant *index, { gboolean ret = FALSE; const char *header; - guint32 objtype; + guchar objtype_u8; guint64 offset; ot_lvariant GVariant *csum_v = NULL; GVariantIter *content_iter = NULL; @@ -1632,12 +1667,12 @@ ostree_validate_structureof_pack_index (GVariant *index, goto out; } - g_variant_get_child (index, 2, "a(uayt)", &content_iter); + g_variant_get_child (index, 2, "a(yayt)", &content_iter); - while (g_variant_iter_loop (content_iter, "(u@ayt)", - &objtype, &csum_v, &offset)) + while (g_variant_iter_loop (content_iter, "(y@ayt)", + &objtype_u8, &csum_v, &offset)) { - if (!ostree_validate_structureof_objtype (objtype, error)) + if (!ostree_validate_structureof_objtype (objtype_u8, error)) goto out; if (!ostree_validate_structureof_csum_v (csum_v, error)) goto out; @@ -1674,7 +1709,15 @@ ostree_validate_structureof_pack_superindex (GVariant *superindex, } g_variant_get_child (superindex, 2, "a(ayay)", &content_iter); + while (g_variant_iter_loop (content_iter, "(@ay@ay)", + &csum_v, &bloom)) + { + if (!ostree_validate_structureof_csum_v (csum_v, error)) + goto out; + } + csum_v = NULL; + g_variant_get_child (superindex, 3, "a(ayay)", &content_iter); while (g_variant_iter_loop (content_iter, "(@ay@ay)", &csum_v, &bloom)) { diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 2c26334f..ee1823c9 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -93,34 +93,44 @@ typedef enum { /* Pack super index * s - OSTv0SUPERPACKINDEX * a{sv} - Metadata - * a(ayay) - (pack file checksum, bloom filter) + * a(ayay) - metadata packs (pack file checksum, bloom filter) + * a(ayay) - data packs (pack file checksum, bloom filter) */ -#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))") +#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))") /* Pack index * s - OSTv0PACKINDEX * a{sv} - Metadata - * a(uayt) - (objtype, checksum, offset into packfile) + * a(yayt) - (objtype, checksum, offset into packfile) */ -#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))") +#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))") typedef enum { OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0, OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0) } OstreePackFileEntryFlag; -/* Pack files - * s - OSTv0PACKFILE +/* Data Pack files + * s - OSTv0PACKDATAFILE * a{sv} - Metadata * t - number of entries * * Repeating pair of: * - * ( uyayay ) - objtype, flags, checksum, data + * ( yyayay ) - objtype, flags, checksum, data */ -#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)") +#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yyayay)") -#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)") +/* Meta Pack files + * s - OSTv0PACKMETAFILE + * a{sv} - Metadata + * t - number of entries + * + * Repeating pair of: + * + * ( yayv ) - objtype, checksum, data + */ +#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)") gboolean ostree_validate_checksum_string (const char *sha256, GError **error); @@ -162,6 +172,16 @@ void ostree_object_from_string (const char *str, char *ostree_get_relative_object_path (const char *checksum, OstreeObjectType type); +char *ostree_get_pack_index_name (gboolean is_meta, + const char *checksum); +char *ostree_get_pack_data_name (gboolean is_meta, + const char *checksum); + +char *ostree_get_relative_pack_index_path (gboolean is_meta, + const char *checksum); +char *ostree_get_relative_pack_data_path (gboolean is_meta, + const char *checksum); + gboolean ostree_get_xattrs_for_file (GFile *f, GVariant **out_xattrs, GCancellable *cancellable, @@ -260,6 +280,7 @@ gboolean ostree_read_pack_entry_raw (guchar *pack_data, guint64 pack_len, guint64 object_offset, gboolean trusted, + gboolean is_meta, GVariant **out_entry, GCancellable *cancellable, GError **error); @@ -280,7 +301,7 @@ gboolean ostree_pack_index_search (GVariant *index, /** VALIDATION **/ -gboolean ostree_validate_structureof_objtype (guint32 objtype, +gboolean ostree_validate_structureof_objtype (guchar objtype, GError **error); gboolean ostree_validate_structureof_csum_v (GVariant *checksum, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 80a65faa..428d7a6e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1507,6 +1507,7 @@ get_checksum_from_pack_name (const char *name) static gboolean list_pack_indexes_from_dir (OstreeRepo *self, + gboolean is_meta, GPtrArray **out_indexes, GCancellable *cancellable, GError **error) @@ -1518,7 +1519,7 @@ list_pack_indexes_from_dir (OstreeRepo *self, ot_lptrarray GPtrArray *ret_indexes = NULL; if (!list_files_in_dir_matching (priv->pack_dir, - "ostpack-", ".index", + is_meta ? "ostmetapack-" : "ostdatapack-", ".index", &index_files, cancellable, error)) goto out; @@ -1539,24 +1540,27 @@ list_pack_indexes_from_dir (OstreeRepo *self, static gboolean list_pack_checksums_from_superindex_file (GFile *superindex_path, - GPtrArray **out_indexes, + GPtrArray **out_meta_indexes, + GPtrArray **out_data_indexes, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; const char *magic; - ot_lptrarray GPtrArray *ret_indexes = NULL; + ot_lptrarray GPtrArray *ret_meta_indexes = NULL; + ot_lptrarray GPtrArray *ret_data_indexes = NULL; ot_lvariant GVariant *superindex_variant = NULL; ot_lvariant GVariant *checksum = NULL; ot_lvariant GVariant *bloom = NULL; - GVariantIter *variant_iter = NULL; + GVariantIter *meta_variant_iter = NULL; + GVariantIter *data_variant_iter = NULL; if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, &superindex_variant, error)) goto out; - g_variant_get (superindex_variant, "(&s@a{sv}a(ayay))", - &magic, NULL, &variant_iter); + g_variant_get (superindex_variant, "(&s@a{sv}a(ayay)a(ayay))", + &magic, NULL, &meta_variant_iter, &data_variant_iter); if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0) { @@ -1565,47 +1569,62 @@ list_pack_checksums_from_superindex_file (GFile *superindex_path, goto out; } - ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); - - while (g_variant_iter_loop (variant_iter, "(@ay@ay)", + ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + while (g_variant_iter_loop (meta_variant_iter, "(@ay@ay)", &checksum, &bloom)) - g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes_v (checksum)); + g_ptr_array_add (ret_meta_indexes, ostree_checksum_from_bytes_v (checksum)); + checksum = NULL; + bloom = NULL; + + ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + while (g_variant_iter_loop (data_variant_iter, "(@ay@ay)", + &checksum, &bloom)) + g_ptr_array_add (ret_data_indexes, ostree_checksum_from_bytes_v (checksum)); checksum = NULL; bloom = NULL; ret = TRUE; - ot_transfer_out_value (out_indexes, &ret_indexes); + ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes); + ot_transfer_out_value (out_data_indexes, &ret_data_indexes); out: - if (variant_iter) - g_variant_iter_free (variant_iter); + if (meta_variant_iter) + g_variant_iter_free (meta_variant_iter); + if (data_variant_iter) + g_variant_iter_free (data_variant_iter); return ret; } gboolean ostree_repo_list_pack_indexes (OstreeRepo *self, - GPtrArray **out_indexes, + GPtrArray **out_meta_indexes, + GPtrArray **out_data_indexes, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); ot_lobj GFile *superindex_path = NULL; - ot_lptrarray GPtrArray *ret_indexes = NULL; + ot_lptrarray GPtrArray *ret_meta_indexes = NULL; + ot_lptrarray GPtrArray *ret_data_indexes = NULL; superindex_path = g_file_get_child (priv->pack_dir, "index"); if (g_file_query_exists (superindex_path, cancellable)) { - if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_indexes, cancellable, error)) + if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_meta_indexes, + &ret_data_indexes, + cancellable, error)) goto out; } else { - ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); + ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free); } ret = TRUE; - ot_transfer_out_value (out_indexes, &ret_indexes); + ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes); + ot_transfer_out_value (out_data_indexes, &ret_data_indexes); out: return ret; } @@ -1631,6 +1650,35 @@ create_index_bloom (OstreeRepo *self, return ret; } +static gboolean +append_index_builder (OstreeRepo *self, + GPtrArray *indexes, + GVariantBuilder *builder, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint i; + + for (i = 0; i < indexes->len; i++) + { + const char *pack_checksum = indexes->pdata[i]; + ot_lvariant GVariant *bloom = NULL; + + if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error)) + goto out; + + g_variant_builder_add (builder, + "(@ay@ay)", + ostree_checksum_to_bytes_v (pack_checksum), + bloom); + } + + ret = TRUE; + out: + return ret; +} + /** * Regenerate the pack superindex file based on the set of pack * indexes currently in the filesystem. @@ -1641,69 +1689,80 @@ ostree_repo_regenerate_pack_index (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - guint i; OstreeRepoPrivate *priv = GET_PRIVATE (self); - ot_lobj GFile *index_path = NULL; + ot_lobj GFile *superindex_path = NULL; ot_lptrarray GPtrArray *pack_indexes = NULL; - ot_lvariant GVariant *index_variant = NULL; - GVariantBuilder *index_content_builder = NULL; + ot_lvariant GVariant *superindex_variant = NULL; + GVariantBuilder *meta_index_content_builder = NULL; + GVariantBuilder *data_index_content_builder = NULL; - if (!list_pack_indexes_from_dir (self, &pack_indexes, cancellable, error)) + superindex_path = g_file_get_child (priv->pack_dir, "index"); + + ot_clear_ptrarray (&pack_indexes); + if (!list_pack_indexes_from_dir (self, TRUE, &pack_indexes, + cancellable, error)) + goto out; + meta_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); + if (!append_index_builder (self, pack_indexes, meta_index_content_builder, + cancellable, error)) goto out; - index_path = g_file_get_child (priv->pack_dir, "index"); + ot_clear_ptrarray (&pack_indexes); + if (!list_pack_indexes_from_dir (self, FALSE, &pack_indexes, + cancellable, error)) + goto out; + data_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); + if (!append_index_builder (self, pack_indexes, data_index_content_builder, + cancellable, error)) + goto out; - index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)")); - - for (i = 0; i < pack_indexes->len; i++) - { - const char *pack_checksum = pack_indexes->pdata[i]; - GVariant *bloom; + superindex_variant = g_variant_new ("(s@a{sv}@a(ayay)@a(ayay))", + "OSTv0SUPERPACKINDEX", + g_variant_new_array (G_VARIANT_TYPE ("{sv}"), + NULL, 0), + g_variant_builder_end (meta_index_content_builder), + g_variant_builder_end (data_index_content_builder)); + g_variant_ref_sink (superindex_variant); - if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error)) - goto out; - - g_variant_builder_add (index_content_builder, - "(@ay@ay)", - ostree_checksum_to_bytes_v (pack_checksum), - bloom); - g_variant_unref (bloom); - } - - index_variant = g_variant_new ("(s@a{sv}@a(ayay))", - "OSTv0SUPERPACKINDEX", - g_variant_new_array (G_VARIANT_TYPE ("{sv}"), - NULL, 0), - g_variant_builder_end (index_content_builder)); - g_variant_ref_sink (index_variant); - - if (!ot_util_variant_save (index_path, index_variant, + if (!ot_util_variant_save (superindex_path, superindex_variant, cancellable, error)) goto out; ret = TRUE; out: - if (index_content_builder) - g_variant_builder_unref (index_content_builder); + if (meta_index_content_builder) + g_variant_builder_unref (meta_index_content_builder); + if (data_index_content_builder) + g_variant_builder_unref (data_index_content_builder); return ret; } - static GFile * -get_pack_index_name_from_checksum (GFile *parent, const char *pack_checksum) +get_pack_index_path (GFile *parent, + gboolean is_meta, + const char *checksum) { - return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".index", NULL); + char *path = ostree_get_pack_index_name (is_meta, checksum); + GFile *ret = g_file_resolve_relative_path (parent, path); + g_free (path); + return ret; } static GFile * -get_pack_data_name_from_checksum (GFile *parent, const char *pack_checksum) +get_pack_data_path (GFile *parent, + gboolean is_meta, + const char *checksum) { - return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".data", NULL); + char *path = ostree_get_pack_data_name (is_meta, checksum); + GFile *ret = g_file_resolve_relative_path (parent, path); + g_free (path); + return ret; } gboolean ostree_repo_add_pack_file (OstreeRepo *self, const char *pack_checksum, + gboolean is_meta, GFile *index_path, GFile *data_path, GCancellable *cancellable, @@ -1717,11 +1776,11 @@ ostree_repo_add_pack_file (OstreeRepo *self, if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error)) goto out; - pack_data_path = get_pack_data_name_from_checksum (priv->pack_dir, pack_checksum); + pack_data_path = get_pack_data_path (priv->pack_dir, is_meta, pack_checksum); if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error)) goto out; - pack_index_path = get_pack_index_name_from_checksum (priv->pack_dir, pack_checksum); + pack_index_path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum); if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error)) goto out; @@ -1752,6 +1811,80 @@ ensure_remote_cache_dir (OstreeRepo *self, return ret; } +static gboolean +delete_no_longer_referenced (OstreeRepo *self, + GFile *cache_path, + const char *prefix, + const char *suffix, + GHashTable *new_files, + GPtrArray *inout_cached, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint i; + ot_lptrarray GPtrArray *current_files = NULL; + ot_lfree char *pack_checksum = NULL; + + if (!list_files_in_dir_matching (cache_path, + prefix, suffix, + ¤t_files, + cancellable, error)) + goto out; + for (i = 0; i < current_files->len; i++) + { + GFile *file = current_files->pdata[i]; + + g_free (pack_checksum); + pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (file)); + + if (!g_hash_table_lookup (new_files, pack_checksum)) + { + if (!ot_gfile_unlink (file, cancellable, error)) + goto out; + } + + if (inout_cached) + { + g_ptr_array_add (inout_cached, pack_checksum); + pack_checksum = NULL; /* transfer ownership */ + } + } + ret = TRUE; + out: + return ret; +} + +static void +gather_uncached (GHashTable *new_files, + GPtrArray *cached, + GPtrArray *inout_uncached) +{ + guint i; + GHashTableIter hash_iter; + gpointer key, value; + + g_hash_table_iter_init (&hash_iter, new_files); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *cur_pack_checksum = key; + gboolean found = FALSE; + + for (i = 0; i < cached->len; i++) + { + const char *checksum = cached->pdata[i]; + if (strcmp (cur_pack_checksum, checksum) == 0) + { + found = TRUE; + break; + } + } + + if (!found) + g_ptr_array_add (inout_uncached, g_strdup (cur_pack_checksum)); + } +} + /** * Take a pack superindex file @superindex_path, and clean up any * no-longer-referenced pack files in the lookaside cache for @@ -1766,23 +1899,27 @@ gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, const char *remote_name, GFile *superindex_path, - GPtrArray **out_cached_indexes, - GPtrArray **out_uncached_indexes, + GPtrArray **out_cached_meta_indexes, + GPtrArray **out_cached_data_indexes, + GPtrArray **out_uncached_meta_indexes, + GPtrArray **out_uncached_data_indexes, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - GHashTableIter hash_iter; - gpointer key, value; - guint i; ot_lvariant GVariant *superindex_variant = NULL; ot_lobj GFile *cache_path = NULL; ot_lobj GFile *superindex_cache_path = NULL; - ot_lptrarray GPtrArray *index_files = NULL; - ot_lptrarray GPtrArray *data_files = NULL; - ot_lhash GHashTable *new_pack_indexes = NULL; - ot_lptrarray GPtrArray *ret_cached_indexes = NULL; - ot_lptrarray GPtrArray *ret_uncached_indexes = NULL; + ot_lptrarray GPtrArray *meta_index_files = NULL; + ot_lptrarray GPtrArray *data_index_files = NULL; + ot_lptrarray GPtrArray *meta_data_files = NULL; + ot_lptrarray GPtrArray *data_data_files = NULL; + ot_lhash GHashTable *new_pack_meta_indexes = NULL; + ot_lhash GHashTable *new_pack_data_indexes = NULL; + ot_lptrarray GPtrArray *ret_cached_meta_indexes = NULL; + ot_lptrarray GPtrArray *ret_cached_data_indexes = NULL; + ot_lptrarray GPtrArray *ret_uncached_meta_indexes = NULL; + ot_lptrarray GPtrArray *ret_uncached_data_indexes = NULL; ot_lvariant GVariant *csum_bytes = NULL; ot_lvariant GVariant *bloom = NULL; ot_lfree char *pack_checksum = NULL; @@ -1791,8 +1928,10 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error)) goto out; - ret_cached_indexes = g_ptr_array_new_with_free_func (g_free); - ret_uncached_indexes = g_ptr_array_new_with_free_func (g_free); + ret_cached_meta_indexes = g_ptr_array_new_with_free_func (g_free); + ret_cached_data_indexes = g_ptr_array_new_with_free_func (g_free); + ret_uncached_meta_indexes = g_ptr_array_new_with_free_func (g_free); + ret_uncached_data_indexes = g_ptr_array_new_with_free_func (g_free); if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, &superindex_variant, error)) @@ -1801,59 +1940,45 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, if (!ostree_validate_structureof_pack_superindex (superindex_variant, error)) goto out; - new_pack_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + new_pack_meta_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + new_pack_data_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + g_variant_get_child (superindex_variant, 2, "a(ayay)", &superindex_contents_iter); - while (g_variant_iter_loop (superindex_contents_iter, "(@ay@ay)", &csum_bytes, &bloom)) { pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); - g_hash_table_insert (new_pack_indexes, pack_checksum, pack_checksum); + g_hash_table_insert (new_pack_meta_indexes, pack_checksum, pack_checksum); pack_checksum = NULL; /* transfer ownership */ } - - if (!list_files_in_dir_matching (cache_path, - "ostpack-", ".index", - &index_files, - cancellable, error)) + + g_variant_get_child (superindex_variant, 3, "a(ayay)", + &superindex_contents_iter); + while (g_variant_iter_loop (superindex_contents_iter, + "(@ay@ay)", &csum_bytes, &bloom)) + { + pack_checksum = ostree_checksum_from_bytes_v (csum_bytes); + g_hash_table_insert (new_pack_data_indexes, pack_checksum, pack_checksum); + pack_checksum = NULL; /* transfer ownership */ + } + + if (!delete_no_longer_referenced (self, cache_path, + "ostmetapack-", ".index", + new_pack_meta_indexes, + ret_cached_meta_indexes, + cancellable, error)) goto out; - - for (i = 0; i < index_files->len; i++) - { - GFile *index_file = index_files->pdata[i]; - - g_free (pack_checksum); - pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (index_file)); - - if (!g_hash_table_lookup (new_pack_indexes, pack_checksum)) - { - if (!ot_gfile_unlink (index_file, cancellable, error)) - goto out; - } - - g_ptr_array_add (ret_cached_indexes, pack_checksum); - pack_checksum = NULL; /* transfer ownership */ - } - g_hash_table_iter_init (&hash_iter, new_pack_indexes); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *cur_pack_checksum = key; - gboolean found = FALSE; + if (!delete_no_longer_referenced (self, cache_path, + "ostdatapack-", ".index", + new_pack_data_indexes, + ret_cached_data_indexes, + cancellable, error)) + goto out; - for (i = 0; i < ret_cached_indexes->len; i++) - { - if (strcmp (cur_pack_checksum, ret_cached_indexes->pdata[i]) == 0) - { - found = TRUE; - break; - } - } - - if (!found) - g_ptr_array_add (ret_uncached_indexes, g_strdup (cur_pack_checksum)); - } + gather_uncached (new_pack_meta_indexes, ret_cached_meta_indexes, ret_uncached_meta_indexes); + gather_uncached (new_pack_data_indexes, ret_cached_data_indexes, ret_uncached_data_indexes); superindex_cache_path = g_file_get_child (cache_path, "index"); if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error)) @@ -1861,29 +1986,22 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, /* Now also delete stale pack files */ - if (!list_files_in_dir_matching (cache_path, - "ostpack-", ".data", - &data_files, - cancellable, error)) + if (!delete_no_longer_referenced (self, cache_path, + "ostmetapack-", ".data", + new_pack_meta_indexes, NULL, + cancellable, error)) + goto out; + if (!delete_no_longer_referenced (self, cache_path, + "ostdatapack-", ".data", + new_pack_data_indexes, NULL, + cancellable, error)) goto out; - - for (i = 0; i < data_files->len; i++) - { - GFile *data_file = data_files->pdata[i]; - - g_free (pack_checksum); - pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (data_file)); - - if (!g_hash_table_lookup (new_pack_indexes, pack_checksum)) - { - if (!ot_gfile_unlink (data_file, cancellable, error)) - goto out; - } - } ret = TRUE; - ot_transfer_out_value (out_cached_indexes, &ret_cached_indexes); - ot_transfer_out_value (out_uncached_indexes, &ret_uncached_indexes); + ot_transfer_out_value (out_cached_meta_indexes, &ret_cached_meta_indexes); + ot_transfer_out_value (out_cached_data_indexes, &ret_cached_data_indexes); + ot_transfer_out_value (out_uncached_meta_indexes, &ret_uncached_data_indexes); + ot_transfer_out_value (out_uncached_data_indexes, &ret_uncached_data_indexes); out: if (superindex_contents_iter) g_variant_iter_free (superindex_contents_iter); @@ -1905,11 +2023,24 @@ ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self, goto out; if (!list_files_in_dir_matching (cache_path, - "ostpack-", ".data", + "ostmetapack-", ".data", &data_files, cancellable, error)) goto out; + for (i = 0; i < data_files->len; i++) + { + GFile *data_file = data_files->pdata[i]; + + if (!ot_gfile_unlink (data_file, cancellable, error)) + goto out; + } + ot_clear_ptrarray (&data_files); + if (!list_files_in_dir_matching (cache_path, + "ostdatapack-", ".data", + &data_files, + cancellable, error)) + goto out; for (i = 0; i < data_files->len; i++) { GFile *data_file = data_files->pdata[i]; @@ -1931,6 +2062,7 @@ gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GVariant **out_variant, GCancellable *cancellable, GError **error) @@ -1944,7 +2076,7 @@ ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, cancellable, error)) goto out; - cached_pack_path = get_pack_index_name_from_checksum (cache_dir, pack_checksum); + cached_pack_path = get_pack_index_path (cache_dir, is_meta, pack_checksum); if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT, &ret_variant, error)) goto out; @@ -1964,6 +2096,7 @@ gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GFile *cached_path, GCancellable *cancellable, GError **error) @@ -1989,7 +2122,7 @@ ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) goto out; - target_path = get_pack_index_name_from_checksum (cachedir, pack_checksum); + target_path = get_pack_index_path (cachedir, is_meta, pack_checksum); if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error)) goto out; @@ -2007,6 +2140,7 @@ gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GFile **out_cached_path, GCancellable *cancellable, GError **error) @@ -2020,7 +2154,7 @@ ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, cancellable, error)) goto out; - cached_pack_path = get_pack_data_name_from_checksum (cache_dir, pack_checksum); + cached_pack_path = get_pack_data_path (cache_dir, is_meta, pack_checksum); if (g_file_query_exists (cached_pack_path, cancellable)) { ret_cached_path = cached_pack_path; @@ -2044,6 +2178,7 @@ gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GFile *cached_path, GCancellable *cancellable, GError **error) @@ -2055,8 +2190,7 @@ ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error)) goto out; - target_path = get_pack_data_name_from_checksum (cachedir, pack_checksum); - + target_path = get_pack_data_path (cachedir, is_meta, pack_checksum); if (!ot_gfile_rename (cached_path, target_path, cancellable, error)) goto out; @@ -2908,37 +3042,10 @@ list_loose_objects (OstreeRepo *self, return ret; } -GFile * -ostree_repo_get_pack_index_path (OstreeRepo *self, - const char *checksum) -{ - char *name; - GFile *ret; - - name = g_strconcat ("ostpack-", checksum, ".index", NULL); - ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name); - g_free (name); - - return ret; -} - -GFile * -ostree_repo_get_pack_data_path (OstreeRepo *self, - const char *checksum) -{ - char *name; - GFile *ret; - - name = g_strconcat ("ostpack-", checksum, ".data", NULL); - ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name); - g_free (name); - - return ret; -} - gboolean ostree_repo_load_pack_index (OstreeRepo *self, - const char *sha256, + const char *pack_checksum, + gboolean is_meta, GVariant **out_variant, GCancellable *cancellable, GError **error) @@ -2948,21 +3055,21 @@ ostree_repo_load_pack_index (OstreeRepo *self, ot_lvariant GVariant *ret_variant = NULL; ot_lobj GFile *path = NULL; - ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256); + ret_variant = g_hash_table_lookup (priv->pack_index_mappings, pack_checksum); if (ret_variant) { g_variant_ref (ret_variant); } else { - path = ostree_repo_get_pack_index_path (self, sha256); + path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum); if (!map_variant_file_check_header_string (path, OSTREE_PACK_INDEX_VARIANT_FORMAT, "OSTv0PACKINDEX", &ret_variant, cancellable, error)) goto out; - g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256), + g_hash_table_insert (priv->pack_index_mappings, g_strdup (pack_checksum), g_variant_ref (ret_variant)); } @@ -2973,19 +3080,16 @@ ostree_repo_load_pack_index (OstreeRepo *self, } /** - * ostree_repo_map_pack_file: - * @self: * @sha256: Checksum of pack file * @out_data: (out): Pointer to pack file data - * @cancellable: - * @error: * * Ensure that the given pack file is mapped into * memory. */ gboolean ostree_repo_map_pack_file (OstreeRepo *self, - const char *sha256, + const char *pack_checksum, + gboolean is_meta, guchar **out_data, guint64 *out_len, GCancellable *cancellable, @@ -2998,16 +3102,16 @@ ostree_repo_map_pack_file (OstreeRepo *self, GMappedFile *map = NULL; ot_lobj GFile *path = NULL; - map = g_hash_table_lookup (priv->pack_data_mappings, sha256); + map = g_hash_table_lookup (priv->pack_data_mappings, pack_checksum); if (map == NULL) { - path = ostree_repo_get_pack_data_path (self, sha256); + path = get_pack_data_path (priv->pack_dir, is_meta, pack_checksum); map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error); if (!map) goto out; - g_hash_table_insert (priv->pack_data_mappings, g_strdup (sha256), map); + g_hash_table_insert (priv->pack_data_mappings, g_strdup (pack_checksum), map); ret_data = g_mapped_file_get_contents (map); } @@ -3058,53 +3162,55 @@ ostree_repo_load_file (OstreeRepo *self, error)) goto out; - /* Blah, right now we need to look up the content too to get the file size */ - if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - checksum, &content_loose_path, - &content_pack_checksum, &content_pack_offset, - cancellable, error)) - goto out; - - if (content_loose_path) + if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR) { - content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); - if (!content_loose_info) + /* Blah, right now we need to look up the content too to get the file size */ + if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, + checksum, &content_loose_path, + &content_pack_checksum, &content_pack_offset, + cancellable, error)) goto out; - - g_file_info_set_attribute_uint64 (ret_file_info, - "standard::size", - g_file_info_get_attribute_uint64 (content_loose_info, "standard::size")); - } - /* fixme - don't have file size for packed =/ */ + + if (content_loose_path) + { + content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); + if (!content_loose_info) + goto out; + + g_file_info_set_attribute_uint64 (ret_file_info, + "standard::size", + g_file_info_get_attribute_uint64 (content_loose_info, "standard::size")); + } + /* fixme - don't have file size for packed =/ */ /* Now, look for the content */ - if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR - && out_input) - { - if (content_pack_checksum != NULL) + if (out_input) { - if (!ostree_repo_map_pack_file (self, content_pack_checksum, - &content_pack_data, &content_pack_len, - cancellable, error)) - goto out; - if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len, - content_pack_offset, TRUE, - &packed_object, cancellable, error)) - goto out; - ret_input = ostree_read_pack_entry_as_stream (packed_object); - } - else if (content_loose_path != NULL) - { - ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error); - if (!ret_input) - goto out; - } - else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Couldn't find object '%s'", checksum); - goto out; + if (content_pack_checksum != NULL) + { + if (!ostree_repo_map_pack_file (self, content_pack_checksum, FALSE, + &content_pack_data, &content_pack_len, + cancellable, error)) + goto out; + if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len, + content_pack_offset, TRUE, FALSE, + &packed_object, cancellable, error)) + goto out; + ret_input = ostree_read_pack_entry_as_stream (packed_object); + } + else if (content_loose_path != NULL) + { + ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error); + if (!ret_input) + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Couldn't find object '%s'", checksum); + goto out; + } } } } @@ -3142,12 +3248,14 @@ ostree_repo_load_file (OstreeRepo *self, static gboolean list_objects_in_index (OstreeRepo *self, const char *pack_checksum, + gboolean is_meta, GHashTable *inout_objects, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - guint32 objtype_u32; + OstreeRepoPrivate *priv = GET_PRIVATE (self); + guint32 objtype_u8; guint64 offset; ot_lobj GFile *index_path = NULL; ot_lvariant GVariant *index_variant = NULL; @@ -3156,15 +3264,16 @@ list_objects_in_index (OstreeRepo *self, ot_lfree char *checksum = NULL; GVariantIter content_iter; - index_path = ostree_repo_get_pack_index_path (self, pack_checksum); + index_path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum); - if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error)) + if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, + &index_variant, cancellable, error)) goto out; contents = g_variant_get_child_value (index_variant, 2); g_variant_iter_init (&content_iter, contents); - while (g_variant_iter_loop (&content_iter, "(u@ayt)", &objtype_u32, &csum_bytes, &offset)) + while (g_variant_iter_loop (&content_iter, "(y@ayt)", &objtype_u8, &csum_bytes, &offset)) { GVariant *obj_key; GVariant *objdata; @@ -3172,7 +3281,7 @@ list_objects_in_index (OstreeRepo *self, GVariantBuilder pack_contents_builder; gboolean is_loose; - objtype = (OstreeObjectType) GUINT32_FROM_BE (objtype_u32); + objtype = (OstreeObjectType) objtype_u8; offset = GUINT64_FROM_BE (offset); g_variant_builder_init (&pack_contents_builder, @@ -3221,19 +3330,27 @@ list_packed_objects (OstreeRepo *self, { gboolean ret = FALSE; guint i; - ot_lptrarray GPtrArray *index_checksums = NULL; + ot_lptrarray GPtrArray *meta_index_checksums = NULL; + ot_lptrarray GPtrArray *data_index_checksums = NULL; - if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error)) + if (!ostree_repo_list_pack_indexes (self, &meta_index_checksums, &data_index_checksums, + cancellable, error)) goto out; - for (i = 0; i < index_checksums->len; i++) + for (i = 0; i < meta_index_checksums->len; i++) { - const char *checksum = index_checksums->pdata[i]; - - if (!list_objects_in_index (self, checksum, inout_objects, cancellable, error)) + const char *checksum = meta_index_checksums->pdata[i]; + if (!list_objects_in_index (self, checksum, TRUE, inout_objects, cancellable, error)) goto out; } + for (i = 0; i < data_index_checksums->len; i++) + { + const char *checksum = data_index_checksums->pdata[i]; + if (!list_objects_in_index (self, checksum, FALSE, inout_objects, cancellable, error)) + goto out; + } + ret = TRUE; out: return ret; @@ -3251,27 +3368,37 @@ find_object_in_packs (OstreeRepo *self, gboolean ret = FALSE; guint i; guint64 ret_pack_offset; + gboolean is_meta; ot_lptrarray GPtrArray *index_checksums = NULL; ot_lfree char *ret_pack_checksum = NULL; - ot_lobj GFile *index_path = NULL; ot_lvariant GVariant *csum_bytes = NULL; ot_lvariant GVariant *index_variant = NULL; csum_bytes = ostree_checksum_to_bytes_v (checksum); - if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error)) - goto out; + is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); + + if (is_meta) + { + if (!ostree_repo_list_pack_indexes (self, &index_checksums, NULL, + cancellable, error)) + goto out; + } + else + { + if (!ostree_repo_list_pack_indexes (self, NULL, &index_checksums, + cancellable, error)) + goto out; + } for (i = 0; i < index_checksums->len; i++) { const char *pack_checksum = index_checksums->pdata[i]; guint64 offset; - g_clear_object (&index_path); - index_path = ostree_repo_get_pack_index_path (self, pack_checksum); - ot_clear_gvariant (&index_variant); - if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error)) + if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, &index_variant, + cancellable, error)) goto out; if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset)) @@ -3388,12 +3515,12 @@ ostree_repo_load_variant (OstreeRepo *self, } else if (pack_checksum != NULL) { - if (!ostree_repo_map_pack_file (self, pack_checksum, &pack_data, &pack_len, + if (!ostree_repo_map_pack_file (self, pack_checksum, TRUE, &pack_data, &pack_len, cancellable, error)) goto out; if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset, - TRUE, &packed_object, cancellable, error)) + TRUE, TRUE, &packed_object, cancellable, error)) goto out; if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index cfb43aaa..98ecbc62 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -150,19 +150,21 @@ gboolean ostree_repo_load_variant (OstreeRepo *self, GError **error); gboolean ostree_repo_load_pack_index (OstreeRepo *self, - const char *sha256, + const char *pack_checksum, + gboolean is_meta, GVariant **out_variant, GCancellable *cancellable, GError **error); gboolean ostree_repo_load_pack_data (OstreeRepo *self, - const char *sha256, + const char *pack_checksum, guchar **out_data, GCancellable *cancellable, GError **error); gboolean ostree_repo_map_pack_file (OstreeRepo *self, const char *sha256, + gboolean is_meta, guchar **out_data, guint64 *out_len, GCancellable *cancellable, @@ -241,6 +243,7 @@ gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self, gboolean ostree_repo_add_pack_file (OstreeRepo *self, const char *checksum, + gboolean is_meta, GFile *pack_index_path, GFile *pack_data_path, GCancellable *cancellable, @@ -249,8 +252,10 @@ gboolean ostree_repo_add_pack_file (OstreeRepo *self, gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self, const char *remote_name, GFile *superindex_path, - GPtrArray **out_cached_indexes, - GPtrArray **out_uncached_indexes, + GPtrArray **out_cached_meta_indexes, + GPtrArray **out_cached_data_indexes, + GPtrArray **out_uncached_meta_indexes, + GPtrArray **out_uncached_data_indexes, GCancellable *cancellable, GError **error); @@ -262,6 +267,7 @@ gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self, gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GVariant **out_variant, GCancellable *cancellable, GError **error); @@ -269,6 +275,7 @@ gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self, gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GFile *cached_path, GCancellable *cancellable, GError **error); @@ -276,6 +283,7 @@ gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self, gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GFile **out_cached_path, GCancellable *cancellable, GError **error); @@ -283,6 +291,7 @@ gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self, gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self, const char *remote_name, const char *pack_checksum, + gboolean is_meta, GFile *cached_path, GCancellable *cancellable, GError **error); @@ -334,16 +343,11 @@ gboolean ostree_repo_list_objects (OstreeRepo *self, GError **error); gboolean ostree_repo_list_pack_indexes (OstreeRepo *self, - GPtrArray **out_indexes, + GPtrArray **out_meta_indexes, + GPtrArray **out_data_indexes, GCancellable *cancellable, GError **error); -GFile * ostree_repo_get_pack_index_path (OstreeRepo *self, - const char *checksum); - -GFile * ostree_repo_get_pack_data_path (OstreeRepo *self, - const char *checksum); - G_END_DECLS #endif /* _OSTREE_REPO */ diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index dbf51b14..afc44937 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -62,7 +62,8 @@ typedef struct { SoupURI *base_uri; gboolean fetched_packs; - GPtrArray *cached_pack_indexes; + GPtrArray *cached_meta_pack_indexes; + GPtrArray *cached_data_pack_indexes; GHashTable *file_checksums_to_fetch; @@ -266,6 +267,7 @@ fetch_uri_contents_utf8 (OtPullData *pull_data, static gboolean fetch_one_pack_file (OtPullData *pull_data, const char *pack_checksum, + gboolean is_meta, GFile **out_cached_path, GCancellable *cancellable, GError **error) @@ -277,26 +279,26 @@ fetch_one_pack_file (OtPullData *pull_data, SoupURI *pack_uri = NULL; if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, &ret_cached_path, + pack_checksum, is_meta, &ret_cached_path, cancellable, error)) goto out; if (ret_cached_path == NULL) { - pack_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL); + pack_name = ostree_get_pack_data_name (is_meta, pack_checksum); pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL); if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error)) goto out; if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, tmp_path, + pack_checksum, is_meta, tmp_path, cancellable, error)) goto out; } if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name, - pack_checksum, &ret_cached_path, + pack_checksum, is_meta, &ret_cached_path, cancellable, error)) goto out; @@ -322,19 +324,25 @@ find_object_in_remote_packs (OtPullData *pull_data, gboolean ret = FALSE; guint64 offset; guint i; + GPtrArray *iter; ot_lvariant GVariant *mapped_pack = NULL; ot_lvariant GVariant *csum_bytes = NULL; ot_lfree char *ret_pack_checksum = NULL; csum_bytes = ostree_checksum_to_bytes_v (checksum); - for (i = 0; i < pull_data->cached_pack_indexes->len; i++) + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + iter = pull_data->cached_meta_pack_indexes; + else + iter = pull_data->cached_data_pack_indexes; + for (i = 0; i < iter->len; i++) { - const char *pack_checksum = pull_data->cached_pack_indexes->pdata[i]; + const char *pack_checksum = iter->pdata[i]; ot_clear_gvariant (&mapped_pack); if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, &mapped_pack, + pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype), + &mapped_pack, cancellable, error)) goto out; @@ -354,17 +362,18 @@ find_object_in_remote_packs (OtPullData *pull_data, } static gboolean -fetch_one_cache_index (OtPullData *pull_data, - const char *pack_checksum, - GCancellable *cancellable, - GError **error) +fetch_one_cache_index (OtPullData *pull_data, + const char *pack_checksum, + gboolean is_meta, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; ot_lobj GFile *tmp_path = NULL; ot_lfree char *pack_index_name = NULL; SoupURI *index_uri = NULL; - pack_index_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL); + pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum); index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL); if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path, @@ -372,7 +381,7 @@ fetch_one_cache_index (OtPullData *pull_data, goto out; if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name, - pack_checksum, tmp_path, + pack_checksum, is_meta, tmp_path, cancellable, error)) goto out; @@ -398,8 +407,10 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data, gboolean ret = FALSE; guint i; ot_lobj GFile *superindex_tmppath = NULL; - ot_lptrarray GPtrArray *cached_indexes = NULL; - ot_lptrarray GPtrArray *uncached_indexes = NULL; + ot_lptrarray GPtrArray *cached_meta_indexes = NULL; + ot_lptrarray GPtrArray *cached_data_indexes = NULL; + ot_lptrarray GPtrArray *uncached_meta_indexes = NULL; + ot_lptrarray GPtrArray *uncached_data_indexes = NULL; ot_lvariant GVariant *superindex_variant = NULL; GVariantIter *contents_iter = NULL; SoupURI *superindex_uri = NULL; @@ -412,22 +423,33 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data, if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name, superindex_tmppath, - &cached_indexes, &uncached_indexes, + &cached_meta_indexes, + &cached_data_indexes, + &uncached_meta_indexes, + &uncached_data_indexes, cancellable, error)) goto out; - for (i = 0; i < cached_indexes->len; i++) - g_ptr_array_add (pull_data->cached_pack_indexes, - g_strdup (cached_indexes->pdata[i])); + for (i = 0; i < cached_meta_indexes->len; i++) + g_ptr_array_add (pull_data->cached_meta_pack_indexes, + g_strdup (cached_meta_indexes->pdata[i])); + for (i = 0; i < cached_data_indexes->len; i++) + g_ptr_array_add (pull_data->cached_data_pack_indexes, + g_strdup (cached_data_indexes->pdata[i])); - for (i = 0; i < uncached_indexes->len; i++) + for (i = 0; i < uncached_meta_indexes->len; i++) { - const char *pack_checksum = uncached_indexes->pdata[i]; - - if (!fetch_one_cache_index (pull_data, pack_checksum, cancellable, error)) + const char *pack_checksum = uncached_meta_indexes->pdata[i]; + if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error)) goto out; - - g_ptr_array_add (pull_data->cached_pack_indexes, g_strdup (pack_checksum)); + g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum)); + } + for (i = 0; i < uncached_data_indexes->len; i++) + { + const char *pack_checksum = uncached_data_indexes->pdata[i]; + if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error)) + goto out; + g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum)); } ret = TRUE; @@ -485,6 +507,7 @@ fetch_object_if_not_stored (OtPullData *pull_data, gboolean ret = FALSE; guint64 pack_offset = 0; gboolean is_stored; + gboolean is_meta; ot_lobj GInputStream *ret_input = NULL; ot_lobj GFile *temp_path = NULL; ot_lobj GFile *stored_path = NULL; @@ -494,6 +517,8 @@ fetch_object_if_not_stored (OtPullData *pull_data, ot_lvariant GVariant *pack_entry = NULL; GMappedFile *pack_map = NULL; + is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); + if (!ostree_repo_find_object (pull_data->repo, objtype, checksum, &stored_path, &local_pack_checksum, NULL, cancellable, error)) @@ -505,7 +530,8 @@ fetch_object_if_not_stored (OtPullData *pull_data, if (!pull_data->fetched_packs) { pull_data->fetched_packs = TRUE; - pull_data->cached_pack_indexes = g_ptr_array_new_with_free_func (g_free); + pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free); + pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free); if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error)) goto out; @@ -521,8 +547,8 @@ fetch_object_if_not_stored (OtPullData *pull_data, { g_assert (!is_stored); - if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path, - cancellable, error)) + if (!fetch_one_pack_file (pull_data, remote_pack_checksum, is_meta, + &pack_path, cancellable, error)) goto out; pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error); @@ -531,7 +557,7 @@ fetch_object_if_not_stored (OtPullData *pull_data, if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map), g_mapped_file_get_length (pack_map), - pack_offset, FALSE, &pack_entry, + pack_offset, FALSE, is_meta, &pack_entry, cancellable, error)) goto out; @@ -1147,8 +1173,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) g_clear_object (&pull_data->session); if (pull_data->base_uri) soup_uri_free (pull_data->base_uri); - if (pull_data->cached_pack_indexes) - g_ptr_array_unref (pull_data->cached_pack_indexes); + ot_clear_ptrarray (&pull_data->cached_meta_pack_indexes); + ot_clear_ptrarray (&pull_data->cached_data_pack_indexes); if (summary_uri) soup_uri_free (summary_uri); return ret; diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index af0b2f95..44b7e279 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -43,88 +43,76 @@ typedef struct { } OtFsckData; static gboolean -fsck_pack_files (OtFsckData *data, - GCancellable *cancellable, - GError **error) +fsck_one_pack_file (OtFsckData *data, + const char *pack_checksum, + gboolean is_meta, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; - guint i; - guint32 objtype; + guchar objtype_u8; guint64 offset; guint64 pack_size; - ot_lptrarray GPtrArray *pack_indexes = NULL; + ot_lfree char *path = NULL; + ot_lobj GFileInfo *pack_info = NULL; + ot_lobj GInputStream *input = NULL; ot_lvariant GVariant *index_variant = NULL; ot_lobj GFile *pack_index_path = NULL; ot_lobj GFile *pack_data_path = NULL; - ot_lobj GFileInfo *pack_info = NULL; - ot_lobj GInputStream *input = NULL; GChecksum *pack_content_checksum = NULL; GVariantIter *index_content_iter = NULL; - if (!ostree_repo_list_pack_indexes (data->repo, &pack_indexes, cancellable, error)) + g_free (path); + path = ostree_get_relative_pack_index_path (is_meta, pack_checksum); + pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path); + + if (!ot_util_variant_map (pack_index_path, + OSTREE_PACK_INDEX_VARIANT_FORMAT, + &index_variant, error)) + goto out; + + if (!ostree_validate_structureof_pack_index (index_variant, error)) goto out; - for (i = 0; i < pack_indexes->len; i++) - { - const char *checksum = pack_indexes->pdata[i]; - - g_clear_object (&pack_index_path); - pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum); - - ot_clear_gvariant (&index_variant); - if (!ot_util_variant_map (pack_index_path, - OSTREE_PACK_INDEX_VARIANT_FORMAT, - &index_variant, error)) - goto out; + g_free (path); + path = ostree_get_relative_pack_data_path (is_meta, pack_checksum); + pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path); - if (!ostree_validate_structureof_pack_index (index_variant, error)) - goto out; + input = (GInputStream*)g_file_read (pack_data_path, cancellable, error); + if (!input) + goto out; - g_clear_object (&pack_data_path); - pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum); - - g_clear_object (&input); - input = (GInputStream*)g_file_read (pack_data_path, cancellable, error); - if (!input) - goto out; - - g_clear_object (&pack_info); - pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO, - cancellable, error); - if (!pack_info) - goto out; - pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size"); + pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO, + cancellable, error); + if (!pack_info) + goto out; + pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size"); - if (pack_content_checksum) - g_checksum_free (pack_content_checksum); - if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error)) - goto out; + if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error)) + goto out; - if (strcmp (g_checksum_get_string (pack_content_checksum), checksum) != 0) + if (strcmp (g_checksum_get_string (pack_content_checksum), pack_checksum) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "corrupted pack '%s', expected checksum %s", + pack_checksum, g_checksum_get_string (pack_content_checksum)); + goto out; + } + + g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter); + + while (g_variant_iter_loop (index_content_iter, "(y@ayt)", + &objtype_u8, NULL, &offset)) + { + offset = GUINT64_FROM_BE (offset); + if (offset > pack_size) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "corrupted pack '%s', expected checksum %s", - checksum, g_checksum_get_string (pack_content_checksum)); + "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT, + pack_checksum, + offset, pack_size); goto out; } - - g_variant_get_child (index_variant, 2, "a(uayt)", &index_content_iter); - - while (g_variant_iter_loop (index_content_iter, "(u@ayt)", - &objtype, NULL, &offset)) - { - offset = GUINT64_FROM_BE (offset); - if (offset > pack_size) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT, - checksum, - offset, pack_size); - goto out; - } - } - - data->n_pack_files++; } ret = TRUE; @@ -135,6 +123,44 @@ fsck_pack_files (OtFsckData *data, return ret; } +static gboolean +fsck_pack_files (OtFsckData *data, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint i; + ot_lptrarray GPtrArray *meta_pack_indexes = NULL; + ot_lptrarray GPtrArray *data_pack_indexes = NULL; + + if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes, + cancellable, error)) + goto out; + + for (i = 0; i < meta_pack_indexes->len; i++) + { + const char *pack_checksum = meta_pack_indexes->pdata[i]; + + if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error)) + goto out; + + data->n_pack_files++; + } + for (i = 0; i < data_pack_indexes->len; i++) + { + const char *pack_checksum = data_pack_indexes->pdata[i]; + + if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error)) + goto out; + + data->n_pack_files++; + } + + ret = TRUE; + out: + return ret; +} + static gboolean fsck_reachable_objects_from_commits (OtFsckData *data, GHashTable *commits, @@ -198,17 +224,26 @@ fsck_reachable_objects_from_commits (OtFsckData *data, if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!ostree_validate_structureof_commit (metadata, error)) - goto out; + { + g_prefix_error (error, "While validating commit metadata '%s': ", checksum); + goto out; + } } else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE) { if (!ostree_validate_structureof_dirtree (metadata, error)) - goto out; + { + g_prefix_error (error, "While validating directory tree '%s': ", checksum); + goto out; + } } else if (objtype == OSTREE_OBJECT_TYPE_DIR_META) { if (!ostree_validate_structureof_dirmeta (metadata, error)) - goto out; + { + g_prefix_error (error, "While validating directory metadata '%s': ", checksum); + goto out; + } } else g_assert_not_reached (); @@ -236,7 +271,10 @@ fsck_reachable_objects_from_commits (OtFsckData *data, mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); if (!ostree_validate_structureof_file_mode (mode, error)) - goto out; + { + g_prefix_error (error, "While validating file '%s': ", checksum); + goto out; + } } else { diff --git a/src/ostree/ot-builtin-pack.c b/src/ostree/ot-builtin-pack.c index 1c402b82..3d24ab4a 100644 --- a/src/ostree/ot-builtin-pack.c +++ b/src/ostree/ot-builtin-pack.c @@ -197,18 +197,16 @@ compare_index_content (gconstpointer ap, gpointer b = *((gpointer*)bp); GVariant *a_v = a; GVariant *b_v = b; - guint32 a_objtype; - guint32 b_objtype; + guchar a_objtype; + guchar b_objtype; guint64 a_offset; guint64 b_offset; int c; ot_lvariant GVariant *a_csum_bytes = NULL; ot_lvariant GVariant *b_csum_bytes = NULL; - g_variant_get (a_v, "(u@ayt)", &a_objtype, &a_csum_bytes, &a_offset); - g_variant_get (b_v, "(u@ayt)", &b_objtype, &b_csum_bytes, &b_offset); - a_objtype = GUINT32_FROM_BE (a_objtype); - b_objtype = GUINT32_FROM_BE (b_objtype); + g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset); + g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset); c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes), ostree_checksum_bytes_peek (b_csum_bytes)); if (c == 0) @@ -270,8 +268,121 @@ delete_loose_object (OtRepackData *data, return ret; } +static gboolean +pack_one_meta_object (OtRepackData *data, + const char *checksum, + OstreeObjectType objtype, + GVariant **out_packed_object, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + ot_lobj GFile *object_path = NULL; + ot_lvariant GVariant *metadata_v = NULL; + ot_lvariant GVariant *ret_packed_object = NULL; + + object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); + + if (!ostree_map_metadata_file (object_path, objtype, &metadata_v, error)) + goto out; + + ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype, + ostree_checksum_to_bytes_v (checksum), + metadata_v); + + ret = TRUE; + ot_transfer_out_value (out_packed_object, &ret_packed_object); + out: + return ret; +} + +static gboolean +pack_one_data_object (OtRepackData *data, + const char *checksum, + OstreeObjectType objtype, + guint64 expected_objsize, + GVariant **out_packed_object, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint64 objsize; + guchar entry_flags = 0; + GInputStream *read_object_in; /* nofree */ + ot_lobj GFile *object_path = NULL; + ot_lobj GFileInputStream *object_input = NULL; + ot_lobj GFileInfo *object_file_info = NULL; + ot_lobj GMemoryOutputStream *object_data_stream = NULL; + ot_lobj GConverter *compressor = NULL; + ot_lobj GConverterInputStream *compressed_object_input = NULL; + ot_lvariant GVariant *ret_packed_object = NULL; + + switch (data->int_compression) + { + case OT_COMPRESSION_GZIP: + { + entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP; + break; + } + default: + { + g_assert_not_reached (); + } + } + + object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); + + object_input = g_file_read (object_path, cancellable, error); + if (!object_input) + goto out; + + object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error); + if (!object_file_info) + goto out; + + objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); + + g_assert_cmpint (objsize, ==, expected_objsize); + + object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); + + if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) + { + compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL); + compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, + "converter", compressor, + "base-stream", object_input, + "close-base-stream", TRUE, + NULL); + read_object_in = (GInputStream*)compressed_object_input; + } + else + { + read_object_in = (GInputStream*)object_input; + } + + if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + cancellable, error)) + goto out; + + { + guchar *data = g_memory_output_stream_get_data (object_data_stream); + gsize data_len = g_memory_output_stream_get_data_size (object_data_stream); + ret_packed_object = g_variant_new ("(yy@ay@ay)", (guchar)objtype, entry_flags, + ostree_checksum_to_bytes_v (checksum), + ot_gvariant_new_bytearray (data, data_len)); + } + + ret = TRUE; + ot_transfer_out_value (out_packed_object, &ret_packed_object); + out: + return ret; +} + static gboolean create_pack_file (OtRepackData *data, + gboolean is_meta, GPtrArray *objects, GCancellable *cancellable, GError **error) @@ -285,19 +396,12 @@ create_pack_file (OtRepackData *data, ot_lobj GOutputStream *index_out = NULL; ot_lobj GFile *pack_temppath = NULL; ot_lobj GOutputStream *pack_out = NULL; - ot_lobj GFile *object_path = NULL; - ot_lobj GFileInfo *object_file_info = NULL; - ot_lobj GFileInputStream *object_input = NULL; - ot_lobj GConverter *compressor = NULL; - ot_lobj GConverterInputStream *compressed_object_input = NULL; ot_lptrarray GPtrArray *index_content_list = NULL; ot_lvariant GVariant *pack_header = NULL; - ot_lvariant GVariant *packed_object = NULL; ot_lvariant GVariant *index_content = NULL; ot_lfree char *pack_name = NULL; ot_lobj GFile *pack_file_path = NULL; ot_lobj GFile *pack_index_path = NULL; - GMemoryOutputStream *object_data_stream = NULL; GVariantBuilder index_content_builder; GChecksum *pack_checksum = NULL; @@ -324,7 +428,7 @@ create_pack_file (OtRepackData *data, pack_checksum = g_checksum_new (G_CHECKSUM_SHA256); pack_header = g_variant_new ("(s@a{sv}t)", - "OSTv0PACKFILE", + is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE", g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), (guint64)objects->len); @@ -339,92 +443,37 @@ create_pack_file (OtRepackData *data, guint32 objtype_u32; OstreeObjectType objtype; guint64 expected_objsize; - guint64 objsize; - GInputStream *read_object_in; - guchar entry_flags = 0; - GVariant *index_entry; + ot_lvariant GVariant *packed_object = NULL; + ot_lvariant GVariant *index_entry = NULL; g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize); objtype = (OstreeObjectType) objtype_u32; - switch (data->int_compression) + if (is_meta) { - case OT_COMPRESSION_GZIP: - { - entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP; - break; - } - default: - { - g_assert_not_reached (); - } - } - - g_clear_object (&object_path); - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - g_clear_object (&object_input); - object_input = g_file_read (object_path, cancellable, error); - if (!object_input) - goto out; - - g_clear_object (&object_file_info); - object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error); - if (!object_file_info) - goto out; - - objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); - - g_assert_cmpint (objsize, ==, expected_objsize); - - g_clear_object (&object_data_stream); - object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - - if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP) - { - g_clear_object (&compressor); - compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL); - - g_clear_object (&compressed_object_input); - compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM, - "converter", compressor, - "base-stream", object_input, - "close-base-stream", TRUE, - NULL); - read_object_in = (GInputStream*)compressed_object_input; + if (!pack_one_meta_object (data, checksum, objtype, &packed_object, + cancellable, error)) + goto out; } else { - read_object_in = (GInputStream*)object_input; + if (!pack_one_data_object (data, checksum, objtype, expected_objsize, + &packed_object, cancellable, error)) + goto out; } - if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in, - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error)) - goto out; - - ot_clear_gvariant (&packed_object); - { - guchar *data = g_memory_output_stream_get_data (object_data_stream); - gsize data_len = g_memory_output_stream_get_data_size (object_data_stream); - packed_object = g_variant_new ("(uy@ay@ay)", GUINT32_TO_BE ((guint32)objtype), - entry_flags, - ostree_checksum_to_bytes_v (checksum), - ot_gvariant_new_bytearray (data, data_len)); - g_clear_object (&object_data_stream); - } - if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error)) goto out; /* offset points to aligned header size */ - index_entry = g_variant_new ("(u@ayt)", - GUINT32_TO_BE ((guint32)objtype), + index_entry = g_variant_new ("(y@ayt)", + (guchar)objtype, ostree_checksum_to_bytes_v (checksum), GUINT64_TO_BE (offset)); g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry)); - + index_entry = NULL; + if (!write_variant_with_size (pack_out, packed_object, pack_checksum, &offset, cancellable, error)) goto out; @@ -433,14 +482,14 @@ create_pack_file (OtRepackData *data, if (!g_output_stream_close (pack_out, cancellable, error)) goto out; - g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(uayt)")); + g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)")); g_ptr_array_sort (index_content_list, compare_index_content); for (i = 0; i < index_content_list->len; i++) { GVariant *index_item = index_content_list->pdata[i]; g_variant_builder_add_value (&index_content_builder, index_item); } - index_content = g_variant_new ("(s@a{sv}@a(uayt))", + index_content = g_variant_new ("(s@a{sv}@a(yayt))", "OSTv0PACKINDEX", g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0), g_variant_builder_end (&index_content_builder)); @@ -458,6 +507,7 @@ create_pack_file (OtRepackData *data, if (!ostree_repo_add_pack_file (data->repo, g_checksum_get_string (pack_checksum), + is_meta, index_temppath, pack_temppath, cancellable, @@ -499,67 +549,14 @@ create_pack_file (OtRepackData *data, return ret; } -/** - * cluster_objects_stupidly: - * @objects: Map from serialized object name to objdata - * @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). - * - * Just sorts by size currently. Also filters out non-regular object - * content. - */ -static gboolean -cluster_objects_stupidly (OtRepackData *data, - GHashTable *objects, - GPtrArray **out_clusters, - GCancellable *cancellable, - GError **error) +static void +cluster_one_object_chain (OtRepackData *data, + GPtrArray *object_list, + GPtrArray *inout_clusters) { - gboolean ret = FALSE; guint i; guint64 current_size; guint current_offset; - GHashTableIter hash_iter; - gpointer key, value; - ot_lptrarray GPtrArray *ret_clusters = NULL; - ot_lptrarray GPtrArray *object_list = NULL; - ot_lobj GFile *object_path = NULL; - ot_lobj GFileInfo *object_info = NULL; - - object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - - g_hash_table_iter_init (&hash_iter, objects); - - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - GVariant *serialized_key = key; - const char *checksum; - OstreeObjectType objtype; - guint64 size; - - ostree_object_name_deserialize (serialized_key, &checksum, &objtype); - - g_clear_object (&object_path); - object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); - - g_clear_object (&object_info); - object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); - if (!object_info) - goto out; - - if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR) - continue; - - size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); - - g_ptr_array_add (object_list, - g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size))); - } - - g_ptr_array_sort (object_list, compare_object_data_by_size); - - ret_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); current_size = 0; current_offset = 0; @@ -582,7 +579,7 @@ cluster_objects_stupidly (OtRepackData *data, { g_ptr_array_add (current, g_variant_ref (object_list->pdata[j])); } - g_ptr_array_add (ret_clusters, current); + g_ptr_array_add (inout_clusters, current); current_size = objsize; current_offset = i+1; } @@ -596,9 +593,83 @@ cluster_objects_stupidly (OtRepackData *data, current_size += objsize; } } +} + +/** + * cluster_objects_stupidly: + * @objects: Map from serialized object name to objdata + * @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). + * @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref(). + * + * Just sorts by size currently. Also filters out non-regular object + * content. + */ +static gboolean +cluster_objects_stupidly (OtRepackData *data, + GHashTable *objects, + GPtrArray **out_meta_clusters, + GPtrArray **out_data_clusters, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + GHashTableIter hash_iter; + gpointer key, value; + ot_lptrarray GPtrArray *ret_meta_clusters = NULL; + ot_lptrarray GPtrArray *ret_data_clusters = NULL; + ot_lptrarray GPtrArray *meta_object_list = NULL; + ot_lptrarray GPtrArray *data_object_list = NULL; + ot_lobj GFile *object_path = NULL; + ot_lobj GFileInfo *object_info = NULL; + + meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + + g_hash_table_iter_init (&hash_iter, objects); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + GVariant *serialized_key = key; + const char *checksum; + OstreeObjectType objtype; + guint64 size; + GVariant *v; + + ostree_object_name_deserialize (serialized_key, &checksum, &objtype); + + g_clear_object (&object_path); + object_path = ostree_repo_get_object_path (data->repo, checksum, objtype); + + g_clear_object (&object_info); + object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!object_info) + goto out; + + if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR) + continue; + + size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE); + + v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)); + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + g_ptr_array_add (meta_object_list, v); + else + g_ptr_array_add (data_object_list, v); + } + + g_ptr_array_sort (meta_object_list, compare_object_data_by_size); + g_ptr_array_sort (data_object_list, compare_object_data_by_size); + + ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); + ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref); + + cluster_one_object_chain (data, meta_object_list, ret_meta_clusters); + cluster_one_object_chain (data, data_object_list, ret_data_clusters); ret = TRUE; - ot_transfer_out_value (out_clusters, &ret_clusters); + ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters); + ot_transfer_out_value (out_data_clusters, &ret_data_clusters); out: return ret; } @@ -792,7 +863,8 @@ do_incremental_pack (OtRepackData *data, gboolean ret = FALSE; guint i; ot_lhash GHashTable *objects = NULL; - ot_lptrarray GPtrArray *clusters = NULL; + ot_lptrarray GPtrArray *meta_clusters = NULL; + ot_lptrarray GPtrArray *data_clusters = NULL; ot_lhash GHashTable *loose_objects = NULL; if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, @@ -805,21 +877,30 @@ do_incremental_pack (OtRepackData *data, g_print ("\n"); g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size); - if (!cluster_objects_stupidly (data, loose_objects, &clusters, cancellable, error)) + if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters, + cancellable, error)) goto out; - if (clusters->len > 0) - g_print ("Going to create %u packfiles\n", clusters->len); + if (meta_clusters->len > 0 || data_clusters->len > 0) + g_print ("Going to create %u meta packfiles, %u data packfiles\n", + meta_clusters->len, data_clusters->len); else g_print ("Nothing to do\n"); - - for (i = 0; i < clusters->len; i++) + + if (!opt_analyze_only) { - GPtrArray *cluster = clusters->pdata[i]; - - if (!opt_analyze_only) + for (i = 0; i < meta_clusters->len; i++) { - if (!create_pack_file (data, cluster, cancellable, error)) + GPtrArray *cluster = meta_clusters->pdata[i]; + + if (!create_pack_file (data, TRUE, cluster, cancellable, error)) + goto out; + } + for (i = 0; i < data_clusters->len; i++) + { + GPtrArray *cluster = data_clusters->pdata[i]; + + if (!create_pack_file (data, FALSE, cluster, cancellable, error)) goto out; } } diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c index e72bf458..4fa14c4b 100644 --- a/src/ostree/ot-builtin-unpack.c +++ b/src/ostree/ot-builtin-unpack.c @@ -144,15 +144,20 @@ unpack_one_object (OstreeRepo *repo, static gboolean delete_one_packfile (OstreeRepo *repo, const char *pack_checksum, + gboolean is_meta, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; + ot_lfree char *data_name = NULL; ot_lobj GFile *data_path = NULL; + ot_lfree char *index_name = NULL; ot_lobj GFile *index_path = NULL; - index_path = ostree_repo_get_pack_index_path (repo, pack_checksum); - data_path = ostree_repo_get_pack_data_path (repo, pack_checksum); + index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum); + index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name); + data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum); + data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name); if (!ot_gfile_unlink (index_path, cancellable, error)) { @@ -185,7 +190,8 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) ot_lhash GHashTable *objects = NULL; ot_lptrarray GPtrArray *clusters = NULL; ot_lhash GHashTable *packed_objects = NULL; - ot_lhash GHashTable *packfiles_to_delete = NULL; + ot_lhash GHashTable *meta_packfiles_to_delete = NULL; + ot_lhash GHashTable *data_packfiles_to_delete = NULL; ot_lobj GFile *objpath = NULL; memset (&data, 0, sizeof (data)); @@ -220,7 +226,8 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) in_transaction = TRUE; - packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); g_hash_table_iter_init (&hash_iter, packed_objects); while (g_hash_table_iter_next (&hash_iter, &key, &value)) @@ -232,6 +239,9 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) OstreeObjectType objtype; gboolean is_loose; GVariantIter *pack_array_iter; + GHashTable *target_hash; + + ostree_object_name_deserialize (objkey, &checksum, &objtype); objdata = g_hash_table_lookup (objects, objkey); g_assert (objdata); @@ -240,12 +250,17 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) g_assert (!is_loose); + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + target_hash = meta_packfiles_to_delete; + else + target_hash = data_packfiles_to_delete; + while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum)) { - if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum)) + if (!g_hash_table_lookup (target_hash, pack_checksum)) { gchar *duped_checksum = g_strdup (pack_checksum); - g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum); + g_hash_table_replace (target_hash, duped_checksum, duped_checksum); } } g_variant_iter_free (pack_array_iter); @@ -261,15 +276,27 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error) if (!ostree_repo_commit_transaction (repo, cancellable, error)) goto out; - if (g_hash_table_size (packfiles_to_delete) == 0) + if (g_hash_table_size (meta_packfiles_to_delete) == 0 + && g_hash_table_size (data_packfiles_to_delete) == 0) g_print ("No pack files; nothing to do\n"); - g_hash_table_iter_init (&hash_iter, packfiles_to_delete); + g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *pack_checksum = key; - if (!delete_one_packfile (repo, pack_checksum, cancellable, error)) + if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error)) + goto out; + + g_print ("Deleted packfile '%s'\n", pack_checksum); + } + + g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *pack_checksum = key; + + if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error)) goto out; g_print ("Deleted packfile '%s'\n", pack_checksum);