shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-19 22:50:35 +03:00
Code Issues Projects Releases Wiki Activity

core: Split pack files into data/metadata

Browse Source 
This will allow us to download metadata first (separately), and in the
future we can have an improved metadata pack format that compresses
heavily.
This commit is contained in:
Colin Walters 2012-04-09 22:46:08 -04:00
parent 11c0fbc738
commit 5a82b141ae
8 changed files with 922 additions and 555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
src/libostree/ostree-core.c
View File

@ -710,6 +710,58 @@ ostree_get_relative_object_path (const char *checksum,
return g_string_free (path, FALSE);
}
static char *
get_pack_name (gboolean is_meta,
gboolean is_index,
const char *prefix,
const char *checksum)
{
GString *path;
g_assert (strlen (checksum) == 64);
path = g_string_new (prefix);
if (is_meta)
g_string_append (path, "ostmetapack-");
else
g_string_append (path, "ostdatapack-");
g_string_append (path, checksum);
if (is_index)
g_string_append (path, ".index");
else
g_string_append (path, ".data");
return g_string_free (path, FALSE);
}
char *
ostree_get_pack_index_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "", checksum);
}
char *
ostree_get_pack_data_name (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "", checksum);
}
char *
ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, TRUE, "objects/pack/", checksum);
}
char *
ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum)
{
return get_pack_name (is_meta, FALSE, "objects/pack/", checksum);
}
GVariant *
ostree_create_archive_file_metadata (GFileInfo *finfo,
GVariant *xattrs)
@ -1169,6 +1221,7 @@ ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 offset,
gboolean trusted,
gboolean is_meta,
GVariant **out_entry,
GCancellable *cancellable,
GError **error)
@ -1215,7 +1268,8 @@ ostree_read_pack_entry_raw (guchar *pack_data,
goto out;
}
ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT,
ret_entry = g_variant_new_from_data (is_meta ? OSTREE_PACK_META_FILE_VARIANT_FORMAT :
OSTREE_PACK_DATA_FILE_VARIANT_FORMAT,
pack_data+entry_start, entry_len,
trusted, NULL, NULL);
g_variant_ref_sink (ret_entry);
@ -1274,32 +1328,13 @@ ostree_read_pack_entry_variant (GVariant *pack_entry,
GError **error)
{
gboolean ret = FALSE;
guint32 actual_type;
ot_lobj GInputStream *stream = NULL;
ot_lvariant GVariant *container_variant = NULL;
ot_lvariant GVariant *ret_variant = NULL;
stream = ostree_read_pack_entry_as_stream (pack_entry);
if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT,
trusted, &container_variant, cancellable, error))
goto out;
g_variant_get (container_variant, "(uv)",
&actual_type, &ret_variant);
actual_type = GUINT32_FROM_BE (actual_type);
if (actual_type != expected_objtype)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted metadata object in pack file; found type %u, expected %u",
actual_type, (guint32)expected_objtype);
goto out;
}
g_variant_get_child (pack_entry, 2, "v", &ret_variant);
ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant);
out:
/* out: */
return ret;
}
@ -1332,16 +1367,15 @@ ostree_pack_index_search (GVariant *index,
while (imax >= imin)
{
GVariant *cur_csum_bytes;
guint32 cur_objtype;
guchar cur_objtype;
guint64 cur_offset;
gsize imid;
int c;
imid = (imin + imax) / 2;
g_variant_get_child (index_contents, imid, "(u@ayt)", &cur_objtype,
g_variant_get_child (index_contents, imid, "(y@ayt)", &cur_objtype,
&cur_csum_bytes, &cur_offset);
cur_objtype = GUINT32_FROM_BE (cur_objtype);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (cur_csum_bytes), csum);
if (c == 0)
@ -1375,12 +1409,12 @@ ostree_pack_index_search (GVariant *index,
}
gboolean
ostree_validate_structureof_objtype (guint32 objtype,
ostree_validate_structureof_objtype (guchar objtype,
GError **error)
{
objtype = GUINT32_FROM_BE (objtype);
if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE
|| objtype > OSTREE_OBJECT_TYPE_COMMIT)
OstreeObjectType objtype_v = (OstreeObjectType) objtype;
if (objtype_v < OSTREE_OBJECT_TYPE_RAW_FILE
|| objtype_v > OSTREE_OBJECT_TYPE_COMMIT)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid object type '%u'", objtype);
@ -1542,7 +1576,8 @@ validate_stat_mode_perms (guint32 mode,
GError **error)
{
gboolean ret = FALSE;
guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO);
guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO &
~S_ISUID & ~S_ISGID & ~S_ISVTX);
if (mode & otherbits)
{
@ -1615,7 +1650,7 @@ ostree_validate_structureof_pack_index (GVariant *index,
{
gboolean ret = FALSE;
const char *header;
guint32 objtype;
guchar objtype_u8;
guint64 offset;
ot_lvariant GVariant *csum_v = NULL;
GVariantIter *content_iter = NULL;
@ -1632,12 +1667,12 @@ ostree_validate_structureof_pack_index (GVariant *index,
goto out;
}
g_variant_get_child (index, 2, "a(uayt)", &content_iter);
g_variant_get_child (index, 2, "a(yayt)", &content_iter);
while (g_variant_iter_loop (content_iter, "(u@ayt)",
&objtype, &csum_v, &offset))
while (g_variant_iter_loop (content_iter, "(y@ayt)",
&objtype_u8, &csum_v, &offset))
{
if (!ostree_validate_structureof_objtype (objtype, error))
if (!ostree_validate_structureof_objtype (objtype_u8, error))
goto out;
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
@ -1674,7 +1709,15 @@ ostree_validate_structureof_pack_superindex (GVariant *superindex,
}
g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
}
csum_v = NULL;
g_variant_get_child (superindex, 3, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_v, &bloom))
{

41
src/libostree/ostree-core.h
View File

@ -93,34 +93,44 @@ typedef enum {
/* Pack super index
* s - OSTv0SUPERPACKINDEX
* a{sv} - Metadata
* a(ayay) - (pack file checksum, bloom filter)
* a(ayay) - metadata packs (pack file checksum, bloom filter)
* a(ayay) - data packs (pack file checksum, bloom filter)
*/
#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))")
#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay)a(ayay))")
/* Pack index
* s - OSTv0PACKINDEX
* a{sv} - Metadata
* a(uayt) - (objtype, checksum, offset into packfile)
* a(yayt) - (objtype, checksum, offset into packfile)
*/
#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))")
#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(yayt))")
typedef enum {
OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
} OstreePackFileEntryFlag;
/* Pack files
* s - OSTv0PACKFILE
/* Data Pack files
* s - OSTv0PACKDATAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( uyayay ) - objtype, flags, checksum, data
* ( yyayay ) - objtype, flags, checksum, data
*/
#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)")
#define OSTREE_PACK_DATA_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yyayay)")
#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)")
/* Meta Pack files
* s - OSTv0PACKMETAFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( yayv ) - objtype, checksum, data
*/
#define OSTREE_PACK_META_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(yayv)")
gboolean ostree_validate_checksum_string (const char *sha256,
GError **error);
@ -162,6 +172,16 @@ void ostree_object_from_string (const char *str,
char *ostree_get_relative_object_path (const char *checksum,
OstreeObjectType type);
char *ostree_get_pack_index_name (gboolean is_meta,
const char *checksum);
char *ostree_get_pack_data_name (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_index_path (gboolean is_meta,
const char *checksum);
char *ostree_get_relative_pack_data_path (gboolean is_meta,
const char *checksum);
gboolean ostree_get_xattrs_for_file (GFile *f,
GVariant **out_xattrs,
GCancellable *cancellable,
@ -260,6 +280,7 @@ gboolean ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 object_offset,
gboolean trusted,
gboolean is_meta,
GVariant **out_entry,
GCancellable *cancellable,
GError **error);
@ -280,7 +301,7 @@ gboolean ostree_pack_index_search (GVariant *index,
/** VALIDATION **/
gboolean ostree_validate_structureof_objtype (guint32 objtype,
gboolean ostree_validate_structureof_objtype (guchar objtype,
GError **error);
gboolean ostree_validate_structureof_csum_v (GVariant *checksum,

605
src/libostree/ostree-repo.c
View File

@ -1507,6 +1507,7 @@ get_checksum_from_pack_name (const char *name)
static gboolean
list_pack_indexes_from_dir (OstreeRepo *self,
gboolean is_meta,
GPtrArray **out_indexes,
GCancellable *cancellable,
GError **error)
@ -1518,7 +1519,7 @@ list_pack_indexes_from_dir (OstreeRepo *self,
ot_lptrarray GPtrArray *ret_indexes = NULL;
if (!list_files_in_dir_matching (priv->pack_dir,
"ostpack-", ".index",
is_meta ? "ostmetapack-" : "ostdatapack-", ".index",
&index_files,
cancellable, error))
goto out;
@ -1539,24 +1540,27 @@ list_pack_indexes_from_dir (OstreeRepo *self,
static gboolean
list_pack_checksums_from_superindex_file (GFile *superindex_path,
GPtrArray **out_indexes,
GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
const char *magic;
ot_lptrarray GPtrArray *ret_indexes = NULL;
ot_lptrarray GPtrArray *ret_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_data_indexes = NULL;
ot_lvariant GVariant *superindex_variant = NULL;
ot_lvariant GVariant *checksum = NULL;
ot_lvariant GVariant *bloom = NULL;
GVariantIter *variant_iter = NULL;
GVariantIter *meta_variant_iter = NULL;
GVariantIter *data_variant_iter = NULL;
if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
&superindex_variant, error))
goto out;
g_variant_get (superindex_variant, "(&s@a{sv}a(ayay))",
&magic, NULL, &variant_iter);
g_variant_get (superindex_variant, "(&s@a{sv}a(ayay)a(ayay))",
&magic, NULL, &meta_variant_iter, &data_variant_iter);
if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0)
{
@ -1565,47 +1569,62 @@ list_pack_checksums_from_superindex_file (GFile *superindex_path,
goto out;
}
ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
while (g_variant_iter_loop (variant_iter, "(@ay@ay)",
ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
while (g_variant_iter_loop (meta_variant_iter, "(@ay@ay)",
&checksum, &bloom))
g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes_v (checksum));
g_ptr_array_add (ret_meta_indexes, ostree_checksum_from_bytes_v (checksum));
checksum = NULL;
bloom = NULL;
ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
while (g_variant_iter_loop (data_variant_iter, "(@ay@ay)",
&checksum, &bloom))
g_ptr_array_add (ret_data_indexes, ostree_checksum_from_bytes_v (checksum));
checksum = NULL;
bloom = NULL;
ret = TRUE;
ot_transfer_out_value (out_indexes, &ret_indexes);
ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes);
ot_transfer_out_value (out_data_indexes, &ret_data_indexes);
out:
if (variant_iter)
g_variant_iter_free (variant_iter);
if (meta_variant_iter)
g_variant_iter_free (meta_variant_iter);
if (data_variant_iter)
g_variant_iter_free (data_variant_iter);
return ret;
}
gboolean
ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_indexes,
GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self);
ot_lobj GFile *superindex_path = NULL;
ot_lptrarray GPtrArray *ret_indexes = NULL;
ot_lptrarray GPtrArray *ret_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_data_indexes = NULL;
superindex_path = g_file_get_child (priv->pack_dir, "index");
if (g_file_query_exists (superindex_path, cancellable))
{
if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_indexes, cancellable, error))
if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_meta_indexes,
&ret_data_indexes,
cancellable, error))
goto out;
}
else
{
ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
ret_meta_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
ret_data_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
}
ret = TRUE;
ot_transfer_out_value (out_indexes, &ret_indexes);
ot_transfer_out_value (out_meta_indexes, &ret_meta_indexes);
ot_transfer_out_value (out_data_indexes, &ret_data_indexes);
out:
return ret;
}
@ -1631,6 +1650,35 @@ create_index_bloom (OstreeRepo *self,
return ret;
}
static gboolean
append_index_builder (OstreeRepo *self,
GPtrArray *indexes,
GVariantBuilder *builder,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
for (i = 0; i < indexes->len; i++)
{
const char *pack_checksum = indexes->pdata[i];
ot_lvariant GVariant *bloom = NULL;
if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error))
goto out;
g_variant_builder_add (builder,
"(@ay@ay)",
ostree_checksum_to_bytes_v (pack_checksum),
bloom);
}
ret = TRUE;
out:
return ret;
}
/**
* Regenerate the pack superindex file based on the set of pack
* indexes currently in the filesystem.
@ -1641,69 +1689,80 @@ ostree_repo_regenerate_pack_index (OstreeRepo *self,
GError **error)
{
gboolean ret = FALSE;
guint i;
OstreeRepoPrivate *priv = GET_PRIVATE (self);
ot_lobj GFile *index_path = NULL;
ot_lobj GFile *superindex_path = NULL;
ot_lptrarray GPtrArray *pack_indexes = NULL;
ot_lvariant GVariant *index_variant = NULL;
GVariantBuilder *index_content_builder = NULL;
ot_lvariant GVariant *superindex_variant = NULL;
GVariantBuilder *meta_index_content_builder = NULL;
GVariantBuilder *data_index_content_builder = NULL;
if (!list_pack_indexes_from_dir (self, &pack_indexes, cancellable, error))
superindex_path = g_file_get_child (priv->pack_dir, "index");
ot_clear_ptrarray (&pack_indexes);
if (!list_pack_indexes_from_dir (self, TRUE, &pack_indexes,
cancellable, error))
goto out;
meta_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
if (!append_index_builder (self, pack_indexes, meta_index_content_builder,
cancellable, error))
goto out;
index_path = g_file_get_child (priv->pack_dir, "index");
ot_clear_ptrarray (&pack_indexes);
if (!list_pack_indexes_from_dir (self, FALSE, &pack_indexes,
cancellable, error))
goto out;
data_index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
if (!append_index_builder (self, pack_indexes, data_index_content_builder,
cancellable, error))
goto out;
index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
for (i = 0; i < pack_indexes->len; i++)
{
const char *pack_checksum = pack_indexes->pdata[i];
GVariant *bloom;
superindex_variant = g_variant_new ("(s@a{sv}@a(ayay)@a(ayay))",
"OSTv0SUPERPACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"),
NULL, 0),
g_variant_builder_end (meta_index_content_builder),
g_variant_builder_end (data_index_content_builder));
g_variant_ref_sink (superindex_variant);
if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error))
goto out;
g_variant_builder_add (index_content_builder,
"(@ay@ay)",
ostree_checksum_to_bytes_v (pack_checksum),
bloom);
g_variant_unref (bloom);
}
index_variant = g_variant_new ("(s@a{sv}@a(ayay))",
"OSTv0SUPERPACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"),
NULL, 0),
g_variant_builder_end (index_content_builder));
g_variant_ref_sink (index_variant);
if (!ot_util_variant_save (index_path, index_variant,
if (!ot_util_variant_save (superindex_path, superindex_variant,
cancellable, error))
goto out;
ret = TRUE;
out:
if (index_content_builder)
g_variant_builder_unref (index_content_builder);
if (meta_index_content_builder)
g_variant_builder_unref (meta_index_content_builder);
if (data_index_content_builder)
g_variant_builder_unref (data_index_content_builder);
return ret;
}
static GFile *
get_pack_index_name_from_checksum (GFile *parent, const char *pack_checksum)
get_pack_index_path (GFile *parent,
gboolean is_meta,
const char *checksum)
{
return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".index", NULL);
char *path = ostree_get_pack_index_name (is_meta, checksum);
GFile *ret = g_file_resolve_relative_path (parent, path);
g_free (path);
return ret;
}
static GFile *
get_pack_data_name_from_checksum (GFile *parent, const char *pack_checksum)
get_pack_data_path (GFile *parent,
gboolean is_meta,
const char *checksum)
{
return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".data", NULL);
char *path = ostree_get_pack_data_name (is_meta, checksum);
GFile *ret = g_file_resolve_relative_path (parent, path);
g_free (path);
return ret;
}
gboolean
ostree_repo_add_pack_file (OstreeRepo *self,
const char *pack_checksum,
gboolean is_meta,
GFile *index_path,
GFile *data_path,
GCancellable *cancellable,
@ -1717,11 +1776,11 @@ ostree_repo_add_pack_file (OstreeRepo *self,
if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error))
goto out;
pack_data_path = get_pack_data_name_from_checksum (priv->pack_dir, pack_checksum);
pack_data_path = get_pack_data_path (priv->pack_dir, is_meta, pack_checksum);
if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error))
goto out;
pack_index_path = get_pack_index_name_from_checksum (priv->pack_dir, pack_checksum);
pack_index_path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum);
if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error))
goto out;
@ -1752,6 +1811,80 @@ ensure_remote_cache_dir (OstreeRepo *self,
return ret;
}
static gboolean
delete_no_longer_referenced (OstreeRepo *self,
GFile *cache_path,
const char *prefix,
const char *suffix,
GHashTable *new_files,
GPtrArray *inout_cached,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lptrarray GPtrArray *current_files = NULL;
ot_lfree char *pack_checksum = NULL;
if (!list_files_in_dir_matching (cache_path,
prefix, suffix,
&current_files,
cancellable, error))
goto out;
for (i = 0; i < current_files->len; i++)
{
GFile *file = current_files->pdata[i];
g_free (pack_checksum);
pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (file));
if (!g_hash_table_lookup (new_files, pack_checksum))
{
if (!ot_gfile_unlink (file, cancellable, error))
goto out;
}
if (inout_cached)
{
g_ptr_array_add (inout_cached, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
}
ret = TRUE;
out:
return ret;
}
static void
gather_uncached (GHashTable *new_files,
GPtrArray *cached,
GPtrArray *inout_uncached)
{
guint i;
GHashTableIter hash_iter;
gpointer key, value;
g_hash_table_iter_init (&hash_iter, new_files);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *cur_pack_checksum = key;
gboolean found = FALSE;
for (i = 0; i < cached->len; i++)
{
const char *checksum = cached->pdata[i];
if (strcmp (cur_pack_checksum, checksum) == 0)
{
found = TRUE;
break;
}
}
if (!found)
g_ptr_array_add (inout_uncached, g_strdup (cur_pack_checksum));
}
}
/**
* Take a pack superindex file @superindex_path, and clean up any
* no-longer-referenced pack files in the lookaside cache for
@ -1766,23 +1899,27 @@ gboolean
ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name,
GFile *superindex_path,
GPtrArray **out_cached_indexes,
GPtrArray **out_uncached_indexes,
GPtrArray **out_cached_meta_indexes,
GPtrArray **out_cached_data_indexes,
GPtrArray **out_uncached_meta_indexes,
GPtrArray **out_uncached_data_indexes,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTableIter hash_iter;
gpointer key, value;
guint i;
ot_lvariant GVariant *superindex_variant = NULL;
ot_lobj GFile *cache_path = NULL;
ot_lobj GFile *superindex_cache_path = NULL;
ot_lptrarray GPtrArray *index_files = NULL;
ot_lptrarray GPtrArray *data_files = NULL;
ot_lhash GHashTable *new_pack_indexes = NULL;
ot_lptrarray GPtrArray *ret_cached_indexes = NULL;
ot_lptrarray GPtrArray *ret_uncached_indexes = NULL;
ot_lptrarray GPtrArray *meta_index_files = NULL;
ot_lptrarray GPtrArray *data_index_files = NULL;
ot_lptrarray GPtrArray *meta_data_files = NULL;
ot_lptrarray GPtrArray *data_data_files = NULL;
ot_lhash GHashTable *new_pack_meta_indexes = NULL;
ot_lhash GHashTable *new_pack_data_indexes = NULL;
ot_lptrarray GPtrArray *ret_cached_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_cached_data_indexes = NULL;
ot_lptrarray GPtrArray *ret_uncached_meta_indexes = NULL;
ot_lptrarray GPtrArray *ret_uncached_data_indexes = NULL;
ot_lvariant GVariant *csum_bytes = NULL;
ot_lvariant GVariant *bloom = NULL;
ot_lfree char *pack_checksum = NULL;
@ -1791,8 +1928,10 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error))
goto out;
ret_cached_indexes = g_ptr_array_new_with_free_func (g_free);
ret_uncached_indexes = g_ptr_array_new_with_free_func (g_free);
ret_cached_meta_indexes = g_ptr_array_new_with_free_func (g_free);
ret_cached_data_indexes = g_ptr_array_new_with_free_func (g_free);
ret_uncached_meta_indexes = g_ptr_array_new_with_free_func (g_free);
ret_uncached_data_indexes = g_ptr_array_new_with_free_func (g_free);
if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
&superindex_variant, error))
@ -1801,59 +1940,45 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
if (!ostree_validate_structureof_pack_superindex (superindex_variant, error))
goto out;
new_pack_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
new_pack_meta_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
new_pack_data_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_variant_get_child (superindex_variant, 2, "a(ayay)",
&superindex_contents_iter);
while (g_variant_iter_loop (superindex_contents_iter,
"(@ay@ay)", &csum_bytes, &bloom))
{
pack_checksum = ostree_checksum_from_bytes_v (csum_bytes);
g_hash_table_insert (new_pack_indexes, pack_checksum, pack_checksum);
g_hash_table_insert (new_pack_meta_indexes, pack_checksum, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
if (!list_files_in_dir_matching (cache_path,
"ostpack-", ".index",
&index_files,
cancellable, error))
g_variant_get_child (superindex_variant, 3, "a(ayay)",
&superindex_contents_iter);
while (g_variant_iter_loop (superindex_contents_iter,
"(@ay@ay)", &csum_bytes, &bloom))
{
pack_checksum = ostree_checksum_from_bytes_v (csum_bytes);
g_hash_table_insert (new_pack_data_indexes, pack_checksum, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
if (!delete_no_longer_referenced (self, cache_path,
"ostmetapack-", ".index",
new_pack_meta_indexes,
ret_cached_meta_indexes,
cancellable, error))
goto out;
for (i = 0; i < index_files->len; i++)
{
GFile *index_file = index_files->pdata[i];
g_free (pack_checksum);
pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (index_file));
if (!g_hash_table_lookup (new_pack_indexes, pack_checksum))
{
if (!ot_gfile_unlink (index_file, cancellable, error))
goto out;
}
g_ptr_array_add (ret_cached_indexes, pack_checksum);
pack_checksum = NULL; /* transfer ownership */
}
g_hash_table_iter_init (&hash_iter, new_pack_indexes);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *cur_pack_checksum = key;
gboolean found = FALSE;
if (!delete_no_longer_referenced (self, cache_path,
"ostdatapack-", ".index",
new_pack_data_indexes,
ret_cached_data_indexes,
cancellable, error))
goto out;
for (i = 0; i < ret_cached_indexes->len; i++)
{
if (strcmp (cur_pack_checksum, ret_cached_indexes->pdata[i]) == 0)
{
found = TRUE;
break;
}
}
if (!found)
g_ptr_array_add (ret_uncached_indexes, g_strdup (cur_pack_checksum));
}
gather_uncached (new_pack_meta_indexes, ret_cached_meta_indexes, ret_uncached_meta_indexes);
gather_uncached (new_pack_data_indexes, ret_cached_data_indexes, ret_uncached_data_indexes);
superindex_cache_path = g_file_get_child (cache_path, "index");
if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error))
@ -1861,29 +1986,22 @@ ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
/* Now also delete stale pack files */
if (!list_files_in_dir_matching (cache_path,
"ostpack-", ".data",
&data_files,
cancellable, error))
if (!delete_no_longer_referenced (self, cache_path,
"ostmetapack-", ".data",
new_pack_meta_indexes, NULL,
cancellable, error))
goto out;
if (!delete_no_longer_referenced (self, cache_path,
"ostdatapack-", ".data",
new_pack_data_indexes, NULL,
cancellable, error))
goto out;
for (i = 0; i < data_files->len; i++)
{
GFile *data_file = data_files->pdata[i];
g_free (pack_checksum);
pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (data_file));
if (!g_hash_table_lookup (new_pack_indexes, pack_checksum))
{
if (!ot_gfile_unlink (data_file, cancellable, error))
goto out;
}
}
ret = TRUE;
ot_transfer_out_value (out_cached_indexes, &ret_cached_indexes);
ot_transfer_out_value (out_uncached_indexes, &ret_uncached_indexes);
ot_transfer_out_value (out_cached_meta_indexes, &ret_cached_meta_indexes);
ot_transfer_out_value (out_cached_data_indexes, &ret_cached_data_indexes);
ot_transfer_out_value (out_uncached_meta_indexes, &ret_uncached_data_indexes);
ot_transfer_out_value (out_uncached_data_indexes, &ret_uncached_data_indexes);
out:
if (superindex_contents_iter)
g_variant_iter_free (superindex_contents_iter);
@ -1905,11 +2023,24 @@ ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
goto out;
if (!list_files_in_dir_matching (cache_path,
"ostpack-", ".data",
"ostmetapack-", ".data",
&data_files,
cancellable, error))
goto out;
for (i = 0; i < data_files->len; i++)
{
GFile *data_file = data_files->pdata[i];
if (!ot_gfile_unlink (data_file, cancellable, error))
goto out;
}
ot_clear_ptrarray (&data_files);
if (!list_files_in_dir_matching (cache_path,
"ostdatapack-", ".data",
&data_files,
cancellable, error))
goto out;
for (i = 0; i < data_files->len; i++)
{
GFile *data_file = data_files->pdata[i];
@ -1931,6 +2062,7 @@ gboolean
ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error)
@ -1944,7 +2076,7 @@ ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
cancellable, error))
goto out;
cached_pack_path = get_pack_index_name_from_checksum (cache_dir, pack_checksum);
cached_pack_path = get_pack_index_path (cache_dir, is_meta, pack_checksum);
if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT,
&ret_variant, error))
goto out;
@ -1964,6 +2096,7 @@ gboolean
ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error)
@ -1989,7 +2122,7 @@ ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
goto out;
target_path = get_pack_index_name_from_checksum (cachedir, pack_checksum);
target_path = get_pack_index_path (cachedir, is_meta, pack_checksum);
if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error))
goto out;
@ -2007,6 +2140,7 @@ gboolean
ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error)
@ -2020,7 +2154,7 @@ ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
cancellable, error))
goto out;
cached_pack_path = get_pack_data_name_from_checksum (cache_dir, pack_checksum);
cached_pack_path = get_pack_data_path (cache_dir, is_meta, pack_checksum);
if (g_file_query_exists (cached_pack_path, cancellable))
{
ret_cached_path = cached_pack_path;
@ -2044,6 +2178,7 @@ gboolean
ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error)
@ -2055,8 +2190,7 @@ ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
goto out;
target_path = get_pack_data_name_from_checksum (cachedir, pack_checksum);
target_path = get_pack_data_path (cachedir, is_meta, pack_checksum);
if (!ot_gfile_rename (cached_path, target_path, cancellable, error))
goto out;
@ -2908,37 +3042,10 @@ list_loose_objects (OstreeRepo *self,
return ret;
}
GFile *
ostree_repo_get_pack_index_path (OstreeRepo *self,
const char *checksum)
{
char *name;
GFile *ret;
name = g_strconcat ("ostpack-", checksum, ".index", NULL);
ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
g_free (name);
return ret;
}
GFile *
ostree_repo_get_pack_data_path (OstreeRepo *self,
const char *checksum)
{
char *name;
GFile *ret;
name = g_strconcat ("ostpack-", checksum, ".data", NULL);
ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
g_free (name);
return ret;
}
gboolean
ostree_repo_load_pack_index (OstreeRepo *self,
const char *sha256,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error)
@ -2948,21 +3055,21 @@ ostree_repo_load_pack_index (OstreeRepo *self,
ot_lvariant GVariant *ret_variant = NULL;
ot_lobj GFile *path = NULL;
ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256);
ret_variant = g_hash_table_lookup (priv->pack_index_mappings, pack_checksum);
if (ret_variant)
{
g_variant_ref (ret_variant);
}
else
{
path = ostree_repo_get_pack_index_path (self, sha256);
path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum);
if (!map_variant_file_check_header_string (path,
OSTREE_PACK_INDEX_VARIANT_FORMAT,
"OSTv0PACKINDEX",
&ret_variant,
cancellable, error))
goto out;
g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256),
g_hash_table_insert (priv->pack_index_mappings, g_strdup (pack_checksum),
g_variant_ref (ret_variant));
}
@ -2973,19 +3080,16 @@ ostree_repo_load_pack_index (OstreeRepo *self,
}
/**
* ostree_repo_map_pack_file:
* @self:
* @sha256: Checksum of pack file
* @out_data: (out): Pointer to pack file data
* @cancellable:
* @error:
*
* Ensure that the given pack file is mapped into
* memory.
*/
gboolean
ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256,
const char *pack_checksum,
gboolean is_meta,
guchar **out_data,
guint64 *out_len,
GCancellable *cancellable,
@ -2998,16 +3102,16 @@ ostree_repo_map_pack_file (OstreeRepo *self,
GMappedFile *map = NULL;
ot_lobj GFile *path = NULL;
map = g_hash_table_lookup (priv->pack_data_mappings, sha256);
map = g_hash_table_lookup (priv->pack_data_mappings, pack_checksum);
if (map == NULL)
{
path = ostree_repo_get_pack_data_path (self, sha256);
path = get_pack_data_path (priv->pack_dir, is_meta, pack_checksum);
map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error);
if (!map)
goto out;
g_hash_table_insert (priv->pack_data_mappings, g_strdup (sha256), map);
g_hash_table_insert (priv->pack_data_mappings, g_strdup (pack_checksum), map);
ret_data = g_mapped_file_get_contents (map);
}
@ -3058,53 +3162,55 @@ ostree_repo_load_file (OstreeRepo *self,
error))
goto out;
/* Blah, right now we need to look up the content too to get the file size */
if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
checksum, &content_loose_path,
&content_pack_checksum, &content_pack_offset,
cancellable, error))
goto out;
if (content_loose_path)
if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR)
{
content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
if (!content_loose_info)
/* Blah, right now we need to look up the content too to get the file size */
if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
checksum, &content_loose_path,
&content_pack_checksum, &content_pack_offset,
cancellable, error))
goto out;
g_file_info_set_attribute_uint64 (ret_file_info,
"standard::size",
g_file_info_get_attribute_uint64 (content_loose_info, "standard::size"));
}
/* fixme - don't have file size for packed =/ */
if (content_loose_path)
{
content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
if (!content_loose_info)
goto out;
g_file_info_set_attribute_uint64 (ret_file_info,
"standard::size",
g_file_info_get_attribute_uint64 (content_loose_info, "standard::size"));
}
/* fixme - don't have file size for packed =/ */
/* Now, look for the content */
if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR
&& out_input)
{
if (content_pack_checksum != NULL)
if (out_input)
{
if (!ostree_repo_map_pack_file (self, content_pack_checksum,
&content_pack_data, &content_pack_len,
cancellable, error))
goto out;
if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len,
content_pack_offset, TRUE,
&packed_object, cancellable, error))
goto out;
ret_input = ostree_read_pack_entry_as_stream (packed_object);
}
else if (content_loose_path != NULL)
{
ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error);
if (!ret_input)
goto out;
}
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Couldn't find object '%s'", checksum);
goto out;
if (content_pack_checksum != NULL)
{
if (!ostree_repo_map_pack_file (self, content_pack_checksum, FALSE,
&content_pack_data, &content_pack_len,
cancellable, error))
goto out;
if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len,
content_pack_offset, TRUE, FALSE,
&packed_object, cancellable, error))
goto out;
ret_input = ostree_read_pack_entry_as_stream (packed_object);
}
else if (content_loose_path != NULL)
{
ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error);
if (!ret_input)
goto out;
}
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Couldn't find object '%s'", checksum);
goto out;
}
}
}
}
@ -3142,12 +3248,14 @@ ostree_repo_load_file (OstreeRepo *self,
static gboolean
list_objects_in_index (OstreeRepo *self,
const char *pack_checksum,
gboolean is_meta,
GHashTable *inout_objects,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint32 objtype_u32;
OstreeRepoPrivate *priv = GET_PRIVATE (self);
guint32 objtype_u8;
guint64 offset;
ot_lobj GFile *index_path = NULL;
ot_lvariant GVariant *index_variant = NULL;
@ -3156,15 +3264,16 @@ list_objects_in_index (OstreeRepo *self,
ot_lfree char *checksum = NULL;
GVariantIter content_iter;
index_path = ostree_repo_get_pack_index_path (self, pack_checksum);
index_path = get_pack_index_path (priv->pack_dir, is_meta, pack_checksum);
if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error))
if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta,
&index_variant, cancellable, error))
goto out;
contents = g_variant_get_child_value (index_variant, 2);
g_variant_iter_init (&content_iter, contents);
while (g_variant_iter_loop (&content_iter, "(u@ayt)", &objtype_u32, &csum_bytes, &offset))
while (g_variant_iter_loop (&content_iter, "(y@ayt)", &objtype_u8, &csum_bytes, &offset))
{
GVariant *obj_key;
GVariant *objdata;
@ -3172,7 +3281,7 @@ list_objects_in_index (OstreeRepo *self,
GVariantBuilder pack_contents_builder;
gboolean is_loose;
objtype = (OstreeObjectType) GUINT32_FROM_BE (objtype_u32);
objtype = (OstreeObjectType) objtype_u8;
offset = GUINT64_FROM_BE (offset);
g_variant_builder_init (&pack_contents_builder,
@ -3221,19 +3330,27 @@ list_packed_objects (OstreeRepo *self,
{
gboolean ret = FALSE;
guint i;
ot_lptrarray GPtrArray *index_checksums = NULL;
ot_lptrarray GPtrArray *meta_index_checksums = NULL;
ot_lptrarray GPtrArray *data_index_checksums = NULL;
if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error))
if (!ostree_repo_list_pack_indexes (self, &meta_index_checksums, &data_index_checksums,
cancellable, error))
goto out;
for (i = 0; i < index_checksums->len; i++)
for (i = 0; i < meta_index_checksums->len; i++)
{
const char *checksum = index_checksums->pdata[i];
if (!list_objects_in_index (self, checksum, inout_objects, cancellable, error))
const char *checksum = meta_index_checksums->pdata[i];
if (!list_objects_in_index (self, checksum, TRUE, inout_objects, cancellable, error))
goto out;
}
for (i = 0; i < data_index_checksums->len; i++)
{
const char *checksum = data_index_checksums->pdata[i];
if (!list_objects_in_index (self, checksum, FALSE, inout_objects, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;
@ -3251,27 +3368,37 @@ find_object_in_packs (OstreeRepo *self,
gboolean ret = FALSE;
guint i;
guint64 ret_pack_offset;
gboolean is_meta;
ot_lptrarray GPtrArray *index_checksums = NULL;
ot_lfree char *ret_pack_checksum = NULL;
ot_lobj GFile *index_path = NULL;
ot_lvariant GVariant *csum_bytes = NULL;
ot_lvariant GVariant *index_variant = NULL;
csum_bytes = ostree_checksum_to_bytes_v (checksum);
if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error))
goto out;
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
if (is_meta)
{
if (!ostree_repo_list_pack_indexes (self, &index_checksums, NULL,
cancellable, error))
goto out;
}
else
{
if (!ostree_repo_list_pack_indexes (self, NULL, &index_checksums,
cancellable, error))
goto out;
}
for (i = 0; i < index_checksums->len; i++)
{
const char *pack_checksum = index_checksums->pdata[i];
guint64 offset;
g_clear_object (&index_path);
index_path = ostree_repo_get_pack_index_path (self, pack_checksum);
ot_clear_gvariant (&index_variant);
if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error))
if (!ostree_repo_load_pack_index (self, pack_checksum, is_meta, &index_variant,
cancellable, error))
goto out;
if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset))
@ -3388,12 +3515,12 @@ ostree_repo_load_variant (OstreeRepo *self,
}
else if (pack_checksum != NULL)
{
if (!ostree_repo_map_pack_file (self, pack_checksum, &pack_data, &pack_len,
if (!ostree_repo_map_pack_file (self, pack_checksum, TRUE, &pack_data, &pack_len,
cancellable, error))
goto out;
if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset,
TRUE, &packed_object, cancellable, error))
TRUE, TRUE, &packed_object, cancellable, error))
goto out;
if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE,

26
src/libostree/ostree-repo.h
View File

@ -150,19 +150,21 @@ gboolean ostree_repo_load_variant (OstreeRepo *self,
GError **error);
gboolean ostree_repo_load_pack_index (OstreeRepo *self,
const char *sha256,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_pack_data (OstreeRepo *self,
const char *sha256,
const char *pack_checksum,
guchar **out_data,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256,
gboolean is_meta,
guchar **out_data,
guint64 *out_len,
GCancellable *cancellable,
@ -241,6 +243,7 @@ gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
gboolean ostree_repo_add_pack_file (OstreeRepo *self,
const char *checksum,
gboolean is_meta,
GFile *pack_index_path,
GFile *pack_data_path,
GCancellable *cancellable,
@ -249,8 +252,10 @@ gboolean ostree_repo_add_pack_file (OstreeRepo *self,
gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name,
GFile *superindex_path,
GPtrArray **out_cached_indexes,
GPtrArray **out_uncached_indexes,
GPtrArray **out_cached_meta_indexes,
GPtrArray **out_cached_data_indexes,
GPtrArray **out_uncached_meta_indexes,
GPtrArray **out_uncached_data_indexes,
GCancellable *cancellable,
GError **error);
@ -262,6 +267,7 @@ gboolean ostree_repo_clean_cached_remote_pack_data (OstreeRepo *self,
gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
@ -269,6 +275,7 @@ gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
@ -276,6 +283,7 @@ gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error);
@ -283,6 +291,7 @@ gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
gboolean is_meta,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
@ -334,16 +343,11 @@ gboolean ostree_repo_list_objects (OstreeRepo *self,
GError **error);
gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_indexes,
GPtrArray **out_meta_indexes,
GPtrArray **out_data_indexes,
GCancellable *cancellable,
GError **error);
GFile * ostree_repo_get_pack_index_path (OstreeRepo *self,
const char *checksum);
GFile * ostree_repo_get_pack_data_path (OstreeRepo *self,
const char *checksum);
G_END_DECLS
#endif /* _OSTREE_REPO */

90
src/ostree/ostree-pull.c
View File

@ -62,7 +62,8 @@ typedef struct {
SoupURI *base_uri;
gboolean fetched_packs;
GPtrArray *cached_pack_indexes;
GPtrArray *cached_meta_pack_indexes;
GPtrArray *cached_data_pack_indexes;
GHashTable *file_checksums_to_fetch;
@ -266,6 +267,7 @@ fetch_uri_contents_utf8 (OtPullData *pull_data,
static gboolean
fetch_one_pack_file (OtPullData *pull_data,
const char *pack_checksum,
gboolean is_meta,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error)
@ -277,26 +279,26 @@ fetch_one_pack_file (OtPullData *pull_data,
SoupURI *pack_uri = NULL;
if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, &ret_cached_path,
pack_checksum, is_meta, &ret_cached_path,
cancellable, error))
goto out;
if (ret_cached_path == NULL)
{
pack_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL);
pack_name = ostree_get_pack_data_name (is_meta, pack_checksum);
pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL);
if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error))
goto out;
if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, tmp_path,
pack_checksum, is_meta, tmp_path,
cancellable, error))
goto out;
}
if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
pack_checksum, &ret_cached_path,
pack_checksum, is_meta, &ret_cached_path,
cancellable, error))
goto out;
@ -322,19 +324,25 @@ find_object_in_remote_packs (OtPullData *pull_data,
gboolean ret = FALSE;
guint64 offset;
guint i;
GPtrArray *iter;
ot_lvariant GVariant *mapped_pack = NULL;
ot_lvariant GVariant *csum_bytes = NULL;
ot_lfree char *ret_pack_checksum = NULL;
csum_bytes = ostree_checksum_to_bytes_v (checksum);
for (i = 0; i < pull_data->cached_pack_indexes->len; i++)
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
iter = pull_data->cached_meta_pack_indexes;
else
iter = pull_data->cached_data_pack_indexes;
for (i = 0; i < iter->len; i++)
{
const char *pack_checksum = pull_data->cached_pack_indexes->pdata[i];
const char *pack_checksum = iter->pdata[i];
ot_clear_gvariant (&mapped_pack);
if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, &mapped_pack,
pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
&mapped_pack,
cancellable, error))
goto out;
@ -354,17 +362,18 @@ find_object_in_remote_packs (OtPullData *pull_data,
}
static gboolean
fetch_one_cache_index (OtPullData *pull_data,
const char *pack_checksum,
GCancellable *cancellable,
GError **error)
fetch_one_cache_index (OtPullData *pull_data,
const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GFile *tmp_path = NULL;
ot_lfree char *pack_index_name = NULL;
SoupURI *index_uri = NULL;
pack_index_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL);
pack_index_name = ostree_get_pack_index_name (is_meta, pack_checksum);
index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL);
if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path,
@ -372,7 +381,7 @@ fetch_one_cache_index (OtPullData *pull_data,
goto out;
if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
pack_checksum, tmp_path,
pack_checksum, is_meta, tmp_path,
cancellable, error))
goto out;
@ -398,8 +407,10 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data,
gboolean ret = FALSE;
guint i;
ot_lobj GFile *superindex_tmppath = NULL;
ot_lptrarray GPtrArray *cached_indexes = NULL;
ot_lptrarray GPtrArray *uncached_indexes = NULL;
ot_lptrarray GPtrArray *cached_meta_indexes = NULL;
ot_lptrarray GPtrArray *cached_data_indexes = NULL;
ot_lptrarray GPtrArray *uncached_meta_indexes = NULL;
ot_lptrarray GPtrArray *uncached_data_indexes = NULL;
ot_lvariant GVariant *superindex_variant = NULL;
GVariantIter *contents_iter = NULL;
SoupURI *superindex_uri = NULL;
@ -412,22 +423,33 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data,
if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name,
superindex_tmppath,
&cached_indexes, &uncached_indexes,
&cached_meta_indexes,
&cached_data_indexes,
&uncached_meta_indexes,
&uncached_data_indexes,
cancellable, error))
goto out;
for (i = 0; i < cached_indexes->len; i++)
g_ptr_array_add (pull_data->cached_pack_indexes,
g_strdup (cached_indexes->pdata[i]));
for (i = 0; i < cached_meta_indexes->len; i++)
g_ptr_array_add (pull_data->cached_meta_pack_indexes,
g_strdup (cached_meta_indexes->pdata[i]));
for (i = 0; i < cached_data_indexes->len; i++)
g_ptr_array_add (pull_data->cached_data_pack_indexes,
g_strdup (cached_data_indexes->pdata[i]));
for (i = 0; i < uncached_indexes->len; i++)
for (i = 0; i < uncached_meta_indexes->len; i++)
{
const char *pack_checksum = uncached_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, cancellable, error))
const char *pack_checksum = uncached_meta_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, TRUE, cancellable, error))
goto out;
g_ptr_array_add (pull_data->cached_pack_indexes, g_strdup (pack_checksum));
g_ptr_array_add (pull_data->cached_meta_pack_indexes, g_strdup (pack_checksum));
}
for (i = 0; i < uncached_data_indexes->len; i++)
{
const char *pack_checksum = uncached_data_indexes->pdata[i];
if (!fetch_one_cache_index (pull_data, pack_checksum, FALSE, cancellable, error))
goto out;
g_ptr_array_add (pull_data->cached_data_pack_indexes, g_strdup (pack_checksum));
}
ret = TRUE;
@ -485,6 +507,7 @@ fetch_object_if_not_stored (OtPullData *pull_data,
gboolean ret = FALSE;
guint64 pack_offset = 0;
gboolean is_stored;
gboolean is_meta;
ot_lobj GInputStream *ret_input = NULL;
ot_lobj GFile *temp_path = NULL;
ot_lobj GFile *stored_path = NULL;
@ -494,6 +517,8 @@ fetch_object_if_not_stored (OtPullData *pull_data,
ot_lvariant GVariant *pack_entry = NULL;
GMappedFile *pack_map = NULL;
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
if (!ostree_repo_find_object (pull_data->repo, objtype, checksum,
&stored_path, &local_pack_checksum, NULL,
cancellable, error))
@ -505,7 +530,8 @@ fetch_object_if_not_stored (OtPullData *pull_data,
if (!pull_data->fetched_packs)
{
pull_data->fetched_packs = TRUE;
pull_data->cached_pack_indexes = g_ptr_array_new_with_free_func (g_free);
pull_data->cached_meta_pack_indexes = g_ptr_array_new_with_free_func (g_free);
pull_data->cached_data_pack_indexes = g_ptr_array_new_with_free_func (g_free);
if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error))
goto out;
@ -521,8 +547,8 @@ fetch_object_if_not_stored (OtPullData *pull_data,
{
g_assert (!is_stored);
if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path,
cancellable, error))
if (!fetch_one_pack_file (pull_data, remote_pack_checksum, is_meta,
&pack_path, cancellable, error))
goto out;
pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error);
@ -531,7 +557,7 @@ fetch_object_if_not_stored (OtPullData *pull_data,
if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
g_mapped_file_get_length (pack_map),
pack_offset, FALSE, &pack_entry,
pack_offset, FALSE, is_meta, &pack_entry,
cancellable, error))
goto out;
@ -1147,8 +1173,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_clear_object (&pull_data->session);
if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri);
if (pull_data->cached_pack_indexes)
g_ptr_array_unref (pull_data->cached_pack_indexes);
ot_clear_ptrarray (&pull_data->cached_meta_pack_indexes);
ot_clear_ptrarray (&pull_data->cached_data_pack_indexes);
if (summary_uri)
soup_uri_free (summary_uri);
return ret;

170
src/ostree/ot-builtin-fsck.c
View File

@ -43,88 +43,76 @@ typedef struct {
} OtFsckData;
static gboolean
fsck_pack_files (OtFsckData *data,
GCancellable *cancellable,
GError **error)
fsck_one_pack_file (OtFsckData *data,
const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
guint32 objtype;
guchar objtype_u8;
guint64 offset;
guint64 pack_size;
ot_lptrarray GPtrArray *pack_indexes = NULL;
ot_lfree char *path = NULL;
ot_lobj GFileInfo *pack_info = NULL;
ot_lobj GInputStream *input = NULL;
ot_lvariant GVariant *index_variant = NULL;
ot_lobj GFile *pack_index_path = NULL;
ot_lobj GFile *pack_data_path = NULL;
ot_lobj GFileInfo *pack_info = NULL;
ot_lobj GInputStream *input = NULL;
GChecksum *pack_content_checksum = NULL;
GVariantIter *index_content_iter = NULL;
if (!ostree_repo_list_pack_indexes (data->repo, &pack_indexes, cancellable, error))
g_free (path);
path = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
pack_index_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
if (!ot_util_variant_map (pack_index_path,
OSTREE_PACK_INDEX_VARIANT_FORMAT,
&index_variant, error))
goto out;
if (!ostree_validate_structureof_pack_index (index_variant, error))
goto out;
for (i = 0; i < pack_indexes->len; i++)
{
const char *checksum = pack_indexes->pdata[i];
g_clear_object (&pack_index_path);
pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum);
ot_clear_gvariant (&index_variant);
if (!ot_util_variant_map (pack_index_path,
OSTREE_PACK_INDEX_VARIANT_FORMAT,
&index_variant, error))
goto out;
g_free (path);
path = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
pack_data_path = g_file_resolve_relative_path (ostree_repo_get_path (data->repo), path);
if (!ostree_validate_structureof_pack_index (index_variant, error))
goto out;
input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
if (!input)
goto out;
g_clear_object (&pack_data_path);
pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum);
g_clear_object (&input);
input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
if (!input)
goto out;
g_clear_object (&pack_info);
pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!pack_info)
goto out;
pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!pack_info)
goto out;
pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
if (pack_content_checksum)
g_checksum_free (pack_content_checksum);
if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error))
goto out;
if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error))
goto out;
if (strcmp (g_checksum_get_string (pack_content_checksum), checksum) != 0)
if (strcmp (g_checksum_get_string (pack_content_checksum), pack_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', expected checksum %s",
pack_checksum, g_checksum_get_string (pack_content_checksum));
goto out;
}
g_variant_get_child (index_variant, 2, "a(yayt)", &index_content_iter);
while (g_variant_iter_loop (index_content_iter, "(y@ayt)",
&objtype_u8, NULL, &offset))
{
offset = GUINT64_FROM_BE (offset);
if (offset > pack_size)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', expected checksum %s",
checksum, g_checksum_get_string (pack_content_checksum));
"corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
pack_checksum,
offset, pack_size);
goto out;
}
g_variant_get_child (index_variant, 2, "a(uayt)", &index_content_iter);
while (g_variant_iter_loop (index_content_iter, "(u@ayt)",
&objtype, NULL, &offset))
{
offset = GUINT64_FROM_BE (offset);
if (offset > pack_size)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
checksum,
offset, pack_size);
goto out;
}
}
data->n_pack_files++;
}
ret = TRUE;
@ -135,6 +123,44 @@ fsck_pack_files (OtFsckData *data,
return ret;
}
static gboolean
fsck_pack_files (OtFsckData *data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
ot_lptrarray GPtrArray *meta_pack_indexes = NULL;
ot_lptrarray GPtrArray *data_pack_indexes = NULL;
if (!ostree_repo_list_pack_indexes (data->repo, &meta_pack_indexes, &data_pack_indexes,
cancellable, error))
goto out;
for (i = 0; i < meta_pack_indexes->len; i++)
{
const char *pack_checksum = meta_pack_indexes->pdata[i];
if (!fsck_one_pack_file (data, pack_checksum, TRUE, cancellable, error))
goto out;
data->n_pack_files++;
}
for (i = 0; i < data_pack_indexes->len; i++)
{
const char *pack_checksum = data_pack_indexes->pdata[i];
if (!fsck_one_pack_file (data, pack_checksum, FALSE, cancellable, error))
goto out;
data->n_pack_files++;
}
ret = TRUE;
out:
return ret;
}
static gboolean
fsck_reachable_objects_from_commits (OtFsckData *data,
GHashTable *commits,
@ -198,17 +224,26 @@ fsck_reachable_objects_from_commits (OtFsckData *data,
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
{
if (!ostree_validate_structureof_commit (metadata, error))
goto out;
{
g_prefix_error (error, "While validating commit metadata '%s': ", checksum);
goto out;
}
}
else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
{
if (!ostree_validate_structureof_dirtree (metadata, error))
goto out;
{
g_prefix_error (error, "While validating directory tree '%s': ", checksum);
goto out;
}
}
else if (objtype == OSTREE_OBJECT_TYPE_DIR_META)
{
if (!ostree_validate_structureof_dirmeta (metadata, error))
goto out;
{
g_prefix_error (error, "While validating directory metadata '%s': ", checksum);
goto out;
}
}
else
g_assert_not_reached ();
@ -236,7 +271,10 @@ fsck_reachable_objects_from_commits (OtFsckData *data,
mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
if (!ostree_validate_structureof_file_mode (mode, error))
goto out;
{
g_prefix_error (error, "While validating file '%s': ", checksum);
goto out;
}
}
else
{

387
src/ostree/ot-builtin-pack.c
View File

@ -197,18 +197,16 @@ compare_index_content (gconstpointer ap,
gpointer b = *((gpointer*)bp);
GVariant *a_v = a;
GVariant *b_v = b;
guint32 a_objtype;
guint32 b_objtype;
guchar a_objtype;
guchar b_objtype;
guint64 a_offset;
guint64 b_offset;
int c;
ot_lvariant GVariant *a_csum_bytes = NULL;
ot_lvariant GVariant *b_csum_bytes = NULL;
g_variant_get (a_v, "(u@ayt)", &a_objtype, &a_csum_bytes, &a_offset);
g_variant_get (b_v, "(u@ayt)", &b_objtype, &b_csum_bytes, &b_offset);
a_objtype = GUINT32_FROM_BE (a_objtype);
b_objtype = GUINT32_FROM_BE (b_objtype);
g_variant_get (a_v, "(y@ayt)", &a_objtype, &a_csum_bytes, &a_offset);
g_variant_get (b_v, "(y@ayt)", &b_objtype, &b_csum_bytes, &b_offset);
c = ostree_cmp_checksum_bytes (ostree_checksum_bytes_peek (a_csum_bytes),
ostree_checksum_bytes_peek (b_csum_bytes));
if (c == 0)
@ -270,8 +268,121 @@ delete_loose_object (OtRepackData *data,
return ret;
}
static gboolean
pack_one_meta_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
GVariant **out_packed_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lobj GFile *object_path = NULL;
ot_lvariant GVariant *metadata_v = NULL;
ot_lvariant GVariant *ret_packed_object = NULL;
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
if (!ostree_map_metadata_file (object_path, objtype, &metadata_v, error))
goto out;
ret_packed_object = g_variant_new ("(y@ayv)", (guchar) objtype,
ostree_checksum_to_bytes_v (checksum),
metadata_v);
ret = TRUE;
ot_transfer_out_value (out_packed_object, &ret_packed_object);
out:
return ret;
}
static gboolean
pack_one_data_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
guint64 expected_objsize,
GVariant **out_packed_object,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 objsize;
guchar entry_flags = 0;
GInputStream *read_object_in; /* nofree */
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInputStream *object_input = NULL;
ot_lobj GFileInfo *object_file_info = NULL;
ot_lobj GMemoryOutputStream *object_data_stream = NULL;
ot_lobj GConverter *compressor = NULL;
ot_lobj GConverterInputStream *compressed_object_input = NULL;
ot_lvariant GVariant *ret_packed_object = NULL;
switch (data->int_compression)
{
case OT_COMPRESSION_GZIP:
{
entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
break;
}
default:
{
g_assert_not_reached ();
}
}
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
object_input = g_file_read (object_path, cancellable, error);
if (!object_input)
goto out;
object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
if (!object_file_info)
goto out;
objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_assert_cmpint (objsize, ==, expected_objsize);
object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", compressor,
"base-stream", object_input,
"close-base-stream", TRUE,
NULL);
read_object_in = (GInputStream*)compressed_object_input;
}
else
{
read_object_in = (GInputStream*)object_input;
}
if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error))
goto out;
{
guchar *data = g_memory_output_stream_get_data (object_data_stream);
gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
ret_packed_object = g_variant_new ("(yy@ay@ay)", (guchar)objtype, entry_flags,
ostree_checksum_to_bytes_v (checksum),
ot_gvariant_new_bytearray (data, data_len));
}
ret = TRUE;
ot_transfer_out_value (out_packed_object, &ret_packed_object);
out:
return ret;
}
static gboolean
create_pack_file (OtRepackData *data,
gboolean is_meta,
GPtrArray *objects,
GCancellable *cancellable,
GError **error)
@ -285,19 +396,12 @@ create_pack_file (OtRepackData *data,
ot_lobj GOutputStream *index_out = NULL;
ot_lobj GFile *pack_temppath = NULL;
ot_lobj GOutputStream *pack_out = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_file_info = NULL;
ot_lobj GFileInputStream *object_input = NULL;
ot_lobj GConverter *compressor = NULL;
ot_lobj GConverterInputStream *compressed_object_input = NULL;
ot_lptrarray GPtrArray *index_content_list = NULL;
ot_lvariant GVariant *pack_header = NULL;
ot_lvariant GVariant *packed_object = NULL;
ot_lvariant GVariant *index_content = NULL;
ot_lfree char *pack_name = NULL;
ot_lobj GFile *pack_file_path = NULL;
ot_lobj GFile *pack_index_path = NULL;
GMemoryOutputStream *object_data_stream = NULL;
GVariantBuilder index_content_builder;
GChecksum *pack_checksum = NULL;
@ -324,7 +428,7 @@ create_pack_file (OtRepackData *data,
pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
pack_header = g_variant_new ("(s@a{sv}t)",
"OSTv0PACKFILE",
is_meta ? "OSTv0PACKMETAFILE" : "OSTv0PACKDATAFILE",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
(guint64)objects->len);
@ -339,92 +443,37 @@ create_pack_file (OtRepackData *data,
guint32 objtype_u32;
OstreeObjectType objtype;
guint64 expected_objsize;
guint64 objsize;
GInputStream *read_object_in;
guchar entry_flags = 0;
GVariant *index_entry;
ot_lvariant GVariant *packed_object = NULL;
ot_lvariant GVariant *index_entry = NULL;
g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32;
switch (data->int_compression)
if (is_meta)
{
case OT_COMPRESSION_GZIP:
{
entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
break;
}
default:
{
g_assert_not_reached ();
}
}
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_input);
object_input = g_file_read (object_path, cancellable, error);
if (!object_input)
goto out;
g_clear_object (&object_file_info);
object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
if (!object_file_info)
goto out;
objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_assert_cmpint (objsize, ==, expected_objsize);
g_clear_object (&object_data_stream);
object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
g_clear_object (&compressor);
compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
g_clear_object (&compressed_object_input);
compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", compressor,
"base-stream", object_input,
"close-base-stream", TRUE,
NULL);
read_object_in = (GInputStream*)compressed_object_input;
if (!pack_one_meta_object (data, checksum, objtype, &packed_object,
cancellable, error))
goto out;
}
else
{
read_object_in = (GInputStream*)object_input;
if (!pack_one_data_object (data, checksum, objtype, expected_objsize,
&packed_object, cancellable, error))
goto out;
}
if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error))
goto out;
ot_clear_gvariant (&packed_object);
{
guchar *data = g_memory_output_stream_get_data (object_data_stream);
gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
packed_object = g_variant_new ("(uy@ay@ay)", GUINT32_TO_BE ((guint32)objtype),
entry_flags,
ostree_checksum_to_bytes_v (checksum),
ot_gvariant_new_bytearray (data, data_len));
g_clear_object (&object_data_stream);
}
if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
goto out;
/* offset points to aligned header size */
index_entry = g_variant_new ("(u@ayt)",
GUINT32_TO_BE ((guint32)objtype),
index_entry = g_variant_new ("(y@ayt)",
(guchar)objtype,
ostree_checksum_to_bytes_v (checksum),
GUINT64_TO_BE (offset));
g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
index_entry = NULL;
if (!write_variant_with_size (pack_out, packed_object, pack_checksum,
&offset, cancellable, error))
goto out;
@ -433,14 +482,14 @@ create_pack_file (OtRepackData *data,
if (!g_output_stream_close (pack_out, cancellable, error))
goto out;
g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(uayt)"));
g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(yayt)"));
g_ptr_array_sort (index_content_list, compare_index_content);
for (i = 0; i < index_content_list->len; i++)
{
GVariant *index_item = index_content_list->pdata[i];
g_variant_builder_add_value (&index_content_builder, index_item);
}
index_content = g_variant_new ("(s@a{sv}@a(uayt))",
index_content = g_variant_new ("(s@a{sv}@a(yayt))",
"OSTv0PACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
g_variant_builder_end (&index_content_builder));
@ -458,6 +507,7 @@ create_pack_file (OtRepackData *data,
if (!ostree_repo_add_pack_file (data->repo,
g_checksum_get_string (pack_checksum),
is_meta,
index_temppath,
pack_temppath,
cancellable,
@ -499,67 +549,14 @@ create_pack_file (OtRepackData *data,
return ret;
}
/**
* cluster_objects_stupidly:
* @objects: Map from serialized object name to objdata
* @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
*
* Just sorts by size currently. Also filters out non-regular object
* content.
*/
static gboolean
cluster_objects_stupidly (OtRepackData *data,
GHashTable *objects,
GPtrArray **out_clusters,
GCancellable *cancellable,
GError **error)
static void
cluster_one_object_chain (OtRepackData *data,
GPtrArray *object_list,
GPtrArray *inout_clusters)
{
gboolean ret = FALSE;
guint i;
guint64 current_size;
guint current_offset;
GHashTableIter hash_iter;
gpointer key, value;
ot_lptrarray GPtrArray *ret_clusters = NULL;
ot_lptrarray GPtrArray *object_list = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_info = NULL;
object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_info);
object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!object_info)
goto out;
if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
continue;
size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_ptr_array_add (object_list,
g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)));
}
g_ptr_array_sort (object_list, compare_object_data_by_size);
ret_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
current_size = 0;
current_offset = 0;
@ -582,7 +579,7 @@ cluster_objects_stupidly (OtRepackData *data,
{
g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
}
g_ptr_array_add (ret_clusters, current);
g_ptr_array_add (inout_clusters, current);
current_size = objsize;
current_offset = i+1;
}
@ -596,9 +593,83 @@ cluster_objects_stupidly (OtRepackData *data,
current_size += objsize;
}
}
}
/**
* cluster_objects_stupidly:
* @objects: Map from serialized object name to objdata
* @out_meta_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
* @out_data_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
*
* Just sorts by size currently. Also filters out non-regular object
* content.
*/
static gboolean
cluster_objects_stupidly (OtRepackData *data,
GHashTable *objects,
GPtrArray **out_meta_clusters,
GPtrArray **out_data_clusters,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTableIter hash_iter;
gpointer key, value;
ot_lptrarray GPtrArray *ret_meta_clusters = NULL;
ot_lptrarray GPtrArray *ret_data_clusters = NULL;
ot_lptrarray GPtrArray *meta_object_list = NULL;
ot_lptrarray GPtrArray *data_object_list = NULL;
ot_lobj GFile *object_path = NULL;
ot_lobj GFileInfo *object_info = NULL;
meta_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
data_object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
GVariant *v;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_info);
object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!object_info)
goto out;
if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
continue;
size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
v = g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size));
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
g_ptr_array_add (meta_object_list, v);
else
g_ptr_array_add (data_object_list, v);
}
g_ptr_array_sort (meta_object_list, compare_object_data_by_size);
g_ptr_array_sort (data_object_list, compare_object_data_by_size);
ret_meta_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
ret_data_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
cluster_one_object_chain (data, meta_object_list, ret_meta_clusters);
cluster_one_object_chain (data, data_object_list, ret_data_clusters);
ret = TRUE;
ot_transfer_out_value (out_clusters, &ret_clusters);
ot_transfer_out_value (out_meta_clusters, &ret_meta_clusters);
ot_transfer_out_value (out_data_clusters, &ret_data_clusters);
out:
return ret;
}
@ -792,7 +863,8 @@ do_incremental_pack (OtRepackData *data,
gboolean ret = FALSE;
guint i;
ot_lhash GHashTable *objects = NULL;
ot_lptrarray GPtrArray *clusters = NULL;
ot_lptrarray GPtrArray *meta_clusters = NULL;
ot_lptrarray GPtrArray *data_clusters = NULL;
ot_lhash GHashTable *loose_objects = NULL;
if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
@ -805,21 +877,30 @@ do_incremental_pack (OtRepackData *data,
g_print ("\n");
g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size);
if (!cluster_objects_stupidly (data, loose_objects, &clusters, cancellable, error))
if (!cluster_objects_stupidly (data, loose_objects, &meta_clusters, &data_clusters,
cancellable, error))
goto out;
if (clusters->len > 0)
g_print ("Going to create %u packfiles\n", clusters->len);
if (meta_clusters->len > 0 || data_clusters->len > 0)
g_print ("Going to create %u meta packfiles, %u data packfiles\n",
meta_clusters->len, data_clusters->len);
else
g_print ("Nothing to do\n");
for (i = 0; i < clusters->len; i++)
if (!opt_analyze_only)
{
GPtrArray *cluster = clusters->pdata[i];
if (!opt_analyze_only)
for (i = 0; i < meta_clusters->len; i++)
{
if (!create_pack_file (data, cluster, cancellable, error))
GPtrArray *cluster = meta_clusters->pdata[i];
if (!create_pack_file (data, TRUE, cluster, cancellable, error))
goto out;
}
for (i = 0; i < data_clusters->len; i++)
{
GPtrArray *cluster = data_clusters->pdata[i];
if (!create_pack_file (data, FALSE, cluster, cancellable, error))
goto out;
}
}

45
src/ostree/ot-builtin-unpack.c
View File

@ -144,15 +144,20 @@ unpack_one_object (OstreeRepo *repo,
static gboolean
delete_one_packfile (OstreeRepo *repo,
const char *pack_checksum,
gboolean is_meta,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
ot_lfree char *data_name = NULL;
ot_lobj GFile *data_path = NULL;
ot_lfree char *index_name = NULL;
ot_lobj GFile *index_path = NULL;
index_path = ostree_repo_get_pack_index_path (repo, pack_checksum);
data_path = ostree_repo_get_pack_data_path (repo, pack_checksum);
index_name = ostree_get_relative_pack_index_path (is_meta, pack_checksum);
index_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), index_name);
data_name = ostree_get_relative_pack_data_path (is_meta, pack_checksum);
data_path = g_file_resolve_relative_path (ostree_repo_get_path (repo), data_name);
if (!ot_gfile_unlink (index_path, cancellable, error))
{
@ -185,7 +190,8 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
ot_lhash GHashTable *objects = NULL;
ot_lptrarray GPtrArray *clusters = NULL;
ot_lhash GHashTable *packed_objects = NULL;
ot_lhash GHashTable *packfiles_to_delete = NULL;
ot_lhash GHashTable *meta_packfiles_to_delete = NULL;
ot_lhash GHashTable *data_packfiles_to_delete = NULL;
ot_lobj GFile *objpath = NULL;
memset (&data, 0, sizeof (data));
@ -220,7 +226,8 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
in_transaction = TRUE;
packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
meta_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
data_packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_iter_init (&hash_iter, packed_objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
@ -232,6 +239,9 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
OstreeObjectType objtype;
gboolean is_loose;
GVariantIter *pack_array_iter;
GHashTable *target_hash;
ostree_object_name_deserialize (objkey, &checksum, &objtype);
objdata = g_hash_table_lookup (objects, objkey);
g_assert (objdata);
@ -240,12 +250,17 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
g_assert (!is_loose);
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
target_hash = meta_packfiles_to_delete;
else
target_hash = data_packfiles_to_delete;
while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
{
if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum))
if (!g_hash_table_lookup (target_hash, pack_checksum))
{
gchar *duped_checksum = g_strdup (pack_checksum);
g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum);
g_hash_table_replace (target_hash, duped_checksum, duped_checksum);
}
}
g_variant_iter_free (pack_array_iter);
@ -261,15 +276,27 @@ ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_commit_transaction (repo, cancellable, error))
goto out;
if (g_hash_table_size (packfiles_to_delete) == 0)
if (g_hash_table_size (meta_packfiles_to_delete) == 0
&& g_hash_table_size (data_packfiles_to_delete) == 0)
g_print ("No pack files; nothing to do\n");
g_hash_table_iter_init (&hash_iter, packfiles_to_delete);
g_hash_table_iter_init (&hash_iter, meta_packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, cancellable, error))
if (!delete_one_packfile (repo, pack_checksum, TRUE, cancellable, error))
goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum);
}
g_hash_table_iter_init (&hash_iter, data_packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, FALSE, cancellable, error))
goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum);