core: Add pack files

This concept is also directly inspired by git.  At present, our
implementation is quite similar, except we don't have delta
compression.
This commit is contained in:
Colin Walters 2012-03-31 10:37:51 -04:00
parent 68cce01556
commit 80bdfd7f42
18 changed files with 4379 additions and 558 deletions

View File

@ -34,6 +34,8 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-ls.c \
src/ostree/ot-builtin-prune.c \
src/ostree/ot-builtin-remote.c \
src/ostree/ot-builtin-pack.c \
src/ostree/ot-builtin-unpack.c \
src/ostree/ot-builtin-rev-parse.c \
src/ostree/ot-builtin-show.c \
src/ostree/ot-main.h \

78
src/libostree/README.md Normal file
View File

@ -0,0 +1,78 @@
Repository design
-----------------
At the heart of OSTree is the repository. It's very similar to git,
with the idea of content-addressed storage. However, OSTree is
designed to store operating system binaries, not source code. There
are several consequences to this. The key difference as compared to
git is that the OSTree definition of "content" includes key Unix
metadata such as owner uid/gid, as well as all extended attributes.
Essentially OSTree is designed so that if two files have the same
OSTree checksum, it's safe to replace them with a hard link. This
fundamental design means that an OSTree repository imposes negligible
overhead. In contrast, a git repository stores copies of
zlib-compressed data.
Key differences versus git
--------------------------
* As mentioned above, extended attributes and owner uid/gid are versioned
* Optimized for Unix hardlinks between repository and checkout
* SHA256 instead of SHA1
* Support for empty directories
Binary files
------------
While this is still in planning, I plan to heavily optimize OSTree for
versioning ELF operating systems. In industry jargon, this would be
"content-aware storage".
Trimming history
----------------
OSTree will also be optimized to trim intermediate history; in theory
one can regenerate binaries from corresponding (git) source code, so
we don't need to keep all possible builds over time.
MILESTONE 1
-----------
* Basic pack files (like git)
MILESTONE 2
-----------
* Store checksums as ay
* Drop version/metadata from tree/dirmeta objects
* Split pack files into metadata/data
* Restructure repository so that links can be generated as a cache;
i.e. objects/raw, pack files are now the canonical
* For files, checksum combination of metadata variant + raw data
- i.e. there is only OSTREE_OBJECT_TYPE_FILE (again)
MILESTONE 3
-----------
* Drop archive/raw distinction - archive repositories always generate
packfiles per commit
* Include git packv4 ideas:
- metadata packfiles have string dictionary (tree filenames and checksums)
- data packfiles match up similar objects
* Rolling checksums for partitioning large files? Kernel debuginfo
* Improved pack clustering
- file fingerprinting?
* ELF-x86 aware deltas
Related work in storage
-----------------------
git: http://git-scm.com/
Venti: http://plan9.bell-labs.com/magic/man2html/6/venti
Elephant FS: http://www.hpl.hp.com/personal/Alistair_Veitch/papers/elephant-hotos/index.html
Compression
-----------
xdelta: http://xdelta.org/
Bsdiff: http://www.daemonology.net/bsdiff/
xz: http://tukaani.org/xz/

View File

@ -28,6 +28,9 @@
#include <sys/types.h>
#include <attr/xattr.h>
#define ALIGN_VALUE(this, boundary) \
(( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1)))
gboolean
ostree_validate_checksum_string (const char *sha256,
GError **error)
@ -486,6 +489,34 @@ ostree_set_xattrs (GFile *f,
return ret;
}
gboolean
ostree_unwrap_metadata (GVariant *container,
OstreeObjectType expected_type,
GVariant **out_variant,
GError **error)
{
gboolean ret = FALSE;
GVariant *ret_variant = NULL;
guint32 actual_type;
g_variant_get (container, "(uv)",
&actual_type, &ret_variant);
actual_type = GUINT32_FROM_BE (actual_type);
if (actual_type != expected_type)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted metadata object; found type %u, expected %u",
actual_type, (guint32)expected_type);
goto out;
}
ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant);
out:
ot_clear_gvariant (&ret_variant);
return ret;
}
gboolean
ostree_map_metadata_file (GFile *file,
OstreeObjectType expected_type,
@ -495,22 +526,16 @@ ostree_map_metadata_file (GFile *file,
gboolean ret = FALSE;
GVariant *ret_variant = NULL;
GVariant *container = NULL;
guint32 actual_type;
if (!ot_util_variant_map (file, OSTREE_SERIALIZED_VARIANT_FORMAT,
&container, error))
goto out;
g_variant_get (container, "(uv)",
&actual_type, &ret_variant);
ot_util_variant_take_ref (ret_variant);
actual_type = GUINT32_FROM_BE (actual_type);
if (actual_type != expected_type)
if (!ostree_unwrap_metadata (container, expected_type, &ret_variant,
error))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted metadata object '%s'; found type %u, expected %u",
ot_gfile_get_path_cached (file),
actual_type, (guint32)expected_type);
g_prefix_error (error, "While parsing '%s': ",
ot_gfile_get_path_cached (file));
goto out;
}
@ -584,6 +609,142 @@ ostree_object_from_string (const char *str,
*out_objtype = ostree_object_type_from_string (dot + 1);
}
guint
ostree_hash_object_name (gconstpointer a)
{
GVariant *variant = (gpointer)a;
const char *checksum;
OstreeObjectType objtype;
gint objtype_int;
ostree_object_name_deserialize (variant, &checksum, &objtype);
objtype_int = (gint) objtype;
return g_str_hash (checksum) + g_int_hash (&objtype_int);
}
int
ostree_cmp_checksum_bytes (GVariant *a,
GVariant *b)
{
gconstpointer a_data;
gconstpointer b_data;
gsize a_n_elts;
gsize b_n_elts;
a_data = g_variant_get_fixed_array (a, &a_n_elts, 1);
g_assert (a_n_elts == 32);
b_data = g_variant_get_fixed_array (b, &b_n_elts, 1);
g_assert (b_n_elts == 32);
return memcmp (a_data, b_data, 32);
}
GVariant *
ostree_object_name_serialize (const char *checksum,
OstreeObjectType objtype)
{
return g_variant_new ("(su)", checksum, (guint32)objtype);
}
void
ostree_object_name_deserialize (GVariant *variant,
const char **out_checksum,
OstreeObjectType *out_objtype)
{
guint32 objtype_u32;
g_variant_get (variant, "(&su)", out_checksum, &objtype_u32);
*out_objtype = (OstreeObjectType)objtype_u32;
}
GVariant *
ostree_checksum_to_bytes (const char *sha256)
{
guchar result[32];
guint i;
guint j;
for (i = 0, j = 0; i < 32; i += 1, j += 2)
{
gint big, little;
g_assert (sha256[j]);
g_assert (sha256[j+1]);
big = g_ascii_xdigit_value (sha256[j]);
little = g_ascii_xdigit_value (sha256[j+1]);
g_assert (big != -1);
g_assert (little != -1);
result[i] = (big << 4) | little;
}
return g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
(guchar*)result, 32, 1);
}
char *
ostree_checksum_from_bytes (GVariant *csum_bytes)
{
static const gchar hexchars[] = "0123456789abcdef";
char *ret;
const guchar *bytes;
gsize n_elts;
guint i, j;
bytes = g_variant_get_fixed_array (csum_bytes, &n_elts, 1);
g_assert (n_elts == 32);
ret = g_malloc (65);
for (i = 0, j = 0; i < 32; i++, j += 2)
{
guchar byte = bytes[i];
ret[j] = hexchars[byte >> 4];
ret[j+1] = hexchars[byte & 0xF];
}
ret[j] = '\0';
return ret;
}
GVariant *
ostree_object_name_serialize_v2 (const char *checksum,
OstreeObjectType objtype)
{
return g_variant_new ("(u@ay)", (guint32)objtype, ostree_checksum_to_bytes (checksum));
}
void
ostree_object_name_deserialize_v2_hex (GVariant *variant,
char **out_checksum,
OstreeObjectType *out_objtype)
{
GVariant *csum_bytes;
guint32 objtype_u32;
g_variant_get (variant, "(u@ay)", &objtype_u32, &csum_bytes);
g_variant_ref_sink (csum_bytes);
*out_checksum = ostree_checksum_from_bytes (csum_bytes);
g_variant_unref (csum_bytes);
*out_objtype = (OstreeObjectType)objtype_u32;
}
void
ostree_object_name_deserialize_v2_bytes (GVariant *variant,
const guchar **out_checksum,
OstreeObjectType *out_objtype)
{
GVariant *csum_bytes;
guint32 objtype_u32;
gsize n_elts;
g_variant_get (variant, "(u@ay)", &objtype_u32, &csum_bytes);
*out_checksum = (guchar*)g_variant_get_fixed_array (csum_bytes, &n_elts, 1);
*out_objtype = (OstreeObjectType)objtype_u32;
}
char *
ostree_get_relative_object_path (const char *checksum,
OstreeObjectType type)
@ -1074,3 +1235,350 @@ ostree_create_temp_hardlink (GFile *dir,
g_clear_object (&possible_file);
return ret;
}
gboolean
ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 offset,
gboolean trusted,
GVariant **out_entry,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GVariant *ret_entry = NULL;
guint64 entry_start;
guint64 entry_end;
guint32 entry_len;
if (G_UNLIKELY (!(offset <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
offset);
goto out;
}
if (G_UNLIKELY (!((offset & 0x3) == 0)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT,
offset);
goto out;
}
entry_start = ALIGN_VALUE (offset + 4, 8);
if (G_UNLIKELY (!(entry_start <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
entry_start);
goto out;
}
g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
entry_end = entry_start + entry_len;
if (G_UNLIKELY (!(entry_end <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range entry length %u",
entry_len);
goto out;
}
ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT,
pack_data+entry_start, entry_len,
trusted, NULL, NULL);
ret = TRUE;
ot_transfer_out_value (out_entry, &ret_entry);
out:
ot_clear_gvariant (&ret_entry);
return ret;
}
GInputStream *
ostree_read_pack_entry_as_stream (GVariant *pack_entry)
{
GInputStream *memory_input;
GInputStream *ret_input = NULL;
GVariant *pack_data = NULL;
guchar entry_flags;
gconstpointer data_ptr;
gsize data_len;
g_variant_get_child (pack_entry, 1, "y", &entry_flags);
g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
data_ptr = g_variant_get_fixed_array (pack_data, &data_len, 1);
memory_input = g_memory_input_stream_new_from_data (data_ptr, data_len, NULL);
g_object_set_data_full ((GObject*)memory_input, "ostree-mem-gvariant",
pack_data, (GDestroyNotify) g_variant_unref);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
GConverter *decompressor;
decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", decompressor,
"base-stream", memory_input,
"close-base-stream", TRUE,
NULL);
g_object_unref (decompressor);
}
else
{
ret_input = memory_input;
memory_input = NULL;
}
return ret_input;
}
gboolean
ostree_read_pack_entry_variant (GVariant *pack_entry,
OstreeObjectType expected_objtype,
gboolean trusted,
GVariant **out_variant,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GInputStream *stream = NULL;
GVariant *container_variant = NULL;
GVariant *ret_variant = NULL;
guint32 actual_type;
stream = ostree_read_pack_entry_as_stream (pack_entry);
if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT,
trusted, &container_variant, cancellable, error))
goto out;
g_variant_ref_sink (container_variant);
g_variant_get (container_variant, "(uv)",
&actual_type, &ret_variant);
actual_type = GUINT32_FROM_BE (actual_type);
if (actual_type != expected_objtype)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted metadata object in pack file; found type %u, expected %u",
actual_type, (guint32)expected_objtype);
goto out;
}
ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant);
out:
g_clear_object (&stream);
ot_clear_gvariant (&ret_variant);
ot_clear_gvariant (&container_variant);
return ret;
}
gboolean
ostree_pack_index_search (GVariant *index,
GVariant *csum_bytes,
OstreeObjectType objtype,
guint64 *out_offset)
{
gboolean ret = FALSE;
GVariant *index_contents;
gsize imax, imin;
gsize n;
guint32 target_objtype;
index_contents = g_variant_get_child_value (index, 2);
target_objtype = (guint32) objtype;
n = g_variant_n_children (index_contents);
if (n == 0)
goto out;
imax = n - 1;
imin = 0;
while (imax >= imin)
{
GVariant *cur_csum_bytes;
guint32 cur_objtype;
guint64 cur_offset;
gsize imid;
int c;
imid = (imin + imax) / 2;
g_variant_get_child (index_contents, imid, "(u@ayt)", &cur_objtype,
&cur_csum_bytes, &cur_offset);
cur_objtype = GUINT32_FROM_BE (cur_objtype);
c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes);
if (c == 0)
{
if (cur_objtype < target_objtype)
c = -1;
else if (cur_objtype > target_objtype)
c = 1;
}
g_variant_unref (cur_csum_bytes);
if (c < 0)
imin = imid + 1;
else if (c > 0)
{
if (imid == 0)
goto out;
imax = imid - 1;
}
else
{
if (out_offset)
*out_offset = GUINT64_FROM_BE (cur_offset);
ret = TRUE;
goto out;
}
}
out:
ot_clear_gvariant (&index_contents);
return ret;
}
gboolean
ostree_validate_structureof_objtype (guint32 objtype,
GError **error)
{
objtype = GUINT32_FROM_BE (objtype);
if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE
|| objtype > OSTREE_OBJECT_TYPE_COMMIT)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid object type '%u'", objtype);
return FALSE;
}
return TRUE;
}
gboolean
ostree_validate_structureof_checksum (GVariant *checksum,
GError **error)
{
gsize n_children = g_variant_n_children (checksum);
if (n_children != 32)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid checksum of length %" G_GUINT64_FORMAT
" expected 32", (guint64) n_children);
return FALSE;
}
return TRUE;
}
static gboolean
validate_variant (GVariant *variant,
const GVariantType *variant_type,
GError **error)
{
if (!g_variant_is_normal_form (variant))
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Not normal form");
return FALSE;
}
if (!g_variant_is_of_type (variant, variant_type))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Doesn't match variant type '%s'",
(char*)variant_type);
return FALSE;
}
return TRUE;
}
gboolean
ostree_validate_structureof_pack_index (GVariant *index,
GError **error)
{
gboolean ret = FALSE;
const char *header;
GVariantIter *content_iter = NULL;
guint32 objtype;
GVariant *csum_bytes = NULL;
guint64 offset;
if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
goto out;
g_variant_get_child (index, 0, "&s", &header);
if (strcmp (header, "OSTv0PACKINDEX") != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid pack index; doesn't match header");
goto out;
}
g_variant_get_child (index, 2, "a(uayt)", &content_iter);
while (g_variant_iter_loop (content_iter, "(u@ayt)",
&objtype, &csum_bytes, &offset))
{
if (!ostree_validate_structureof_objtype (objtype, error))
goto out;
if (!ostree_validate_structureof_checksum (csum_bytes, error))
goto out;
}
csum_bytes = NULL;
ret = TRUE;
out:
if (content_iter)
g_variant_iter_free (content_iter);
ot_clear_gvariant (&csum_bytes);
return ret;
}
gboolean
ostree_validate_structureof_pack_superindex (GVariant *superindex,
GError **error)
{
gboolean ret = FALSE;
const char *header;
GVariant *csum_bytes = NULL;
GVariant *bloom = NULL;
GVariantIter *content_iter = NULL;
if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
goto out;
g_variant_get_child (superindex, 0, "&s", &header);
if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid pack superindex; doesn't match header");
goto out;
}
g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
while (g_variant_iter_loop (content_iter, "(@ay@ay)",
&csum_bytes, &bloom))
{
if (!ostree_validate_structureof_checksum (csum_bytes, error))
goto out;
}
csum_bytes = NULL;
ret = TRUE;
out:
if (content_iter)
g_variant_iter_free (content_iter);
ot_clear_gvariant (&csum_bytes);
ot_clear_gvariant (&bloom);
return ret;
}

View File

@ -97,9 +97,43 @@ typedef enum {
*/
#define OSTREE_ARCHIVED_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(uuuuusa(ayay))")
/* Pack super index
* s - OSTv0SUPERPACKINDEX
* a{sv} - Metadata
* a(say) - (pack file checksum, bloom filter)
*/
#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))")
/* Pack index
* s - OSTv0PACKINDEX
* a{sv} - Metadata
* a(uayt) - (objtype, checksum, offset into packfile)
*/
#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))")
typedef enum {
OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
} OstreePackFileEntryFlag;
/* Pack files
* s - OSTv0PACKFILE
* a{sv} - Metadata
* t - number of entries
*
* Repeating pair of:
* <padding to alignment of 8>
* ( uyayay ) - objtype, flags, checksum, data
*/
#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)")
#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)")
gboolean ostree_validate_checksum_string (const char *sha256,
GError **error);
GVariant *ostree_checksum_to_bytes (const char *sha256);
gboolean ostree_validate_rev (const char *rev, GError **error);
void ostree_checksum_update_stat (GChecksum *checksum, guint32 uid, guint32 gid, guint32 mode);
@ -108,6 +142,32 @@ const char * ostree_object_type_to_string (OstreeObjectType objtype);
OstreeObjectType ostree_object_type_from_string (const char *str);
guint ostree_hash_object_name (gconstpointer a);
int ostree_cmp_checksum_bytes (GVariant *a, GVariant *b);
GVariant *ostree_object_name_serialize (const char *checksum,
OstreeObjectType objtype);
void ostree_object_name_deserialize (GVariant *variant,
const char **out_checksum,
OstreeObjectType *out_objtype);
GVariant *ostree_object_name_serialize_v2 (const char *checksum,
OstreeObjectType objtype);
void ostree_object_name_deserialize_v2_hex (GVariant *variant,
char **out_checksum,
OstreeObjectType *out_objtype);
void ostree_object_name_deserialize_v2_bytes (GVariant *variant,
const guchar **out_checksum,
OstreeObjectType *out_objtype);
GVariant * ostree_checksum_to_bytes (const char *sha256);
char * ostree_checksum_from_bytes (GVariant *bytes);
char * ostree_object_to_string (const char *checksum,
OstreeObjectType objtype);
@ -123,6 +183,11 @@ GVariant *ostree_get_xattrs_for_file (GFile *f,
GVariant *ostree_wrap_metadata_variant (OstreeObjectType type, GVariant *metadata);
gboolean ostree_unwrap_metadata (GVariant *container,
OstreeObjectType expected_type,
GVariant **out_variant,
GError **error);
gboolean ostree_set_xattrs (GFile *f, GVariant *xattrs,
GCancellable *cancellable, GError **error);
@ -205,5 +270,40 @@ gboolean ostree_parse_archived_file_meta (GVariant *data,
GVariant **out_xattrs,
GError **error);
gboolean ostree_read_pack_entry_raw (guchar *pack_data,
guint64 pack_len,
guint64 object_offset,
gboolean trusted,
GVariant **out_entry,
GCancellable *cancellable,
GError **error);
GInputStream *ostree_read_pack_entry_as_stream (GVariant *pack_entry);
gboolean ostree_read_pack_entry_variant (GVariant *pack_entry,
OstreeObjectType expected_objtype,
gboolean trusted,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_pack_index_search (GVariant *index,
GVariant *csum_bytes,
OstreeObjectType objtype,
guint64 *out_offset);
/** VALIDATION **/
gboolean ostree_validate_structureof_objtype (guint32 objtype,
GError **error);
gboolean ostree_validate_structureof_checksum (GVariant *checksum,
GError **error);
gboolean ostree_validate_structureof_pack_index (GVariant *index,
GError **error);
gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex,
GError **error);
#endif /* _OSTREE_REPO */

View File

@ -724,39 +724,46 @@ bsearch_in_file_variant (GVariant *variant,
const char *name,
int *out_pos)
{
int i, n;
int m;
gsize imax, imin;
gsize imid;
gsize n;
i = 0;
n = g_variant_n_children (variant) - 1;
m = 0;
n = g_variant_n_children (variant);
if (n == 0)
return FALSE;
while (i <= n)
imax = n - 1;
imin = 0;
while (imax >= imin)
{
GVariant *child;
const char *cur;
int cmp;
m = i + ((n - i) / 2);
imid = (imin + imax) / 2;
child = g_variant_get_child_value (variant, m);
child = g_variant_get_child_value (variant, imid);
g_variant_get_child (child, 0, "&s", &cur, NULL);
cmp = strcmp (cur, name);
if (cmp < 0)
i = m + 1;
imin = imid + 1;
else if (cmp > 0)
n = m - 1;
{
if (imid == 0)
break;
imax = imid - 1;
}
else
{
ot_clear_gvariant (&child);
*out_pos = m;
*out_pos = imid;
return TRUE;
}
ot_clear_gvariant (&child);
}
*out_pos = m;
*out_pos = imid;
return FALSE;
}
@ -817,13 +824,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self,
const char *name = NULL;
gboolean ret = FALSE;
GFileInfo *ret_info = NULL;
GFile *archive_data_path = NULL;
GFileInfo *archive_data_info = NULL;
GVariant *archive_metadata = NULL;
GVariant *files_variant = NULL;
GVariant *dirs_variant = NULL;
GVariant *tree_child_metadata = NULL;
GFile *local_child = NULL;
GFileAttributeMatcher *matcher = NULL;
int c;
@ -844,40 +847,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self,
g_variant_get_child (files_variant, n, "(&s&s)", &name, &checksum);
local_child = ostree_repo_get_file_object_path (self->repo, checksum);
if (ostree_repo_get_mode (self->repo) == OSTREE_REPO_MODE_ARCHIVE)
{
if (!ostree_map_metadata_file (local_child, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
&archive_metadata, error))
if (!ostree_repo_load_file (self->repo, checksum, NULL, &ret_info, NULL,
cancellable, error))
goto out;
if (!ostree_parse_archived_file_meta (archive_metadata, &ret_info, NULL, error))
goto out;
archive_data_path = ostree_repo_get_object_path (self->repo, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
archive_data_info = g_file_query_info (archive_data_path,
OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable,
error);
if (!archive_data_info)
goto out;
g_file_info_set_attribute_uint64 (ret_info, "standard::size",
g_file_info_get_attribute_uint64 (archive_data_info,
"standard::size"));
}
else
{
ret_info = g_file_query_info (local_child,
OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable,
error);
if (!ret_info)
goto out;
}
}
else
{
@ -918,12 +890,8 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self,
ot_transfer_out_value(out_info, &ret_info);
out:
g_clear_object (&ret_info);
g_clear_object (&local_child);
g_clear_object (&archive_data_path);
g_clear_object (&archive_data_info);
if (matcher)
g_file_attribute_matcher_unref (matcher);
ot_clear_gvariant (&archive_metadata);
ot_clear_gvariant (&tree_child_metadata);
ot_clear_gvariant (&files_variant);
ot_clear_gvariant (&dirs_variant);

File diff suppressed because it is too large Load Diff

View File

@ -97,6 +97,8 @@ gboolean ostree_repo_find_object (OstreeRepo *self,
const char *checksum,
GFile **out_stored_path,
GFile **out_pending_path,
char **out_pack_checksum,
guint64 *out_pack_offset,
GCancellable *cancellable,
GError **error);
@ -112,6 +114,7 @@ gboolean ostree_repo_stage_object (OstreeRepo *self,
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *content,
@ -141,6 +144,33 @@ gboolean ostree_repo_load_variant (OstreeRepo *self,
GVariant **out_variant,
GError **error);
gboolean ostree_repo_load_pack_index (OstreeRepo *self,
const char *sha256,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_pack_data (OstreeRepo *self,
const char *sha256,
guchar **out_data,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_pack_file (OstreeRepo *self,
const char *sha256,
guchar **out_data,
guint64 *out_len,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_load_file (OstreeRepo *self,
const char *entry_sha256,
GInputStream **out_input,
GFileInfo **out_file_info,
GVariant **out_xattrs,
GCancellable *cancellable,
GError **error);
typedef enum {
OSTREE_REPO_COMMIT_FILTER_ALLOW,
OSTREE_REPO_COMMIT_FILTER_SKIP
@ -200,6 +230,53 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_add_pack_file (OstreeRepo *self,
const char *checksum,
GFile *pack_index_path,
GFile *pack_data_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
const char *remote_name,
GFile *superindex_path,
GPtrArray **out_cached_indexes,
GPtrArray **out_uncached_indexes,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
GVariant **out_variant,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
GFile **out_cached_path,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
const char *remote_name,
const char *pack_checksum,
GFile *cached_path,
GCancellable *cancellable,
GError **error);
typedef enum {
OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
OSTREE_REPO_CHECKOUT_MODE_USER = 1
@ -226,18 +303,37 @@ gboolean ostree_repo_read_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
typedef void (*OstreeRepoObjectIter) (OstreeRepo *self,
const char *checksum,
OstreeObjectType type,
GFile *path,
GFileInfo *fileinfo,
gpointer user_data);
typedef enum {
OSTREE_REPO_LIST_OBJECTS_LOOSE = (1 << 0),
OSTREE_REPO_LIST_OBJECTS_PACKED = (1 << 1),
OSTREE_REPO_LIST_OBJECTS_ALL = (1 << 2)
} OstreeRepoListObjectsFlags;
gboolean ostree_repo_iter_objects (OstreeRepo *self,
OstreeRepoObjectIter callback,
gpointer user_data,
/**
* OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE:
*
* b - %TRUE if object is available "loose"
* as - List of pack file checksums in which this object appears
*/
#define OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE (G_VARIANT_TYPE ("(bas)")
gboolean ostree_repo_list_objects (OstreeRepo *self,
OstreeRepoListObjectsFlags flags,
GHashTable **out_objects,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_indexes,
GCancellable *cancellable,
GError **error);
GFile * ostree_repo_get_pack_index_path (OstreeRepo *self,
const char *checksum);
GFile * ostree_repo_get_pack_data_path (OstreeRepo *self,
const char *checksum);
G_END_DECLS
#endif /* _OSTREE_REPO */

View File

@ -42,10 +42,12 @@ static OstreeBuiltin builtins[] = {
{ "ls", ostree_builtin_ls, 0 },
{ "prune", ostree_builtin_prune, 0 },
{ "fsck", ostree_builtin_fsck, 0 },
{ "pack", ostree_builtin_pack, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "show", ostree_builtin_show, 0 },
{ "unpack", ostree_builtin_unpack, 0 },
{ NULL }
};

File diff suppressed because it is too large Load Diff

View File

@ -26,6 +26,7 @@
#include "ostree.h"
#include <glib/gi18n.h>
#include <glib/gprintf.h>
static gboolean quiet;
static gboolean delete;
@ -38,8 +39,8 @@ static GOptionEntry options[] = {
typedef struct {
OstreeRepo *repo;
guint n_objects;
gboolean had_error;
guint n_loose_objects;
guint n_pack_files;
} OtFsckData;
static gboolean
@ -123,62 +124,157 @@ checksum_archived_file (OtFsckData *data,
return ret;
}
static void
object_iter_callback (OstreeRepo *repo,
static gboolean
fsck_loose_object (OtFsckData *data,
const char *exp_checksum,
OstreeObjectType objtype,
GFile *objf,
GFileInfo *file_info,
gpointer user_data)
GCancellable *cancellable,
GError **error)
{
OtFsckData *data = user_data;
gboolean ret = FALSE;
GFile *objf = NULL;
GChecksum *real_checksum = NULL;
GError *error = NULL;
/* nlinks = g_file_info_get_attribute_uint32 (file_info, "unix::nlink");
if (nlinks < 2 && !quiet)
g_printerr ("note: floating object: %s\n", path); */
objf = ostree_repo_get_object_path (data->repo, exp_checksum, objtype);
if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
{
if (!g_str_has_suffix (ot_gfile_get_path_cached (objf), ".archive-meta"))
{
g_set_error (&error, G_IO_ERROR, G_IO_ERROR_FAILED,
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid archive filename '%s'",
ot_gfile_get_path_cached (objf));
goto out;
}
if (!checksum_archived_file (data, exp_checksum, objf, &real_checksum, &error))
if (!checksum_archived_file (data, exp_checksum, objf, &real_checksum, error))
goto out;
}
else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT)
; /* Handled above */
else
{
if (!ostree_checksum_file (objf, objtype, &real_checksum, NULL, &error))
if (!ostree_checksum_file (objf, objtype, &real_checksum, NULL, error))
goto out;
}
if (real_checksum && strcmp (exp_checksum, g_checksum_get_string (real_checksum)) != 0)
{
data->had_error = TRUE;
g_printerr ("ERROR: corrupted object '%s'; actual checksum: %s\n",
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted loose object '%s'; actual checksum: %s",
ot_gfile_get_path_cached (objf), g_checksum_get_string (real_checksum));
if (delete)
(void) unlink (ot_gfile_get_path_cached (objf));
goto out;
}
data->n_objects++;
data->n_loose_objects++;
ret = TRUE;
out:
ot_clear_checksum (&real_checksum);
if (error != NULL)
return ret;
}
static gboolean
fsck_pack_files (OtFsckData *data,
GCancellable *cancellable,
GError **error)
{
g_printerr ("%s\n", error->message);
g_clear_error (&error);
gboolean ret = FALSE;
GPtrArray *pack_indexes = NULL;
GVariant *index_variant = NULL;
GFile *pack_index_path = NULL;
GFile *pack_data_path = NULL;
GFileInfo *pack_info = NULL;
GInputStream *input = NULL;
GChecksum *pack_content_checksum = NULL;
GVariantIter *index_content_iter = NULL;
guint i;
guint32 objtype;
guint64 offset;
guint64 pack_size;
if (!ostree_repo_list_pack_indexes (data->repo, &pack_indexes, cancellable, error))
goto out;
for (i = 0; i < pack_indexes->len; i++)
{
const char *checksum = pack_indexes->pdata[i];
g_clear_object (&pack_index_path);
pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum);
ot_clear_gvariant (&index_variant);
if (!ot_util_variant_map (pack_index_path,
OSTREE_PACK_INDEX_VARIANT_FORMAT,
&index_variant, error))
goto out;
if (!ostree_validate_structureof_pack_index (index_variant, error))
goto out;
g_clear_object (&pack_data_path);
pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum);
g_clear_object (&input);
input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
if (!input)
goto out;
g_clear_object (&pack_info);
pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
cancellable, error);
if (!pack_info)
goto out;
pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
if (pack_content_checksum)
g_checksum_free (pack_content_checksum);
if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error))
goto out;
if (strcmp (g_checksum_get_string (pack_content_checksum), checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', expected checksum %s",
checksum, g_checksum_get_string (pack_content_checksum));
goto out;
}
g_variant_get_child (index_variant, 2, "a(uayt)", &index_content_iter);
while (g_variant_iter_loop (index_content_iter, "(u@ayt)",
&objtype, NULL, &offset))
{
offset = GUINT64_FROM_BE (offset);
if (offset > pack_size)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
checksum,
offset, pack_size);
goto out;
}
}
data->n_pack_files++;
}
ret = TRUE;
out:
if (index_content_iter)
g_variant_iter_free (index_content_iter);
if (pack_content_checksum)
g_checksum_free (pack_content_checksum);
if (pack_indexes)
g_ptr_array_unref (pack_indexes);
g_clear_object (&pack_info);
g_clear_object (&pack_data_path);
g_clear_object (&input);
return ret;
}
gboolean
ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
{
@ -186,6 +282,10 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
OtFsckData data;
gboolean ret = FALSE;
OstreeRepo *repo = NULL;
GHashTable *objects = NULL;
GCancellable *cancellable = NULL;
GHashTableIter hash_iter;
gpointer key, value;
context = g_option_context_new ("- Check the repository for consistency");
g_option_context_add_main_entries (context, options, NULL);
@ -197,26 +297,47 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_check (repo, error))
goto out;
memset (&data, 0, sizeof (data));
data.repo = repo;
data.n_objects = 0;
data.had_error = FALSE;
if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error))
if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL,
&objects, cancellable, error))
goto out;
if (data.had_error)
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Encountered filesystem consistency errors");
GVariant *serialized_key = key;
GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
gboolean is_loose;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_variant_get_child (objdata, 0, "b", &is_loose);
if (is_loose)
{
if (!fsck_loose_object (&data, checksum, objtype, cancellable, error))
goto out;
}
}
if (!fsck_pack_files (&data, cancellable, error))
goto out;
if (!quiet)
g_printerr ("Total Objects: %u\n", data.n_objects);
g_print ("Loose Objects: %u\n", data.n_loose_objects);
g_print ("Pack files: %u\n", data.n_pack_files);
ret = TRUE;
out:
if (context)
g_option_context_free (context);
g_clear_object (&repo);
if (objects)
g_hash_table_unref (objects);
return ret;
}

View File

@ -45,7 +45,9 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
gboolean ret = FALSE;
GFile *child = NULL;
GFile *grandchild = NULL;
GCancellable *cancellable = NULL;
GString *config_data = NULL;
OstreeRepo *repo = NULL;
context = g_option_context_new ("- Initialize a new empty repository");
g_option_context_add_main_entries (context, options, NULL);
@ -63,38 +65,53 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
NULL, FALSE, 0, NULL,
NULL, error))
goto out;
g_clear_object (&child);
g_clear_object (&child);
child = g_file_get_child (repo_path, "objects");
if (!g_file_make_directory (child, NULL, error))
goto out;
g_clear_object (&child);
g_clear_object (&grandchild);
grandchild = g_file_get_child (child, "pack");
if (!g_file_make_directory (grandchild, NULL, error))
goto out;
g_clear_object (&child);
child = g_file_get_child (repo_path, "tmp");
if (!g_file_make_directory (child, NULL, error))
goto out;
g_clear_object (&child);
g_clear_object (&child);
child = g_file_get_child (repo_path, "refs");
if (!g_file_make_directory (child, NULL, error))
goto out;
g_clear_object (&grandchild);
grandchild = g_file_get_child (child, "heads");
if (!g_file_make_directory (grandchild, NULL, error))
goto out;
g_clear_object (&grandchild);
g_clear_object (&grandchild);
grandchild = g_file_get_child (child, "remotes");
if (!g_file_make_directory (grandchild, NULL, error))
goto out;
g_clear_object (&grandchild);
g_clear_object (&child);
child = g_file_get_child (repo_path, "tags");
if (!g_file_make_directory (child, NULL, error))
goto out;
g_clear_object (&child);
child = g_file_get_child (repo_path, "remote-cache");
if (!g_file_make_directory (child, NULL, error))
goto out;
repo = ostree_repo_new (repo_path);
if (!ostree_repo_check (repo, error))
goto out;
if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
goto out;
ret = TRUE;
out:
@ -104,5 +121,6 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
g_string_free (config_data, TRUE);
g_clear_object (&child);
g_clear_object (&grandchild);
g_clear_object (&repo);
return ret;
}

View File

@ -97,23 +97,29 @@ copy_dir_contents_recurse (GFile *src,
return ret;
}
static void
object_iter_callback (OstreeRepo *repo,
static gboolean
import_loose_object (OtLocalCloneData *data,
const char *checksum,
OstreeObjectType objtype,
GFile *objfile,
GFileInfo *file_info,
gpointer user_data)
GCancellable *cancellable,
GError **error)
{
OtLocalCloneData *data = user_data;
GError *real_error = NULL;
GError **error = &real_error;
gboolean ret = FALSE;
GFile *objfile = NULL;
GFileInfo *file_info = NULL;
GFile *content_path = NULL;
GFileInfo *archive_info = NULL;
GVariant *archive_metadata = NULL;
GVariant *xattrs = NULL;
GInputStream *input = NULL;
objfile = ostree_repo_get_object_path (data->src_repo, checksum, objtype);
file_info = g_file_query_info (objfile, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
if (file_info == NULL)
goto out;
if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE)
xattrs = ostree_get_xattrs_for_file (objfile, error);
@ -121,13 +127,13 @@ object_iter_callback (OstreeRepo *repo,
;
else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
{
if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error))
if (!ostree_repo_load_variant (data->src_repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error))
goto out;
if (!ostree_parse_archived_file_meta (archive_metadata, &archive_info, &xattrs, error))
goto out;
content_path = ostree_repo_get_object_path (repo, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
content_path = ostree_repo_get_object_path (data->src_repo, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
if (g_file_info_get_file_type (archive_info) == G_FILE_TYPE_REGULAR)
{
@ -136,8 +142,8 @@ object_iter_callback (OstreeRepo *repo,
goto out;
}
if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
archive_info, xattrs, input,
if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE,
checksum, FALSE, archive_info, xattrs, input,
NULL, error))
goto out;
}
@ -151,23 +157,21 @@ object_iter_callback (OstreeRepo *repo,
}
if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype, checksum,
file_info, xattrs, input,
FALSE, file_info, xattrs, input,
NULL, error))
goto out;
}
ret = TRUE;
out:
ot_clear_gvariant (&archive_metadata);
ot_clear_gvariant (&xattrs);
g_clear_object (&archive_info);
g_clear_object (&input);
g_clear_object (&content_path);
if (real_error != NULL)
{
g_printerr ("%s\n", real_error->message);
g_clear_error (error);
exit (1);
}
g_clear_object (&file_info);
g_clear_object (&objfile);
return ret;
}
static gboolean
@ -209,6 +213,7 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
{
gboolean ret = FALSE;
GCancellable *cancellable = NULL;
GHashTable *objects = NULL;
GOptionContext *context;
const char *destination;
GFile *dest_f = NULL;
@ -220,6 +225,8 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
GFile *src_dir = NULL;
GFile *dest_dir = NULL;
int i;
GHashTableIter hash_iter;
gpointer key, value;
context = g_option_context_new ("DEST ... - Create new repository DEST");
g_option_context_add_main_entries (context, options, NULL);
@ -266,11 +273,33 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
data.uids_differ = g_file_info_get_attribute_uint32 (src_info, "unix::uid") != g_file_info_get_attribute_uint32 (dest_info, "unix::uid");
if (!ostree_repo_list_objects (data.src_repo, OSTREE_REPO_LIST_OBJECTS_ALL,
&objects, cancellable, error))
goto out;
if (!ostree_repo_prepare_transaction (data.dest_repo, NULL, error))
goto out;
if (!ostree_repo_iter_objects (data.src_repo, object_iter_callback, &data, error))
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
gboolean is_loose;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_variant_get_child (objdata, 0, "b", &is_loose);
if (is_loose)
{
if (!import_loose_object (&data, checksum, objtype, cancellable, error))
goto out;
}
}
if (!ostree_repo_commit_transaction (data.dest_repo, NULL, error))
goto out;
@ -311,5 +340,7 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
g_clear_object (&dest_dir);
g_clear_object (&data.src_repo);
g_clear_object (&data.dest_repo);
if (objects)
g_hash_table_unref (objects);
return ret;
}

View File

@ -0,0 +1,920 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2012 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Author: Colin Walters <walters@verbum.org>
*/
#include "config.h"
#include "ot-builtins.h"
#include "ostree.h"
#include <glib/gi18n.h>
#include <glib/gprintf.h>
#include <gio/gunixinputstream.h>
#include <gio/gunixoutputstream.h>
#define OT_DEFAULT_PACK_SIZE_BYTES (50*1024*1024)
#define OT_GZIP_COMPRESSION_LEVEL (8)
static gboolean opt_analyze_only;
static gboolean opt_reindex_only;
static gboolean opt_keep_loose;
static char* opt_pack_size;
static char* opt_int_compression;
static char* opt_ext_compression;
typedef enum {
OT_COMPRESSION_NONE,
OT_COMPRESSION_GZIP,
OT_COMPRESSION_XZ
} OtCompressionType;
static GOptionEntry options[] = {
{ "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g", "BYTES" },
{ "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" },
{ "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" },
{ "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL },
{ "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL },
{ "keep-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_loose, "Don't delete loose objects", NULL },
{ NULL }
};
typedef struct {
OstreeRepo *repo;
guint64 pack_size;
OtCompressionType int_compression;
OtCompressionType ext_compression;
gboolean had_error;
GError **error;
} OtRepackData;
typedef struct {
GOutputStream *out;
GPtrArray *compressor_argv;
GPid compress_child_pid;
} OtBuildRepackFile;
static gint
compare_object_data_by_size (gconstpointer ap,
gconstpointer bp)
{
GVariant *a = *(void **)ap;
GVariant *b = *(void **)bp;
guint64 a_size;
guint64 b_size;
g_variant_get_child (a, 2, "t", &a_size);
g_variant_get_child (b, 2, "t", &b_size);
if (a == b)
return 0;
else if (a > b)
return 1;
else
return -1;
}
static gboolean
write_bytes_update_checksum (GOutputStream *output,
gconstpointer bytes,
gsize len,
GChecksum *checksum,
guint64 *inout_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gsize bytes_written;
if (len > 0)
{
g_checksum_update (checksum, (guchar*) bytes, len);
if (!g_output_stream_write_all (output, bytes, len, &bytes_written,
cancellable, error))
goto out;
g_assert_cmpint (bytes_written, ==, len);
*inout_offset += bytes_written;
}
ret = TRUE;
out:
return ret;
}
static gboolean
write_padding (GOutputStream *output,
guint alignment,
GChecksum *checksum,
guint64 *inout_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint bits;
guint padding_len;
guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
if (alignment == 8)
bits = ((*inout_offset) & 7);
else
bits = ((*inout_offset) & 3);
if (bits > 0)
{
padding_len = alignment - bits;
if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len,
checksum, inout_offset, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;
}
static gboolean
write_variant_with_size (GOutputStream *output,
GVariant *variant,
GChecksum *checksum,
guint64 *inout_offset,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint64 variant_size;
guint32 variant_size_u32_be;
g_assert ((*inout_offset & 3) == 0);
/* Write variant size */
variant_size = g_variant_get_size (variant);
g_assert (variant_size < G_MAXUINT32);
variant_size_u32_be = GUINT32_TO_BE((guint32) variant_size);
if (!write_bytes_update_checksum (output, (guchar*)&variant_size_u32_be, 4,
checksum, inout_offset, cancellable, error))
goto out;
/* Pad to offset of 8, write variant */
if (!write_padding (output, 8, checksum, inout_offset, cancellable, error))
goto out;
g_assert ((*inout_offset & 7) == 0);
if (!write_bytes_update_checksum (output, g_variant_get_data (variant),
variant_size, checksum,
inout_offset, cancellable, error))
goto out;
ret = TRUE;
out:
return ret;
}
static gint
compare_index_content (gconstpointer ap,
gconstpointer bp)
{
gpointer a = *((gpointer*)ap);
gpointer b = *((gpointer*)bp);
GVariant *a_v = a;
GVariant *b_v = b;
GVariant *a_csum_bytes;
GVariant *b_csum_bytes;
guint32 a_objtype;
guint32 b_objtype;
guint64 a_offset;
guint64 b_offset;
int c;
g_variant_get (a_v, "(u@ayt)", &a_objtype, &a_csum_bytes, &a_offset);
g_variant_get (b_v, "(u@ayt)", &b_objtype, &b_csum_bytes, &b_offset);
a_objtype = GUINT32_FROM_BE (a_objtype);
b_objtype = GUINT32_FROM_BE (b_objtype);
c = ostree_cmp_checksum_bytes (a_csum_bytes, b_csum_bytes);
if (c == 0)
{
if (a_objtype < b_objtype)
c = -1;
else if (a_objtype > b_objtype)
c = 1;
}
return c;
}
static gboolean
delete_loose_object (OtRepackData *data,
const char *checksum,
OstreeObjectType objtype,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GFile *object_path = NULL;
GFile *content_object_path = NULL;
GVariant *archive_meta = NULL;
GFileInfo *file_info = NULL;
GVariant *xattrs = NULL;
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
/* This is gross - we need to specially clean up symbolic link object content */
if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
{
if (!ostree_map_metadata_file (object_path, objtype, &archive_meta, error))
goto out;
if (!ostree_parse_archived_file_meta (archive_meta, &file_info, &xattrs, error))
goto out;
if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR)
{
content_object_path = ostree_repo_get_object_path (data->repo, checksum,
OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
if (!ot_gfile_unlink (content_object_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete archived content '%s'",
ot_gfile_get_path_cached (content_object_path));
goto out;
}
}
}
if (!ot_gfile_unlink (object_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete archived file metadata '%s'",
ot_gfile_get_path_cached (object_path));
goto out;
}
ret = TRUE;
out:
g_clear_object (&object_path);
g_clear_object (&content_object_path);
ot_clear_gvariant (&archive_meta);
g_clear_object (&file_info);
ot_clear_gvariant (&xattrs);
return ret;
}
static gboolean
create_pack_file (OtRepackData *data,
GPtrArray *objects,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GFile *pack_dir = NULL;
GFile *index_temppath = NULL;
GOutputStream *index_out = NULL;
GFile *pack_temppath = NULL;
GOutputStream *pack_out = NULL;
GFile *object_path = NULL;
GFileInfo *object_file_info = NULL;
GFileInputStream *object_input = NULL;
GConverter *compressor = NULL;
GConverterInputStream *compressed_object_input = NULL;
guint i;
guint64 offset;
gsize bytes_written;
GPtrArray *index_content_list = NULL;
GVariant *pack_header = NULL;
GVariant *packed_object = NULL;
GVariant *index_content = NULL;
GVariantBuilder index_content_builder;
GChecksum *pack_checksum = NULL;
char *pack_name = NULL;
GFile *pack_file_path = NULL;
GFile *pack_index_path = NULL;
GMemoryOutputStream *object_data_stream = NULL;
if (g_cancellable_set_error_if_cancelled (cancellable, error))
return FALSE;
if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
"pack-index", NULL,
&index_temppath,
&index_out,
cancellable, error))
goto out;
if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
"pack-content", NULL,
&pack_temppath,
&pack_out,
cancellable, error))
goto out;
index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
offset = 0;
pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
pack_header = g_variant_new ("(s@a{sv}t)",
"OSTv0PACKFILE",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
(guint64)objects->len);
if (!write_variant_with_size (pack_out, pack_header, pack_checksum, &offset,
cancellable, error))
goto out;
for (i = 0; i < objects->len; i++)
{
GVariant *object_data = objects->pdata[i];
const char *checksum;
guint32 objtype_u32;
OstreeObjectType objtype;
guint64 expected_objsize;
guint64 objsize;
GInputStream *read_object_in;
guchar entry_flags = 0;
GVariant *index_entry;
g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32;
switch (data->int_compression)
{
case OT_COMPRESSION_GZIP:
{
entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
break;
}
default:
{
g_assert_not_reached ();
}
}
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_input);
object_input = g_file_read (object_path, cancellable, error);
if (!object_input)
goto out;
g_clear_object (&object_file_info);
object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
if (!object_file_info)
goto out;
objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_assert_cmpint (objsize, ==, expected_objsize);
g_clear_object (&object_data_stream);
object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
{
g_clear_object (&compressor);
compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
g_clear_object (&compressed_object_input);
compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
"converter", compressor,
"base-stream", object_input,
"close-base-stream", TRUE,
NULL);
read_object_in = (GInputStream*)compressed_object_input;
}
else
{
read_object_in = (GInputStream*)object_input;
}
if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error))
goto out;
ot_clear_gvariant (&packed_object);
{
guchar *data = g_memory_output_stream_get_data (object_data_stream);
gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
packed_object = g_variant_new ("(uy@ay@ay)", GUINT32_TO_BE ((guint32)objtype),
entry_flags,
ostree_checksum_to_bytes (checksum),
g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
data, data_len,
1));
g_clear_object (&object_data_stream);
}
if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
goto out;
/* offset points to aligned header size */
index_entry = g_variant_new ("(u@ayt)",
GUINT32_TO_BE ((guint32)objtype),
ostree_checksum_to_bytes (checksum),
GUINT64_TO_BE (offset));
g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
if (!write_variant_with_size (pack_out, packed_object, pack_checksum,
&offset, cancellable, error))
goto out;
}
if (!g_output_stream_close (pack_out, cancellable, error))
goto out;
g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(uayt)"));
g_ptr_array_sort (index_content_list, compare_index_content);
for (i = 0; i < index_content_list->len; i++)
{
GVariant *index_item = index_content_list->pdata[i];
g_variant_builder_add_value (&index_content_builder, index_item);
}
index_content = g_variant_new ("(s@a{sv}@a(uayt))",
"OSTv0PACKINDEX",
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
g_variant_builder_end (&index_content_builder));
if (!g_output_stream_write_all (index_out,
g_variant_get_data (index_content),
g_variant_get_size (index_content),
&bytes_written,
cancellable,
error))
goto out;
if (!g_output_stream_close (index_out, cancellable, error))
goto out;
if (!ostree_repo_add_pack_file (data->repo,
g_checksum_get_string (pack_checksum),
index_temppath,
pack_temppath,
cancellable,
error))
goto out;
if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error))
goto out;
g_print ("Created pack file '%s' with %u objects\n", g_checksum_get_string (pack_checksum), objects->len);
if (!opt_keep_loose)
{
for (i = 0; i < objects->len; i++)
{
GVariant *object_data = objects->pdata[i];
const char *checksum;
guint32 objtype_u32;
OstreeObjectType objtype;
guint64 expected_objsize;
g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32;
if (!delete_loose_object (data, checksum, objtype, cancellable, error))
goto out;
}
}
ret = TRUE;
out:
if (index_temppath)
(void) unlink (ot_gfile_get_path_cached (index_temppath));
g_clear_object (&index_temppath);
g_clear_object (&index_out);
if (pack_temppath)
(void) unlink (ot_gfile_get_path_cached (pack_temppath));
g_clear_object (&pack_temppath);
g_clear_object (&pack_out);
g_clear_object (&object_path);
g_clear_object (&object_input);
g_clear_object (&compressor);
g_clear_object (&compressed_object_input);
g_clear_object (&object_file_info);
if (pack_checksum)
g_checksum_free (pack_checksum);
g_clear_object (&pack_dir);
ot_clear_gvariant (&index_content);
g_free (pack_name);
g_clear_object (&pack_file_path);
g_clear_object (&pack_index_path);
if (index_content_list)
g_ptr_array_unref (index_content_list);
return ret;
}
/**
* cluster_objects_stupidly:
* @objects: Map from serialized object name to objdata
* @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
*
* Just sorts by size currently. Also filters out non-regular object
* content.
*/
static gboolean
cluster_objects_stupidly (OtRepackData *data,
GHashTable *objects,
GPtrArray **out_clusters,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GPtrArray *ret_clusters = NULL;
GPtrArray *object_list = NULL;
guint i;
guint64 current_size;
guint current_offset;
GHashTableIter hash_iter;
gpointer key, value;
GFile *object_path = NULL;
GFileInfo *object_info = NULL;
object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
g_clear_object (&object_info);
object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!object_info)
goto out;
if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
continue;
size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_ptr_array_add (object_list,
g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)));
}
g_ptr_array_sort (object_list, compare_object_data_by_size);
ret_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
current_size = 0;
current_offset = 0;
for (i = 0; i < object_list->len; i++)
{
GVariant *objdata = object_list->pdata[i];
guint64 objsize;
g_variant_get_child (objdata, 2, "t", &objsize);
if (current_size + objsize > data->pack_size || i == (object_list->len - 1))
{
guint j;
GPtrArray *current;
if (current_offset < i)
{
current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
for (j = current_offset; j <= i; j++)
{
g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
}
g_ptr_array_add (ret_clusters, current);
current_size = objsize;
current_offset = i+1;
}
}
else if (objsize > data->pack_size)
{
break;
}
else
{
current_size += objsize;
}
}
ret = TRUE;
ot_transfer_out_value (out_clusters, &ret_clusters);
out:
if (object_list)
g_ptr_array_unref (object_list);
return ret;
}
static gboolean
parse_size_spec_with_suffix (const char *spec,
guint64 default_value,
guint64 *out_size,
GError **error)
{
gboolean ret = FALSE;
char *endptr = NULL;
guint64 ret_size;
if (spec == NULL)
{
ret_size = default_value;
endptr = NULL;
}
else
{
ret_size = g_ascii_strtoull (spec, &endptr, 10);
if (endptr && *endptr)
{
char suffix = *endptr;
switch (suffix)
{
case 'k':
case 'K':
{
ret_size *= 1024;
break;
}
case 'm':
case 'M':
{
ret_size *= (1024 * 1024);
break;
}
case 'g':
case 'G':
{
ret_size *= (1024 * 1024 * 1024);
break;
}
default:
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid size suffix '%c'", suffix);
goto out;
}
}
}
ret = TRUE;
*out_size = ret_size;
out:
return ret;
}
static gboolean
parse_compression_string (const char *compstr,
OtCompressionType *out_comptype,
GError **error)
{
gboolean ret = FALSE;
OtCompressionType ret_comptype;
if (compstr == NULL)
ret_comptype = OT_COMPRESSION_NONE;
else if (strcmp (compstr, "gzip") == 0)
ret_comptype = OT_COMPRESSION_GZIP;
else if (strcmp (compstr, "xz") == 0)
ret_comptype = OT_COMPRESSION_XZ;
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid compression '%s'", compstr);
goto out;
}
ret = TRUE;
*out_comptype = ret_comptype;
out:
return ret;
}
static gboolean
do_stats_gather_loose (OtRepackData *data,
GHashTable *objects,
GHashTable **out_loose,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTable *ret_loose = NULL;
guint n_loose = 0;
guint n_loose_and_packed = 0;
guint n_packed = 0;
guint n_dup_packed = 0;
guint n_commits = 0;
guint n_dirmeta = 0;
guint n_dirtree = 0;
guint n_files = 0;
GHashTableIter hash_iter;
gpointer key, value;
ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify) g_variant_unref,
NULL);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
gboolean is_loose;
gboolean is_packed;
GVariant *pack_array;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_variant_get (objdata, "(b@as)", &is_loose, &pack_array);
is_packed = g_variant_n_children (pack_array) > 0;
if (is_loose && is_packed)
{
n_loose_and_packed++;
}
else if (is_loose)
{
GVariant *copy = g_variant_ref (serialized_key);
g_hash_table_replace (ret_loose, copy, copy);
n_loose++;
}
else if (g_variant_n_children (pack_array) > 1)
{
n_dup_packed++;
}
else
{
n_packed++;
}
switch (objtype)
{
case OSTREE_OBJECT_TYPE_COMMIT:
n_commits++;
break;
case OSTREE_OBJECT_TYPE_DIR_TREE:
n_dirtree++;
break;
case OSTREE_OBJECT_TYPE_DIR_META:
n_dirmeta++;
break;
case OSTREE_OBJECT_TYPE_RAW_FILE:
case OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META:
n_files++;
break;
case OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT:
/* Counted under files by META */
break;
}
}
g_print ("Commits: %u\n", n_commits);
g_print ("Tree contents: %u\n", n_dirtree);
g_print ("Tree meta: %u\n", n_dirmeta);
g_print ("Files: %u\n", n_files);
g_print ("\n");
g_print ("Loose+packed objects: %u\n", n_loose_and_packed);
g_print ("Loose-only objects: %u\n", n_loose);
g_print ("Duplicate packed objects: %u\n", n_dup_packed);
g_print ("Packed-only objects: %u\n", n_packed);
ret = TRUE;
ot_transfer_out_value (out_loose, &ret_loose);
/* out: */
if (ret_loose)
g_hash_table_unref (ret_loose);
return ret;
}
static gboolean
do_incremental_pack (OtRepackData *data,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTable *objects = NULL;
guint i;
GPtrArray *clusters = NULL;
GHashTable *loose_objects = NULL;
if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
cancellable, error))
goto out;
if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error))
goto out;
g_print ("\n");
g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size);
if (!cluster_objects_stupidly (data, loose_objects, &clusters, cancellable, error))
goto out;
if (clusters->len > 0)
g_print ("Going to create %u packfiles\n", clusters->len);
else
g_print ("Nothing to do\n");
for (i = 0; i < clusters->len; i++)
{
GPtrArray *cluster = clusters->pdata[i];
if (!opt_analyze_only)
{
if (!create_pack_file (data, cluster, cancellable, error))
goto out;
}
}
ret = TRUE;
out:
if (clusters)
g_ptr_array_unref (clusters);
if (loose_objects)
g_hash_table_unref (loose_objects);
if (objects)
g_hash_table_unref (objects);
return ret;
}
gboolean
ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error)
{
gboolean ret = FALSE;
GOptionContext *context;
OtRepackData data;
OstreeRepo *repo = NULL;
GCancellable *cancellable = NULL;
memset (&data, 0, sizeof (data));
context = g_option_context_new ("- Recompress objects");
g_option_context_add_main_entries (context, options, NULL);
if (!g_option_context_parse (context, &argc, &argv, error))
goto out;
repo = ostree_repo_new (repo_path);
if (!ostree_repo_check (repo, error))
goto out;
if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"Can't repack bare repositories yet");
goto out;
}
data.repo = repo;
data.error = error;
if (!parse_size_spec_with_suffix (opt_pack_size, OT_DEFAULT_PACK_SIZE_BYTES, &data.pack_size, error))
goto out;
/* Default internal compression to gzip */
if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error))
goto out;
if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error))
goto out;
if (opt_reindex_only)
{
if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
goto out;
}
else
{
if (!do_incremental_pack (&data, cancellable, error))
goto out;
}
ret = TRUE;
out:
if (context)
g_option_context_free (context);
g_clear_object (&repo);
return ret;
}

View File

@ -187,24 +187,27 @@ compute_reachable_objects_from_commit (OstreeRepo *repo,
return ret;
}
static void
object_iter_callback (OstreeRepo *repo,
static gboolean
prune_loose_object (OtPruneData *data,
const char *checksum,
OstreeObjectType objtype,
GFile *objf,
GFileInfo *file_info,
gpointer user_data)
GCancellable *cancellable,
GError **error)
{
OtPruneData *data = user_data;
gboolean ret = FALSE;
char *key;
GFile *objf = NULL;
key = ostree_object_to_string (checksum, objtype);
objf = ostree_repo_get_object_path (data->repo, checksum, objtype);
if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL))
{
if (delete)
{
(void) unlink (ot_gfile_get_path_cached (objf));
if (!g_file_delete (objf, cancellable, error))
goto out;
g_print ("Deleted: %s\n", key);
}
else
@ -216,16 +219,21 @@ object_iter_callback (OstreeRepo *repo,
else
data->n_reachable++;
ret = TRUE;
out:
g_clear_object (&objf);
g_free (key);
return ret;
}
gboolean
ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error)
{
gboolean ret = FALSE;
GOptionContext *context;
OtPruneData data;
gboolean ret = FALSE;
GHashTable *objects = NULL;
OstreeRepo *repo = NULL;
GHashTable *all_refs = NULL;
GHashTableIter hash_iter;
@ -266,11 +274,37 @@ ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error)
goto out;
}
g_hash_table_iter_init (&hash_iter, data.reachable);
if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error))
if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
goto out;
g_hash_table_iter_init (&hash_iter, objects);
if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL,
&objects, cancellable, error))
goto out;
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
gboolean is_loose;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
g_variant_get_child (objdata, 0, "b", &is_loose);
if (is_loose)
{
if (!prune_loose_object (&data, checksum, objtype, cancellable, error))
goto out;
}
}
if (data.had_error)
goto out;
@ -286,5 +320,7 @@ ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error)
if (context)
g_option_context_free (context);
g_clear_object (&repo);
if (objects)
g_hash_table_unref (objects);
return ret;
}

View File

@ -0,0 +1,305 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2012 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Author: Colin Walters <walters@verbum.org>
*/
#include "config.h"
#include "ot-builtins.h"
#include "ostree.h"
#include <glib/gi18n.h>
#include <glib/gprintf.h>
#include <gio/gunixinputstream.h>
#include <gio/gunixoutputstream.h>
static GOptionEntry options[] = {
{ NULL }
};
typedef struct {
OstreeRepo *repo;
} OtUnpackData;
static gboolean
gather_packed (OtUnpackData *data,
GHashTable *objects,
GHashTable **out_packed,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GHashTable *ret_packed = NULL;
GHashTableIter hash_iter;
gpointer key, value;
GVariant *pack_array = NULL;
ret_packed = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify) g_variant_unref,
NULL);
g_hash_table_iter_init (&hash_iter, objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
GVariant *key_copy;
GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
gboolean is_loose;
gboolean is_packed;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
ot_clear_gvariant (&pack_array);
g_variant_get (objdata, "(b@as)", &is_loose, &pack_array);
is_packed = g_variant_n_children (pack_array) > 0;
if (is_loose)
continue;
g_assert (is_packed);
key_copy = g_variant_ref (serialized_key);
g_hash_table_replace (ret_packed, key_copy, key_copy);
}
ret = TRUE;
ot_transfer_out_value (out_packed, &ret_packed);
/* out: */
ot_clear_gvariant (&pack_array);
if (ret_packed)
g_hash_table_unref (ret_packed);
return ret;
}
static gboolean
unpack_one_object (OstreeRepo *repo,
const char *checksum,
OstreeObjectType objtype,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GInputStream *input = NULL;
GFileInfo *file_info = NULL;
GVariant *xattrs = NULL;
GVariant *meta = NULL;
GVariant *serialized_meta = NULL;
g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
{
if (!ostree_repo_load_file (repo, checksum,
&input, &file_info, &xattrs,
cancellable, error))
goto out;
if (!ostree_repo_stage_object_trusted (repo, OSTREE_OBJECT_TYPE_RAW_FILE,
checksum, TRUE, file_info, xattrs, input,
cancellable, error))
goto out;
}
else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT)
{
/* nothing; handled in META case */
}
else
{
if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error))
goto out;
serialized_meta = ostree_wrap_metadata_variant (objtype, meta);
input = g_memory_input_stream_new_from_data (g_variant_get_data (serialized_meta),
g_variant_get_size (serialized_meta), NULL);
if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE,
NULL, NULL, input, cancellable, error))
goto out;
}
ret = TRUE;
out:
g_clear_object (&input);
g_clear_object (&file_info);
ot_clear_gvariant (&xattrs);
ot_clear_gvariant (&meta);
ot_clear_gvariant (&serialized_meta);
return ret;
}
static gboolean
delete_one_packfile (OstreeRepo *repo,
const char *pack_checksum,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GFile *data_path = NULL;
GFile *index_path = NULL;
index_path = ostree_repo_get_pack_index_path (repo, pack_checksum);
data_path = ostree_repo_get_pack_data_path (repo, pack_checksum);
if (!ot_gfile_unlink (index_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path));
goto out;
}
if (!ot_gfile_unlink (data_path, cancellable, error))
{
g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path));
goto out;
}
ret = TRUE;
out:
g_clear_object (&index_path);
g_clear_object (&data_path);
return ret;
}
gboolean
ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
{
gboolean ret = FALSE;
GOptionContext *context;
gboolean in_transaction = FALSE;
OtUnpackData data;
OstreeRepo *repo = NULL;
GHashTable *objects = NULL;
GCancellable *cancellable = NULL;
GPtrArray *clusters = NULL;
GHashTable *packed_objects = NULL;
GHashTableIter hash_iter;
GHashTable *packfiles_to_delete = NULL;
gpointer key, value;
GFile *objpath = NULL;
guint64 unpacked_object_count = 0;
memset (&data, 0, sizeof (data));
context = g_option_context_new ("- Uncompress objects");
g_option_context_add_main_entries (context, options, NULL);
if (!g_option_context_parse (context, &argc, &argv, error))
goto out;
repo = ostree_repo_new (repo_path);
if (!ostree_repo_check (repo, error))
goto out;
if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"Can't unpack bare repositories yet");
goto out;
}
data.repo = repo;
if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
goto out;
if (!gather_packed (&data, objects, &packed_objects, cancellable, error))
goto out;
if (!ostree_repo_prepare_transaction (repo, cancellable, error))
goto out;
in_transaction = TRUE;
packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_iter_init (&hash_iter, packed_objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *objkey = key;
GVariant *objdata;
const char *checksum;
const char *pack_checksum;
OstreeObjectType objtype;
gboolean is_loose;
GVariantIter *pack_array_iter;
objdata = g_hash_table_lookup (objects, objkey);
g_assert (objdata);
g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter);
g_assert (!is_loose);
while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
{
if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum))
{
gchar *duped_checksum = g_strdup (pack_checksum);
g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum);
}
}
g_variant_iter_free (pack_array_iter);
ostree_object_name_deserialize (objkey, &checksum, &objtype);
if (!unpack_one_object (repo, checksum, objtype, cancellable, error))
goto out;
unpacked_object_count++;
}
if (!ostree_repo_commit_transaction (repo, cancellable, error))
goto out;
if (g_hash_table_size (packfiles_to_delete) == 0)
g_print ("No pack files; nothing to do\n");
g_hash_table_iter_init (&hash_iter, packfiles_to_delete);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *pack_checksum = key;
if (!delete_one_packfile (repo, pack_checksum, cancellable, error))
goto out;
g_print ("Deleted packfile '%s'\n", pack_checksum);
}
ret = TRUE;
out:
if (in_transaction)
(void) ostree_repo_abort_transaction (repo, cancellable, NULL);
g_clear_object (&objpath);
if (context)
g_option_context_free (context);
g_clear_object (&repo);
if (clusters)
g_ptr_array_unref (clusters);
if (packfiles_to_delete)
g_hash_table_unref (packfiles_to_delete);
if (packed_objects)
g_hash_table_unref (packed_objects);
if (objects)
g_hash_table_unref (objects);
return ret;
}

View File

@ -40,8 +40,10 @@ gboolean ostree_builtin_ls (int argc, char **argv, GFile *repo_path, GError **er
gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error);
G_END_DECLS

View File

@ -21,7 +21,7 @@ set -e
. libtest.sh
echo '1..10'
echo '1..19'
setup_test_repository "archive"
echo "ok setup"
@ -67,3 +67,36 @@ cd ${test_tmpdir}
$OSTREE cat test2 /baz/cow > cow-contents
assert_file_has_content cow-contents "moo"
echo "ok cat-file"
cd ${test_tmpdir}
$OSTREE pack --keep-loose
echo "ok pack"
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
$OSTREE checkout test2 checkout-test2-from-packed
echo "ok checkout union 1"
cd ${test_tmpdir}
$OSTREE pack
echo "ok pack delete loose"
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
$OSTREE pack --analyze-only
echo "ok pack analyze"
$OSTREE unpack
echo "ok unpack"
cd ${test_tmpdir}
$OSTREE fsck
echo "ok fsck"
cd ${test_tmpdir}
$OSTREE checkout test2 checkout-test2-from-unpacked
echo "ok checkout union 2"

View File

@ -21,7 +21,7 @@ set -e
. libtest.sh
echo '1..2'
echo '1..4'
setup_fake_remote_repo1
cd ${test_tmpdir}
@ -29,6 +29,7 @@ mkdir repo
ostree --repo=repo init
ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
ostree-pull --repo=repo origin main
ostree --repo=repo fsck
echo "ok pull"
cd ${test_tmpdir}
@ -37,3 +38,21 @@ cd checkout-origin-main
assert_file_has_content firstfile '^first$'
assert_file_has_content baz/cow '^moo$'
echo "ok pull contents"
cd ${test_tmpdir}
ostree --repo=$(pwd)/ostree-srv/gnomerepo pack
rm -rf repo
mkdir repo
ostree --repo=repo init
ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
ostree-pull --repo=repo origin main
ostree --repo=repo fsck
echo "ok pull packed"
cd ${test_tmpdir}
rm -rf checkout-origin-main
$OSTREE checkout origin/main checkout-origin-main
cd checkout-origin-main
assert_file_has_content firstfile '^first$'
assert_file_has_content baz/cow '^moo$'
echo "ok pull contents packed"