shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-13 00:58:43 +03:00
Code Issues Projects Releases Wiki Activity

deltas: Use base64 for csums, add version to parts

Browse Source
This commit is contained in:
Colin Walters 2015-01-25 23:51:41 -05:00
parent ee4e393fa1
commit 92cc3b5968
8 changed files with 133 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
src/libostree/ostree-core.c
View File

@ -1103,6 +1103,35 @@ ostree_object_name_deserialize (GVariant *variant,
*out_objtype = (OstreeObjectType)objtype_u32;
}
/**
* ostree_checksum_b64_inplace_to_bytes: (skip)
* @checksum: (array fixed-size=32): An binary checksum of length 32
* @buf: Output location, must be at least 45 bytes in length
*
* Overwrite the contents of @buf with stringified version of @csum.
*/
void
ostree_checksum_b64_inplace_to_bytes (const char *checksum,
guchar *buf)
{
int state = 0;
guint save = 0;
char tmpbuf[44];
int i;
for (i = 0; i < 43; i++)
{
char c = checksum[i];
if (c == '_')
tmpbuf[i] = '/';
else
tmpbuf[i] = c;
}
tmpbuf[43] = '=';
g_base64_decode_step (tmpbuf, sizeof (tmpbuf), (guchar *) buf, &state, &save);
}
/**
* ostree_checksum_inplace_to_bytes:
* @checksum: a SHA256 string
@ -1186,6 +1215,48 @@ ostree_checksum_inplace_from_bytes (const guchar *csum,
buf[j] = '\0';
}
/**
* ostree_checksum_b64_inplace_from_bytes: (skip)
* @csum: (array fixed-size=32): An binary checksum of length 32
* @buf: Output location, must be at least 44 bytes in length
*
* Overwrite the contents of @buf with modified base64 encoding of @csum.
* The "modified" term refers to the fact that instead of '/', the '_'
* character is used.
*/
void
ostree_checksum_b64_inplace_from_bytes (const guchar *csum,
char *buf)
{
char tmpbuf[44];
int save = 0;
int state = 0;
gsize outlen;
int i;
/* At some point, we can optimize this, but for now it's
* a lot easier to reuse GLib's base64 encoder and postprocess it
* to replace the '/' with '_'.
*/
outlen = g_base64_encode_step (csum, 32, FALSE, tmpbuf, &state, &save);
outlen += g_base64_encode_close (FALSE, tmpbuf+outlen, &state, &save);
g_assert (outlen == 44);
for (i = 0; i < sizeof (tmpbuf); i++)
{
char c = tmpbuf[i];
if (c == '=')
{
g_assert (i == 43);
buf[i] = '\0';
}
else if (c == '/')
buf[i] = '_';
else
buf[i] = c;
}
}
/**
* ostree_checksum_from_bytes:
* @csum: (array fixed-size=32): An binary checksum of length 32
@ -1360,21 +1431,35 @@ get_delta_path (const char *from,
const char *target)
{
char prefix[3];
guint8 csum_to[32];
char to_b64[44];
guint8 csum_to_copy[32];
ostree_checksum_inplace_to_bytes (to, csum_to);
ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy);
g_assert (memcmp (csum_to, csum_to_copy, 32) == 0);
if (from == NULL)
{
prefix[0] = to[0];
prefix[1] = to[1];
prefix[0] = to_b64[0];
prefix[1] = to_b64[1];
prefix[2] = '\0';
to += 2;
return g_strconcat ("deltas/", prefix, "/", to, "/", target, NULL);
return g_strconcat ("deltas/", prefix, "/", ((char*)to_b64)+2, "/", target, NULL);
}
else
{
prefix[0] = from[0];
prefix[1] = from[1];
guint8 csum_from[32];
char from_b64[44];
ostree_checksum_inplace_to_bytes (from, csum_from);
ostree_checksum_b64_inplace_from_bytes (csum_from, from_b64);
prefix[0] = from_b64[0];
prefix[1] = from_b64[1];
prefix[2] = '\0';
from += 2;
return g_strconcat ("deltas/", prefix, "/", from, "-", to, "/", target, NULL);
return g_strconcat ("deltas/", prefix, "/", ((char*)from_b64)+2, "-", to_b64, "/", target, NULL);
}
}

4
src/libostree/ostree-core.h
View File

@ -164,12 +164,16 @@ gboolean ostree_validate_checksum_string (const char *sha256,
guchar *ostree_checksum_to_bytes (const char *checksum);
GVariant *ostree_checksum_to_bytes_v (const char *checksum);
void ostree_checksum_b64_inplace_to_bytes (const char *checksum,
guint8 *buf);
char * ostree_checksum_from_bytes (const guchar *csum);
char * ostree_checksum_from_bytes_v (GVariant *csum_v);
void ostree_checksum_inplace_from_bytes (const guchar *csum,
char *buf);
void ostree_checksum_b64_inplace_from_bytes (const guchar *csum,
char *buf);
void ostree_checksum_inplace_to_bytes (const char *checksum,
guchar *buf);

2
src/libostree/ostree-repo-private.h
View File

@ -24,6 +24,8 @@
G_BEGIN_DECLS
#define OSTREE_DELTAPART_VERSION (0)
#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay"
/**

10
src/libostree/ostree-repo-pull.c
View File

@ -1513,9 +1513,17 @@ process_one_static_delta (OtPullData *pull_data,
gs_unref_variant GVariant *csum_v = NULL;
gs_unref_variant GVariant *objects = NULL;
guint64 size, usize;
guint32 version;
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(@aytt@ay)", &csum_v, &size, &usize, &objects);
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
if (version > OSTREE_DELTAPART_VERSION)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Delta part has too new version %u", version);
goto out;
}
csum = ostree_checksum_bytes_peek_validate (csum_v, error);
if (!csum)

13
src/libostree/ostree-repo-static-delta-compilation.c
View File

@ -548,7 +548,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
checksum_bytes = g_bytes_new (part_checksum, 32);
objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
delta_part_header = g_variant_new ("(@aytt@ay)",
delta_part_header = g_variant_new ("(u@aytt@ay)",
OSTREE_DELTAPART_VERSION,
ot_gvariant_new_ay_bytes (checksum_bytes),
g_variant_get_size (delta_part),
part_builder->uncompressed_size,
@ -586,13 +587,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
metadata_source = metadata;
else
{
GVariantBuilder tmpbuilder;
g_variant_builder_init (&tmpbuilder, G_VARIANT_TYPE ("(a(ss)a(say))"));
g_variant_builder_add (&tmpbuilder, "a(ss)", NULL);
g_variant_builder_add (&tmpbuilder, "a(say)", NULL);
tmp_metadata = g_variant_builder_end (&tmpbuilder);
g_variant_ref_sink (tmp_metadata);
metadata_source = tmp_metadata;
metadata_source = ot_gvariant_new_empty_string_dict ();
}
if (!get_fallback_headers (self, &builder, &fallback_headers,
@ -607,7 +602,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
/* floating */ GVariant *to_csum_v =
ostree_checksum_to_bytes_v (to);
delta_descriptor = g_variant_new ("(@(a(ss)a(say))t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
delta_descriptor = g_variant_new ("(@a{sv}t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
"a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
"@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
metadata_source,

19
src/libostree/ostree-repo-static-delta-core.c
View File

@ -127,7 +127,24 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
if (g_file_query_exists (meta_path, NULL))
{
g_ptr_array_add (ret_deltas, g_strconcat (name1, name2, NULL));
gs_free char *buf = g_strconcat (name1, name2, NULL);
GString *out = g_string_new ("");
char checksum[65];
guchar csum[32];
const char *dash = strchr (buf, '-');
ostree_checksum_b64_inplace_to_bytes (buf, csum);
ostree_checksum_inplace_from_bytes (csum, checksum);
g_string_append (out, checksum);
if (dash)
{
g_string_append_c (out, '-');
ostree_checksum_b64_inplace_to_bytes (dash+1, csum);
ostree_checksum_inplace_from_bytes (csum, checksum);
g_string_append (out, checksum);
}
g_ptr_array_add (ret_deltas, g_string_free (out, FALSE));
}
}
}

7
src/libostree/ostree-repo-static-delta-private.h
View File

@ -42,6 +42,7 @@ G_BEGIN_DECLS
/**
* OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
*
* u: version
* ay checksum
* guint64 size: Total size of delta (sum of parts)
* guint64 usize: Uncompressed size of resulting objects on disk
@ -51,7 +52,7 @@ G_BEGIN_DECLS
* represents an OSTree object which will be created by the deltapart.
*/
#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(uayttay)"
/**
@ -131,9 +132,7 @@ gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo,
typedef enum {
OSTREE_STATIC_DELTA_OP_WRITE = 1,
OSTREE_STATIC_DELTA_OP_GUNZIP = 2,
OSTREE_STATIC_DELTA_OP_CLOSE = 3,
OSTREE_STATIC_DELTA_OP_READOBJECT = 4,
OSTREE_STATIC_DELTA_OP_READPAYLOAD = 5
OSTREE_STATIC_DELTA_OP_CLOSE = 3
} OstreeStaticDeltaOpCode;
gboolean

15
tests/test-delta.sh
View File

@ -72,21 +72,6 @@ if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to
assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded"
fi
origstart=$(echo ${origrev} | dd bs=1 count=2 2>/dev/null)
origend=$(echo ${origrev} | dd bs=1 skip=2 2>/dev/null)
assert_has_dir repo/deltas/${origstart}/${origend}-${newrev}
assert_has_dir repo/deltas/${origstart}/${origend}
mkdir repo2
ostree --repo=repo2 init --mode=archive-z2
ostree --repo=repo2 pull-local repo ${origrev}
ostree --repo=repo2 static-delta apply-offline repo/deltas/${origstart}/${origend}-${newrev}
ostree --repo=repo2 fsck
ostree --repo=repo2 show ${newrev}
mkdir repo3
ostree --repo=repo3 init --mode=archive-z2
ostree --repo=repo3 static-delta apply-offline repo/deltas/${origstart}/${origend}
ostree --repo=repo3 fsck
ostree --repo=repo3 show ${origrev}