shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-01-09 01:18:35 +03:00
Code Issues Projects Releases Wiki Activity

src/ostree: Add --commit-only option to ostree prune

Browse Source 
Recently we have noticed exceedingly long execution times
for multiple invocations of ostree prune. This is a result of
calculating full reachability on each invocation.

The --commit-only flag provides an alternative strategy. It will only
traverse and delete commit objects to avoid the more expensive
reachability calculations. This allows us to chain multiple --commit-only
commands cheaply, and then follow with a more expensive ostree prune
invocation at the end to clean up orphaned meta and content objects.
This commit is contained in:
Saqib Ali 2022-02-07 10:53:08 -05:00
parent 00e39ebedc
commit 725d50a3b5
7 changed files with 179 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile-libostree.am
View File

@ -171,9 +171,9 @@ endif # USE_GPGME
symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
# Uncomment this include when adding new development symbols.
# if BUILDOPT_IS_DEVEL_BUILD
# symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
# endif
if BUILDOPT_IS_DEVEL_BUILD
symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
endif
# http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
wl_versionscript_arg = -Wl,--version-script=

1
apidoc/ostree-sections.txt
View File

@ -451,6 +451,7 @@ ostree_repo_traverse_parents_get_commits
ostree_repo_traverse_commit
ostree_repo_traverse_commit_union
ostree_repo_traverse_commit_union_with_parents
ostree_repo_traverse_commit_with_flags
ostree_repo_commit_traverse_iter_cleanup
ostree_repo_commit_traverse_iter_clear
ostree_repo_commit_traverse_iter_get_dir

5
src/libostree/libostree-devel.sym
View File

@ -20,6 +20,11 @@
- uncomment the include in Makefile-libostree.am
*/
LIBOSTREE_2022.2 {
global:
ostree_repo_traverse_commit_with_flags;
} LIBOSTREE_2021.5;
/* Stub section for the stable release *after* this development one; don't
* edit this other than to update the year. This is just a copy/paste
* source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION

109
src/libostree/ostree-repo-prune.c
View File

@ -48,6 +48,10 @@ maybe_prune_loose_object (OtPruneData *data,
OstreeObjectType objtype;
ostree_object_name_deserialize (key, &checksum, &objtype);
/* Return if we only want to delete commits and this object is not a commit object. */
gboolean commit_only = flags & OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY;
if (commit_only && (objtype != OSTREE_OBJECT_TYPE_COMMIT))
goto exit;
if (g_hash_table_lookup_extended (data->reachable, key, NULL, NULL))
reachable = TRUE;
@ -125,7 +129,11 @@ maybe_prune_loose_object (OtPruneData *data,
else
data->n_reachable_content++;
}
if (commit_only && (objtype != OSTREE_OBJECT_TYPE_COMMIT))
{
g_debug ("Keeping object (not commit) %s.%s", checksum,
ostree_object_type_to_string (objtype));
}
return TRUE;
}
@ -299,6 +307,52 @@ repo_prune_internal (OstreeRepo *self,
return TRUE;
}
static gboolean
traverse_reachable_internal (OstreeRepo *self,
OstreeRepoCommitTraverseFlags flags,
guint depth,
GHashTable *reachable,
GCancellable *cancellable,
GError **error)
{
g_autoptr(OstreeRepoAutoLock) lock =
ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_SHARED, cancellable, error);
if (!lock)
return FALSE;
/* Ignoring collections. */
g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */
if (!ostree_repo_list_refs (self, NULL, &all_refs,
cancellable, error))
return FALSE;
GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum)
{
g_debug ("Finding objects to keep for commit %s", checksum);
if (!ostree_repo_traverse_commit_with_flags (self, flags, checksum, depth, reachable,
NULL, cancellable, error))
return FALSE;
}
/* Using collections. */
g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */
if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs,
OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, cancellable, error))
return FALSE;
GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum)
{
g_debug ("Finding objects to keep for commit %s", checksum);
if (!ostree_repo_traverse_commit_with_flags (self, flags, checksum, depth, reachable,
NULL, cancellable, error))
return FALSE;
}
return TRUE;
}
/**
* ostree_repo_traverse_reachable_refs:
* @self: Repo
@ -319,42 +373,10 @@ ostree_repo_traverse_reachable_refs (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
g_autoptr(OstreeRepoAutoLock) lock =
ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_SHARED, cancellable, error);
if (!lock)
return FALSE;
/* Ignoring collections. */
g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */
if (!ostree_repo_list_refs (self, NULL, &all_refs,
cancellable, error))
return FALSE;
GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum)
{
g_debug ("Finding objects to keep for commit %s", checksum);
if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
cancellable, error))
return FALSE;
}
/* Using collections. */
g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */
if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs,
OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, cancellable, error))
return FALSE;
GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum)
{
g_debug ("Finding objects to keep for commit %s", checksum);
if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
cancellable, error))
return FALSE;
}
return TRUE;
return traverse_reachable_internal (self,
OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
depth, reachable,
cancellable, error);
}
/**
@ -401,6 +423,7 @@ ostree_repo_prune (OstreeRepo *self,
g_autoptr(GHashTable) objects = NULL;
gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
gboolean commit_only = flags & OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY;
g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable ();
@ -409,9 +432,15 @@ ostree_repo_prune (OstreeRepo *self,
* the deletion.
*/
OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE;
if (commit_only)
traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
if (refs_only)
{
if (!ostree_repo_traverse_reachable_refs (self, depth, reachable, cancellable, error))
if (!traverse_reachable_internal (self, traverse_flags,
depth, reachable,
cancellable, error))
return FALSE;
}
@ -432,8 +461,8 @@ ostree_repo_prune (OstreeRepo *self,
continue;
g_debug ("Finding objects to keep for commit %s", checksum);
if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
cancellable, error))
if (!ostree_repo_traverse_commit_with_flags (self, traverse_flags, checksum, depth, reachable,
NULL, cancellable, error))
return FALSE;
}
}

74
src/libostree/ostree-repo-traverse.c
View File

@ -542,8 +542,9 @@ traverse_dirtree (OstreeRepo *repo,
}
/**
* ostree_repo_traverse_commit_union_with_parents: (skip)
* ostree_repo_traverse_commit_with_flags: (skip)
* @repo: Repo
* @flags: change traversal behaviour according to these flags
* @commit_checksum: ASCII SHA256 checksum
* @maxdepth: Traverse this many parent commits, -1 for unlimited
* @inout_reachable: Set of reachable objects
@ -561,15 +562,17 @@ traverse_dirtree (OstreeRepo *repo,
* Since: 2018.5
*/
gboolean
ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo,
const char *commit_checksum,
int maxdepth,
GHashTable *inout_reachable,
GHashTable *inout_parents,
GCancellable *cancellable,
GError **error)
ostree_repo_traverse_commit_with_flags (OstreeRepo *repo,
OstreeRepoCommitTraverseFlags flags,
const char *commit_checksum,
int maxdepth,
GHashTable *inout_reachable,
GHashTable *inout_parents,
GCancellable *cancellable,
GError **error)
{
g_autofree char *tmp_checksum = NULL;
gboolean commit_only = flags & OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
while (TRUE)
{
@ -603,16 +606,20 @@ ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo,
g_hash_table_add (inout_reachable, g_variant_ref (key));
g_debug ("Traversing commit %s", commit_checksum);
ostree_cleanup_repo_commit_traverse_iter
OstreeRepoCommitTraverseIter iter = { 0, };
if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit,
OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
error))
return FALSE;
/* Save time by skipping traversal of non-commit objects */
if (!commit_only)
{
g_debug ("Traversing commit %s", commit_checksum);
ostree_cleanup_repo_commit_traverse_iter
OstreeRepoCommitTraverseIter iter = { 0, };
if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit,
OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
error))
return FALSE;
if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error))
return FALSE;
if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error))
return FALSE;
}
gboolean recurse = FALSE;
if (maxdepth == -1 || maxdepth > 0)
@ -634,6 +641,39 @@ ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo,
return TRUE;
}
/**
* ostree_repo_traverse_commit_union_with_parents: (skip)
* @repo: Repo
* @commit_checksum: ASCII SHA256 checksum
* @maxdepth: Traverse this many parent commits, -1 for unlimited
* @inout_reachable: Set of reachable objects
* @inout_parents: Map from object to parent object
* @cancellable: Cancellable
* @error: Error
*
* Update the set @inout_reachable containing all objects reachable
* from @commit_checksum, traversing @maxdepth parent commits.
*
* Additionally this constructs a mapping from each object to the parents
* of the object, which can be used to track which commits an object
* belongs to.
*
* Since: 2018.5
*/
gboolean
ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo,
const char *commit_checksum,
int maxdepth,
GHashTable *inout_reachable,
GHashTable *inout_parents,
GCancellable *cancellable,
GError **error)
{
return ostree_repo_traverse_commit_with_flags(repo, OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE,
commit_checksum, maxdepth, inout_reachable, inout_parents,
cancellable, error);
}
/**
* ostree_repo_traverse_commit_union: (skip)
* @repo: Repo

27
src/libostree/ostree-repo.h
View File

@ -1118,6 +1118,16 @@ typedef enum {
OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0,
} OstreeStaticDeltaIndexFlags;
/**
* OstreeRepoCommitTraverseFlags:
* @OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE: No special options for traverse
* @OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY: Traverse and retrieve only commit objects. (Since: 2022.2)
*/
typedef enum {
OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0),
OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY = (1 << 1),
} OstreeRepoCommitTraverseFlags;
_OSTREE_PUBLIC
gboolean ostree_repo_static_delta_reindex (OstreeRepo *repo,
OstreeStaticDeltaIndexFlags flags,
@ -1171,6 +1181,7 @@ gboolean ostree_repo_traverse_commit_union (OstreeRepo *repo,
GHashTable *inout_reachable,
GCancellable *cancellable,
GError **error);
_OSTREE_PUBLIC
gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo,
const char *commit_checksum,
@ -1180,6 +1191,16 @@ gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo *rep
GCancellable *cancellable,
GError **error);
_OSTREE_PUBLIC
gboolean ostree_repo_traverse_commit_with_flags (OstreeRepo *repo,
OstreeRepoCommitTraverseFlags flags,
const char *commit_checksum,
int maxdepth,
GHashTable *inout_reachable,
GHashTable *inout_parents,
GCancellable *cancellable,
GError **error);
struct _OstreeRepoCommitTraverseIter {
gboolean initialized;
/* 4 byte hole on 64 bit */
@ -1189,10 +1210,6 @@ struct _OstreeRepoCommitTraverseIter {
typedef struct _OstreeRepoCommitTraverseIter OstreeRepoCommitTraverseIter;
typedef enum {
OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0)
} OstreeRepoCommitTraverseFlags;
_OSTREE_PUBLIC
gboolean
ostree_repo_commit_traverse_iter_init_commit (OstreeRepoCommitTraverseIter *iter,
@ -1245,11 +1262,13 @@ void ostree_repo_commit_traverse_iter_cleanup (void *p);
* @OSTREE_REPO_PRUNE_FLAGS_NONE: No special options for pruning
* @OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE: Don't actually delete objects
* @OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY: Do not traverse individual commit objects, only follow refs
* @OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY: Only traverse commit objects. (Since 2022.2)
*/
typedef enum {
OSTREE_REPO_PRUNE_FLAGS_NONE = 0,
OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE = (1 << 0),
OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY = (1 << 1),
OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY = (1 << 2),
} OstreeRepoPruneFlags;
_OSTREE_PUBLIC

28
src/ostree/ot-builtin-prune.c
View File

@ -35,6 +35,7 @@ static char *opt_delete_commit;
static char *opt_keep_younger_than;
static char **opt_retain_branch_depth;
static char **opt_only_branches;
static gboolean opt_commit_only;
/* ATTENTION:
* Please remember to update the bash-completion script (bash/ostree) and
@ -50,6 +51,7 @@ static GOptionEntry options[] = {
{ "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" },
{ "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" },
{ "only-branch", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_only_branches, "Only prune BRANCH (may be specified multiple times)", "BRANCH" },
{ "commit-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Only traverse and delete commit objects.", NULL },
{ NULL }
};
@ -99,12 +101,15 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
GCancellable *cancellable, GError **error)
{
g_autofree char *next_checksum = g_strdup (checksum);
OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE;
if (opt_commit_only)
traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
/* This is the first commit in our loop, which has a ref pointing to it. We
* don't want to auto-prune it.
*/
if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable,
cancellable, error))
if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, checksum, 0, reachable,
NULL, cancellable, error))
return FALSE;
while (TRUE)
@ -121,8 +126,8 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
if (commit_timestamp >= ts->tv_sec)
{
/* It's newer, traverse it */
if (!ostree_repo_traverse_commit_union (repo, next_checksum, 0, reachable,
cancellable, error))
if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, next_checksum, 0, reachable,
NULL, cancellable, error))
return FALSE;
g_free (next_checksum);
@ -183,6 +188,8 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
pruneflags |= OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
if (opt_no_prune)
pruneflags |= OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE;
if (opt_commit_only)
pruneflags |= OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY;
/* If no newer more complex options are specified, drop down to the original
* prune API - both to avoid code duplication, and to keep it run from the
@ -285,6 +292,10 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
/* Traverse each ref, and gather all objects pointed to by it up to a
* specific depth (if configured).
*/
OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE;
if (opt_commit_only)
/** We can avoid looking at all objects if --commit-only is specified **/
traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY;
g_hash_table_iter_init (&hash_iter, all_refs);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
@ -316,8 +327,8 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
the global default */
g_debug ("Finding objects to keep for commit %s", checksum);
if (!ostree_repo_traverse_commit_union (repo, checksum, depth, reachable,
cancellable, error))
if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, checksum, depth, reachable,
NULL, cancellable, error))
return FALSE;
}
@ -333,7 +344,10 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation
}
g_autofree char *formatted_freed_size = g_format_size_full (objsize_total, 0);
g_print ("Total objects: %u\n", n_objects_total);
if (opt_commit_only)
g_print("Total (commit only) objects: %u\n", n_objects_total);
else
g_print ("Total objects: %u\n", n_objects_total);
if (n_objects_pruned == 0)
g_print ("No unreachable objects\n");
else if (pruneflags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)