shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-01-07 21:18:55 +03:00
Code Issues Projects Releases Wiki Activity

lib/commit: Refactor file commits to separate subdir from content

Browse Source 
One major thing we can do to speed up local commits is multithreading. In
preparation for that, split up the recursion function so that the subdirectory
case is separate from the content (regfile/symlink) case. Then for non-subdirs,
we can easily peel off worker threads and gather the final checksums and update
the mtree from the main thread.

The diff here looks large but it's pretty straightforward; amazingly this change
compiled the very first time I tried it!

Closes: #1365
Approved by: jlebon
This commit is contained in:
Colin Walters 2017-12-05 16:33:31 -05:00 committed by Atomic Bot
parent e108bef816
commit 9bb59511ae
1 changed files with 258 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
src/libostree/ostree-repo-commit.c
View File

@ -2833,12 +2833,112 @@ typedef enum {
WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT = 1,
} WriteDirContentFlags;
/* Given either a dir_enum or a dfd_iter, writes the directory entry to the mtree. For
* subdirs, we go back through either write_dfd_iter_to_mtree_internal (dfd_iter case) or
* write_directory_to_mtree_internal (dir_enum case) which will do the actual dirmeta +
* dirent iteration. */
/* Given either a dir_enum or a dfd_iter, writes the directory entry (which is
* itself a directory) to the mtree. For subdirs, we go back through either
* write_dfd_iter_to_mtree_internal (dfd_iter case) or
* write_directory_to_mtree_internal (dir_enum case) which will do the actual
* dirmeta + dirent iteration. */
static gboolean
write_directory_content_to_mtree_internal (OstreeRepo *self,
write_dir_entry_to_mtree_internal (OstreeRepo *self,
OstreeRepoFile *repo_dir,
GFileEnumerator *dir_enum,
GLnxDirFdIterator *dfd_iter,
WriteDirContentFlags writeflags,
GFileInfo *child_info,
OstreeMutableTree *mtree,
OstreeRepoCommitModifier *modifier,
GPtrArray *path,
GCancellable *cancellable,
GError **error)
{
g_assert (dir_enum != NULL || dfd_iter != NULL);
g_assert (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY);
const char *name = g_file_info_get_name (child_info);
/* We currently only honor the CONSUME flag in the dfd_iter case to avoid even
* more complexity in this function, and it'd mostly only be useful when
* operating on local filesystems anyways.
*/
const gboolean delete_after_commit = dfd_iter && modifier &&
(modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
/* Build the full path which we need for callbacks */
g_ptr_array_add (path, (char*)name);
g_autofree char *child_relpath = ptrarray_path_join (path);
/* Call the filter */
g_autoptr(GFileInfo) modified_info = NULL;
OstreeRepoCommitFilterResult filter_result =
_ostree_repo_commit_modifier_apply (self, modifier, child_relpath, child_info, &modified_info);
if (filter_result != OSTREE_REPO_COMMIT_FILTER_ALLOW)
{
g_ptr_array_remove_index (path, path->len - 1);
if (delete_after_commit)
{
g_assert (dfd_iter);
if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error))
return FALSE;
}
/* Note: early return */
return TRUE;
}
g_autoptr(GFile) child = NULL;
if (dir_enum != NULL)
child = g_file_enumerator_get_child (dir_enum, child_info);
g_autoptr(OstreeMutableTree) child_mtree = NULL;
if (!ostree_mutable_tree_ensure_dir (mtree, name, &child_mtree, error))
return FALSE;
/* Finally, recurse on the dir */
if (dir_enum != NULL)
{
if (!write_directory_to_mtree_internal (self, child, child_mtree,
modifier, path,
cancellable, error))
return FALSE;
}
else if (repo_dir)
{
g_assert (dir_enum != NULL);
g_debug ("Adding: %s", gs_file_get_path_cached (child));
if (!ostree_mutable_tree_replace_file (mtree, name,
ostree_repo_file_get_checksum ((OstreeRepoFile*) child),
error))
return FALSE;
}
else
{
g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, };
if (!glnx_dirfd_iterator_init_at (dfd_iter->fd, name, FALSE, &child_dfd_iter, error))
return FALSE;
if (!write_dfd_iter_to_mtree_internal (self, &child_dfd_iter, child_mtree,
modifier, path,
cancellable, error))
return FALSE;
if (delete_after_commit)
{
if (!glnx_unlinkat (dfd_iter->fd, name, AT_REMOVEDIR, error))
return FALSE;
}
}
g_ptr_array_remove_index (path, path->len - 1);
return TRUE;
}
/* Given either a dir_enum or a dfd_iter, writes a non-dir (regfile/symlink) to
* the mtree.
*/
static gboolean
write_content_to_mtree_internal (OstreeRepo *self,
OstreeRepoFile *repo_dir,
GFileEnumerator *dir_enum,
GLnxDirFdIterator *dfd_iter,
@ -2871,7 +2971,7 @@ write_directory_content_to_mtree_internal (OstreeRepo *self,
/* See if we have a devino hit; this is used below in a few places. */
const char *loose_checksum = NULL;
if (dfd_iter != NULL && (file_type != G_FILE_TYPE_DIRECTORY))
if (dfd_iter != NULL)
{
guint32 dev = g_file_info_get_attribute_uint32 (child_info, "unix::device");
guint64 inode = g_file_info_get_attribute_uint64 (child_info, "unix::inode");
@ -2933,7 +3033,6 @@ write_directory_content_to_mtree_internal (OstreeRepo *self,
switch (file_type)
{
case G_FILE_TYPE_DIRECTORY:
case G_FILE_TYPE_SYMBOLIC_LINK:
case G_FILE_TYPE_REGULAR:
break;
@ -2945,49 +3044,7 @@ write_directory_content_to_mtree_internal (OstreeRepo *self,
if (dir_enum != NULL)
child = g_file_enumerator_get_child (dir_enum, child_info);
if (file_type == G_FILE_TYPE_DIRECTORY)
{
g_autoptr(OstreeMutableTree) child_mtree = NULL;
if (!ostree_mutable_tree_ensure_dir (mtree, name, &child_mtree, error))
return FALSE;
if (dir_enum != NULL)
{
if (!write_directory_to_mtree_internal (self, child, child_mtree,
modifier, path,
cancellable, error))
return FALSE;
}
else
{
g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, };
if (!glnx_dirfd_iterator_init_at (dfd_iter->fd, name, FALSE, &child_dfd_iter, error))
return FALSE;
if (!write_dfd_iter_to_mtree_internal (self, &child_dfd_iter, child_mtree,
modifier, path,
cancellable, error))
return FALSE;
if (delete_after_commit)
{
if (!glnx_unlinkat (dfd_iter->fd, name, AT_REMOVEDIR, error))
return FALSE;
}
}
}
else if (repo_dir)
{
g_assert (dir_enum != NULL);
g_debug ("Adding: %s", gs_file_get_path_cached (child));
if (!ostree_mutable_tree_replace_file (mtree, name,
ostree_repo_file_get_checksum ((OstreeRepoFile*) child),
error))
return FALSE;
}
else
{
/* Our filters have passed, etc.; now we prepare to write the content object */
glnx_autofd int file_input_fd = -1;
/* Open the file now, since it's better for reading xattrs
@ -3121,7 +3178,6 @@ write_directory_content_to_mtree_internal (OstreeRepo *self,
if (!glnx_unlinkat (dfd_iter->fd, name, 0, error))
return FALSE;
}
}
g_ptr_array_remove_index (path, path->len - 1);
@ -3129,7 +3185,7 @@ write_directory_content_to_mtree_internal (OstreeRepo *self,
}
/* Handles the dirmeta for the given GFile dir and then calls
* write_directory_content_to_mtree_internal() for each directory entry. */
* write_{dir_entry,content}_to_mtree_internal() for each directory entry. */
static gboolean
write_directory_to_mtree_internal (OstreeRepo *self,
GFile *dir,
@ -3214,7 +3270,18 @@ write_directory_to_mtree_internal (OstreeRepo *self,
if (child_info == NULL)
break;
if (!write_directory_content_to_mtree_internal (self, repo_dir, dir_enum, NULL,
if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
{
if (!write_dir_entry_to_mtree_internal (self, repo_dir, dir_enum, NULL,
WRITE_DIR_CONTENT_FLAGS_NONE,
child_info,
mtree, modifier, path,
cancellable, error))
return FALSE;
}
else
{
if (!write_content_to_mtree_internal (self, repo_dir, dir_enum, NULL,
WRITE_DIR_CONTENT_FLAGS_NONE,
child_info,
mtree, modifier, path,
@ -3222,12 +3289,13 @@ write_directory_to_mtree_internal (OstreeRepo *self,
return FALSE;
}
}
}
return TRUE;
}
/* Handles the dirmeta for the dir described by src_dfd_iter and then calls
* write_directory_content_to_mtree_internal() for each directory entry. */
* write_{dir_entry,content}_to_mtree_internal() for each directory entry. */
static gboolean
write_dfd_iter_to_mtree_internal (OstreeRepo *self,
GLnxDirFdIterator *src_dfd_iter,
@ -3304,6 +3372,18 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self,
g_autoptr(GFileInfo) child_info = _ostree_stbuf_to_gfileinfo (&stbuf);
g_file_info_set_name (child_info, dent->d_name);
if (S_ISDIR (stbuf.st_mode))
{
if (!write_dir_entry_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
flags, child_info,
mtree, modifier, path,
cancellable, error))
return FALSE;
/* We handled the dir, move onto the next */
continue;
}
if (S_ISREG (stbuf.st_mode))
;
else if (S_ISLNK (stbuf.st_mode))
@ -3312,15 +3392,14 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self,
child_info, cancellable, error))
return FALSE;
}
else if (S_ISDIR (stbuf.st_mode))
;
else
{
return glnx_throw (error, "Not a regular file or symlink: %s",
dent->d_name);
}
if (!write_directory_content_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
/* Write a content object, we handled directories above */
if (!write_content_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
flags, child_info,
mtree, modifier, path,
cancellable, error))