andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mount/fuse: expose auto-invalidation as a mount option

Browse Source 
Auto invalidation is necessary when same (meta)data is shared/access
across multiple mounts. However, if (meta)data is not shared, all
relevant I/O goes through the cache of single mount and hence is
coherent with (meta)data on bricks always. So, fuse-auto-invalidation
can be disabled for this case which gives a huge performance boost for
workloads that write data and then immediately read the data they just
wrote.

From glusterfs --help,

<snip>
      --auto-invalidation[=BOOL]   controls whether fuse-kernel can
                             auto-invalidate attribute, dentry and page-cache.
                             Disable this only if same files/directories are
                             not accessed across two different mounts
                             concurrently [default: "on"]
</snip>

Details on how disabling auto-invalidation helped to reduce pgbench
init times can be found at [1]. Time taken for pgbench init of scale
8000 was 8340s. That will be an improvement of 86% (59280s vs 8340s)
with auto-invalidations turned off along with other
optimizations. Just disabling auto-invalidation contributed 56%
improvement by reducing the total time taken by 33260s.

[1] https://www.spinics.net/lists/gluster-devel/msg25907.html

Change-Id: I0ed730dba9064bd9c576ad1800170a21e100e1ce
Signed-off-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
updates: bz#1664934
This commit is contained in:
Raghavendra Gowdappa 2019-01-29 08:05:07 +05:30 committed by Amar Tumballi
parent 4674678951
commit a229ee1c8c
10 changed files with 77 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/glusterfs.8
View File

@ -139,6 +139,11 @@ Enable fuse in-kernel writeback cache.
\fB\-\-negative\-timeout=SECONDS\fR
Set negative timeout to SECONDS in fuse kernel module (the default is 0).
.TP
\fB\-\-auto\-invalidation=BOOL\fR
controls whether fuse-kernel can auto-invalidate attribute, dentry and
page-cache. Disable this only if same files/directories are not
accessed across two different mounts concurrently [default: on].
.TP
\fB\-\-volfile-check\fR
Enable strict volume file checking.

5
doc/glusterfsd.8
View File

@ -107,6 +107,11 @@ Enable/Disable direct-io mode in fuse module [default: enable]
.TP
\fB\-\-resolve-gids\fR
Resolve all auxiliary groups in fuse translator (max 32 otherwise)
.TP
\fB\-\-auto\-invalidation=BOOL\fR
controls whether fuse-kernel can auto-invalidate attribute, dentry and
page-cache. Disable this only if same files/directories are not
accessed across two different mounts concurrently [default: on]
.SS "Miscellaneous Options"
.PP

5
doc/mount.glusterfs.8
View File

@ -146,6 +146,11 @@ Enable fuse in-kernel writeback cache [default: off]
.TP
\fBattr\-times\-granularity=\fRNS
Declare supported granularity of file attribute [default: 0]
.TP
\fBauto\-invalidation=\fRBOOL
controls whether fuse-kernel can auto-invalidate attribute, dentry and
page-cache. Disable this only if same files/directories are not
accessed across two different mounts concurrently [default: on]
.PP
.SH FILES
.TP

24
glusterfsd/src/glusterfsd.c
View File

@ -270,6 +270,11 @@ static struct argp_option gf_options[] = {
{"fuse-flush-handle-interrupt", ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY,
"BOOL", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
"handle interrupt in fuse FLUSH handler"},
{"auto-invalidation", ARGP_FUSE_AUTO_INVAL_KEY, "BOOL", OPTION_ARG_OPTIONAL,
"controls whether fuse-kernel can auto-invalidate "
"attribute, dentry and page-cache. "
"Disable this only if same files/directories are not accessed across "
"two different mounts concurrently [default: \"on\"]"},
{0, 0, 0, 0, "Miscellaneous Options:"},
{
0,
@ -623,6 +628,15 @@ set_fuse_mount_options(glusterfs_ctx_t *ctx, dict_t *options)
goto err;
}
}
ret = dict_set_uint32(options, "auto-invalidation",
cmd_args->fuse_auto_inval);
if (ret < 0) {
gf_msg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
"failed to set dict value for key auto-invalidation");
goto err;
}
switch (cmd_args->kernel_writeback_cache) {
case GF_OPTION_ENABLE:
ret = dict_set_static_ptr(options, "kernel-writeback-cache", "on");
@ -1483,6 +1497,16 @@ parse_opts(int key, char *arg, struct argp_state *state)
"unknown fuse flush handle interrupt setting \"%s\"",
arg);
break;
case ARGP_FUSE_AUTO_INVAL_KEY:
if (!arg)
arg = "yes";
if (gf_string2boolean(arg, &b) == 0) {
cmd_args->fuse_auto_inval = b;
break;
}
break;
}
return 0;
}

1
glusterfsd/src/glusterfsd.h
View File

@ -110,6 +110,7 @@ enum argp_option_keys {
ARGP_PRINT_LIBEXECDIR_KEY = 188,
ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY = 189,
ARGP_FUSE_LRU_LIMIT_KEY = 190,
ARGP_FUSE_AUTO_INVAL_KEY = 191,
};
struct _gfd_vol_top_priv {

1
libglusterfs/src/glusterfs/glusterfs.h
View File

@ -574,6 +574,7 @@ struct _cmd_args {
uint32_t attr_times_granularity;
int fuse_flush_handle_interrupt;
int fuse_auto_inval;
};
typedef struct _cmd_args cmd_args_t;

3
tests/basic/mount-options.disabled
View File

@ -127,6 +127,9 @@ EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-transport=ib-verbs
EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --auto-invalidation=off
EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-port=socket
TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volume-name=$V0

35
xlators/mount/fuse/src/fuse-bridge.c
View File

@ -5067,6 +5067,9 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
/* If user did not explicitly set --fopen-keep-cache[=off],
then check if kernel support FUSE_AUTO_INVAL_DATA and ...
*/
priv->fopen_keep_cache = 1;
#if FUSE_KERNEL_MINOR_VERSION >= 20
if (fini->flags & FUSE_AUTO_INVAL_DATA) {
/* ... enable fopen_keep_cache mode if supported.
@ -5075,25 +5078,26 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
"Detected "
"support for FUSE_AUTO_INVAL_DATA. Enabling "
"fopen_keep_cache automatically.");
fino.flags |= FUSE_AUTO_INVAL_DATA;
priv->fopen_keep_cache = 1;
if (priv->fuse_auto_inval)
fino.flags |= FUSE_AUTO_INVAL_DATA;
} else
#endif
{
gf_log("glusterfs-fuse", GF_LOG_DEBUG,
"No support "
"for FUSE_AUTO_INVAL_DATA. Disabling "
"fopen_keep_cache.");
/* ... else disable. */
priv->fopen_keep_cache = 0;
if (priv->fuse_auto_inval) {
gf_log("glusterfs-fuse", GF_LOG_DEBUG,
"No support for FUSE_AUTO_INVAL_DATA. Disabling "
"fopen_keep_cache.");
/* ... else disable. */
priv->fopen_keep_cache = 0;
}
}
} else if (priv->fopen_keep_cache == 1) {
/* If user explicitly set --fopen-keep-cache[=on],
then enable FUSE_AUTO_INVAL_DATA if possible.
*/
#if FUSE_KERNEL_MINOR_VERSION >= 20
if (fini->flags & FUSE_AUTO_INVAL_DATA) {
if (priv->fuse_auto_inval && (fini->flags & FUSE_AUTO_INVAL_DATA)) {
gf_log("glusterfs-fuse", GF_LOG_DEBUG,
"fopen_keep_cache "
"is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
@ -6507,6 +6511,8 @@ init(xlator_t *this_xl)
GF_OPTION_INIT("reader-thread-count", priv->reader_thread_count, uint32,
cleanup_exit);
GF_OPTION_INIT("auto-invalidation", priv->fuse_auto_inval, bool,
cleanup_exit);
GF_OPTION_INIT(ZR_ENTRY_TIMEOUT_OPT, priv->entry_timeout, double,
cleanup_exit);
@ -6944,6 +6950,15 @@ struct volume_options options[] = {
.description = "makes glusterfs invalidate kernel inodes after "
"reaching this limit (0 means 'unlimited')",
},
{
.key = {"auto-invalidation"},
.type = GF_OPTION_TYPE_BOOL,
.default_value = "true",
.description = "controls whether fuse-kernel can auto-invalidate "
"attribute, dentry and page-cache. Disable this only "
"if same files/directories are not accessed across "
"two different mounts concurrently",
},
{.key = {NULL}},
};

1
xlators/mount/fuse/src/fuse-bridge.h
View File

@ -187,6 +187,7 @@ struct fuse_private {
pthread_mutex_t interrupt_mutex;
gf_boolean_t flush_handle_interrupt;
gf_boolean_t fuse_auto_inval;
/* LRU Limit, if not set, default is 128k for now */
uint32_t lru_limit;

7
xlators/mount/fuse/utils/mount.glusterfs.in
View File

@ -229,6 +229,10 @@ start_glusterfs ()
cmd_line=$(echo "$cmd_line --reader-thread-count=$reader_thread_count");
fi
if [ -n "$auto-invalidation" ]; then
cmd_line=$(echo "$cmd_line --auto-invalidation=$fuse_auto_invalidation");
fi
if [ -n "$volume_name" ]; then
cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
fi
@ -529,6 +533,9 @@ with_options()
"reader-thread-count")
reader_thread_count=$value
;;
"auto-invalidation")
fuse_auto_invalidation=$value
;;
"no-root-squash")
if [ $value = "yes" ] ||
[ $value = "on" ] ||