andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

performance/md-cache: introduce an option to control invalidation of inodes

Browse Source 
Explicit invalidation by calling inode_invalidate is necessary when
same (meta)data is shared/access across multiple mounts. Without an
explicit inode_invalidate call, caches in the mount which didn't
witness writes wouldn't be aware of changes as writes wouldn't have
passed through them. However, if (meta)data is not shared, all
relevant I/O goes through the cache of single mount and hence is
coherent with (meta)data on bricks always. So, explicit inode
invalidation can be disabled for this case which gives a huge
performance boost for workloads that write data and then immediately
read the data they just wrote. Note that otherwise, local writes
(which pass through the cache) will change ctime and cause unnecessary
invalidations.

The name of the option that controls this behavior is
"performance.global-cache-invalidation". This option is global and it
purges caches both in glusterfs and kernel stack for native FUSE
mounts. For non-native FUSE mounts, it purges cache only from
glusterfs stack. This option is effective only when
performance.stat-prefetch is on.

Note that there is a similar option "performance.cache-invalidation",
but the scope of that option is limited to quick-read and md-cache.

Change-Id: I462bb4b65ff9aae1f6ba76f50b1f2f94fb10323b
Signed-off-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
updates: bz#1674364
(cherry picked from commit 2b5aa4489de2017a03bcb6ec8986286f0c76a670)
This commit is contained in:
Raghavendra Gowdappa 2019-02-08 09:51:17 +05:30 committed by Shyamsundar Ranganathan
parent beca73e04d
commit 2026d24667
2 changed files with 50 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
xlators/mgmt/glusterd/src/glusterd-volume-set.c
View File

@ -1795,6 +1795,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.op_version = GD_OP_VERSION_3_9_0,
.flags = VOLOPT_FLAG_CLIENT_OPT},
{.key = "performance.global-cache-invalidation",
.voltype = "performance/md-cache",
.option = "global-cache-invalidation",
.op_version = GD_OP_VERSION_6_0,
.flags = VOLOPT_FLAG_CLIENT_OPT},
/* Feature translators */
{.key = "features.uss",
.voltype = "features/snapview-server",

54
xlators/performance/md-cache/src/md-cache.c
View File

@ -71,6 +71,8 @@ struct mdc_conf {
gf_boolean_t cache_swift_metadata;
gf_boolean_t cache_samba_metadata;
gf_boolean_t mdc_invalidation;
gf_boolean_t global_invalidation;
time_t last_child_down;
gf_lock_t lock;
struct mdc_statistics mdc_counter;
@ -470,6 +472,7 @@ mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
uint32_t rollover = 0;
uint64_t gen = 0;
gf_boolean_t update_xa_time = _gf_false;
struct mdc_conf *conf = this->private;
mdc = mdc_inode_prep(this, inode);
if (!mdc) {
@ -533,18 +536,20 @@ mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
(iatt->ia_mtime_nsec != mdc->md_mtime_nsec) ||
(iatt->ia_ctime != mdc->md_ctime) ||
(iatt->ia_ctime_nsec != mdc->md_ctime_nsec)) {
if (!prebuf || (prebuf->ia_ctime != mdc->md_ctime) ||
(prebuf->ia_ctime_nsec != mdc->md_ctime_nsec) ||
(prebuf->ia_mtime != mdc->md_mtime) ||
(prebuf->ia_mtime_nsec != mdc->md_mtime_nsec)) {
gf_msg_trace("md-cache", 0,
"prebuf doesn't "
"match the value we have cached,"
" invalidate the inode(%s)",
uuid_utoa(inode->gfid));
if (conf->global_invalidation &&
(!prebuf || (prebuf->ia_mtime != mdc->md_mtime) ||
(prebuf->ia_mtime_nsec != mdc->md_mtime_nsec) ||
(prebuf->ia_ctime != mdc->md_ctime) ||
(prebuf->ia_ctime_nsec != mdc->md_ctime_nsec))) {
if (IA_ISREG(inode->ia_type)) {
gf_msg("md-cache", GF_LOG_TRACE, 0,
MD_CACHE_MSG_DISCARD_UPDATE,
"prebuf doesn't match the value we have cached,"
" invalidate the inode(%s)",
uuid_utoa(inode->gfid));
if (IA_ISREG(inode->ia_type))
inode_invalidate(inode);
}
} else {
update_xa_time = _gf_true;
}
@ -3551,6 +3556,9 @@ mdc_reconfigure(xlator_t *this, dict_t *options)
GF_OPTION_RECONF("cache-invalidation", conf->mdc_invalidation, options,
bool, out);
GF_OPTION_RECONF("global-cache-invalidation", conf->global_invalidation,
options, bool, out);
GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
GF_OPTION_RECONF("md-cache-statfs", conf->cache_statfs, options, bool, out);
@ -3622,6 +3630,9 @@ mdc_init(xlator_t *this)
GF_OPTION_INIT("cache-invalidation", conf->mdc_invalidation, bool, out);
GF_OPTION_INIT("global-cache-invalidation", conf->global_invalidation, bool,
out);
GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
pthread_mutex_init(&conf->statfs_cache.lock, NULL);
@ -3858,6 +3869,29 @@ struct volume_options mdc_options[] = {
.description = "When \"on\", invalidates/updates the metadata cache,"
" on receiving the cache-invalidation notifications",
},
{
.key = {"global-cache-invalidation"},
.type = GF_OPTION_TYPE_BOOL,
.default_value = "true",
.op_version = {GD_OP_VERSION_6_0},
.flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
.description =
"When \"on\", purges all read caches in kernel and glusterfs stack "
"whenever a stat change is detected. Stat changes can be detected "
"while processing responses to file operations (fop) or through "
"upcall notifications. Since purging caches can be an expensive "
"operation, it's advised to have this option \"on\" only when a "
"file "
"can be accessed from multiple different Glusterfs mounts and "
"caches across these different mounts are required to be coherent. "
"If a file is not accessed across different mounts "
"(simple example is having only one mount for a volume), its "
"advised to keep "
"this option \"off\" as all file modifications go through caches "
"keeping them "
"coherent. This option overrides value of "
"performance.cache-invalidation.",
},
{
.key = {"md-cache-statfs"},
.type = GF_OPTION_TYPE_BOOL,