495f2ae9e3
Fix the fileserver rotation so that it doesn't use RTT as the basis for deciding which server and address to use as this doesn't necessarily give a good indication of the best path. Instead, use the configurable preference list in conjunction with whatever probes have succeeded at the time of looking. To this end, make the following changes: (1) Keep an array of "server states" to track what addresses we've tried on each server and move the waitqueue entries there that we'll need for probing. (2) Each afs_server_state struct is made to pin the corresponding server's endpoint state rather than the afs_operation struct carrying a pin on the server we're currently looking at. (3) Drop the server list preference; we now always rescan the server list. (4) afs_wait_for_probes() now uses the server state list to guide it in what it waits for (and to provide the waitqueue entries) and returns an indication of whether we'd got a response, run out of responsive addresses or the endpoint state had been superseded and we need to restart the iteration. (5) Call afs_get_address_preferences*() occasionally to refresh the preference values. (6) When picking a server, scan the addresses of the servers for which we have as-yet untested communications, looking for the highest priority one and use that instead of trying all the addresses for a particular server in ascending-RTT order. (7) When a Busy or Offline state is seen across all available servers, do a short sleep. (8) If we detect that we accessed a future RO volume version whilst it is undergoing replication, reissue the op against the older version until at least half of the servers are replicated. (9) Whilst RO replication is ongoing, increase the frequency of Volume Location server checks for that volume to every ten minutes instead of hourly. Also add a tracepoint to track progress through the rotation algorithm. Signed-off-by: David Howells <dhowells@redhat.com> cc: Marc Dionne <marc.dionne@auristor.com> cc: linux-afs@lists.infradead.org
470 lines
12 KiB
C
470 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/* AFS volume management
|
|
*
|
|
* Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved.
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/slab.h>
|
|
#include "internal.h"
|
|
|
|
static unsigned __read_mostly afs_volume_record_life = 60 * 60;
|
|
|
|
static void afs_destroy_volume(struct work_struct *work);
|
|
|
|
/*
|
|
* Insert a volume into a cell. If there's an existing volume record, that is
|
|
* returned instead with a ref held.
|
|
*/
|
|
static struct afs_volume *afs_insert_volume_into_cell(struct afs_cell *cell,
|
|
struct afs_volume *volume)
|
|
{
|
|
struct afs_volume *p;
|
|
struct rb_node *parent = NULL, **pp;
|
|
|
|
write_seqlock(&cell->volume_lock);
|
|
|
|
pp = &cell->volumes.rb_node;
|
|
while (*pp) {
|
|
parent = *pp;
|
|
p = rb_entry(parent, struct afs_volume, cell_node);
|
|
if (p->vid < volume->vid) {
|
|
pp = &(*pp)->rb_left;
|
|
} else if (p->vid > volume->vid) {
|
|
pp = &(*pp)->rb_right;
|
|
} else {
|
|
if (afs_try_get_volume(p, afs_volume_trace_get_cell_insert)) {
|
|
volume = p;
|
|
goto found;
|
|
}
|
|
|
|
set_bit(AFS_VOLUME_RM_TREE, &volume->flags);
|
|
rb_replace_node_rcu(&p->cell_node, &volume->cell_node, &cell->volumes);
|
|
}
|
|
}
|
|
|
|
rb_link_node_rcu(&volume->cell_node, parent, pp);
|
|
rb_insert_color(&volume->cell_node, &cell->volumes);
|
|
hlist_add_head_rcu(&volume->proc_link, &cell->proc_volumes);
|
|
|
|
found:
|
|
write_sequnlock(&cell->volume_lock);
|
|
return volume;
|
|
|
|
}
|
|
|
|
static void afs_remove_volume_from_cell(struct afs_volume *volume)
|
|
{
|
|
struct afs_cell *cell = volume->cell;
|
|
|
|
if (!hlist_unhashed(&volume->proc_link)) {
|
|
trace_afs_volume(volume->vid, refcount_read(&cell->ref),
|
|
afs_volume_trace_remove);
|
|
write_seqlock(&cell->volume_lock);
|
|
hlist_del_rcu(&volume->proc_link);
|
|
if (!test_and_set_bit(AFS_VOLUME_RM_TREE, &volume->flags))
|
|
rb_erase(&volume->cell_node, &cell->volumes);
|
|
write_sequnlock(&cell->volume_lock);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Allocate a volume record and load it up from a vldb record.
|
|
*/
|
|
static struct afs_volume *afs_alloc_volume(struct afs_fs_context *params,
|
|
struct afs_vldb_entry *vldb,
|
|
struct afs_server_list **_slist)
|
|
{
|
|
struct afs_server_list *slist;
|
|
struct afs_volume *volume;
|
|
int ret = -ENOMEM, i;
|
|
|
|
volume = kzalloc(sizeof(struct afs_volume), GFP_KERNEL);
|
|
if (!volume)
|
|
goto error_0;
|
|
|
|
volume->vid = vldb->vid[params->type];
|
|
volume->update_at = ktime_get_real_seconds() + afs_volume_record_life;
|
|
volume->cell = afs_get_cell(params->cell, afs_cell_trace_get_vol);
|
|
volume->type = params->type;
|
|
volume->type_force = params->force;
|
|
volume->name_len = vldb->name_len;
|
|
volume->creation_time = TIME64_MIN;
|
|
volume->update_time = TIME64_MIN;
|
|
|
|
refcount_set(&volume->ref, 1);
|
|
INIT_HLIST_NODE(&volume->proc_link);
|
|
INIT_WORK(&volume->destructor, afs_destroy_volume);
|
|
rwlock_init(&volume->servers_lock);
|
|
mutex_init(&volume->volsync_lock);
|
|
mutex_init(&volume->cb_check_lock);
|
|
rwlock_init(&volume->cb_v_break_lock);
|
|
INIT_LIST_HEAD(&volume->open_mmaps);
|
|
init_rwsem(&volume->open_mmaps_lock);
|
|
memcpy(volume->name, vldb->name, vldb->name_len + 1);
|
|
|
|
for (i = 0; i < AFS_MAXTYPES; i++)
|
|
volume->vids[i] = vldb->vid[i];
|
|
|
|
slist = afs_alloc_server_list(volume, params->key, vldb);
|
|
if (IS_ERR(slist)) {
|
|
ret = PTR_ERR(slist);
|
|
goto error_1;
|
|
}
|
|
|
|
*_slist = slist;
|
|
rcu_assign_pointer(volume->servers, slist);
|
|
trace_afs_volume(volume->vid, 1, afs_volume_trace_alloc);
|
|
return volume;
|
|
|
|
error_1:
|
|
afs_put_cell(volume->cell, afs_cell_trace_put_vol);
|
|
kfree(volume);
|
|
error_0:
|
|
return ERR_PTR(ret);
|
|
}
|
|
|
|
/*
|
|
* Look up or allocate a volume record.
|
|
*/
|
|
static struct afs_volume *afs_lookup_volume(struct afs_fs_context *params,
|
|
struct afs_vldb_entry *vldb)
|
|
{
|
|
struct afs_server_list *slist;
|
|
struct afs_volume *candidate, *volume;
|
|
|
|
candidate = afs_alloc_volume(params, vldb, &slist);
|
|
if (IS_ERR(candidate))
|
|
return candidate;
|
|
|
|
volume = afs_insert_volume_into_cell(params->cell, candidate);
|
|
if (volume == candidate)
|
|
afs_attach_volume_to_servers(volume, slist);
|
|
else
|
|
afs_put_volume(candidate, afs_volume_trace_put_cell_dup);
|
|
return volume;
|
|
}
|
|
|
|
/*
|
|
* Look up a VLDB record for a volume.
|
|
*/
|
|
static struct afs_vldb_entry *afs_vl_lookup_vldb(struct afs_cell *cell,
|
|
struct key *key,
|
|
const char *volname,
|
|
size_t volnamesz)
|
|
{
|
|
struct afs_vldb_entry *vldb = ERR_PTR(-EDESTADDRREQ);
|
|
struct afs_vl_cursor vc;
|
|
int ret;
|
|
|
|
if (!afs_begin_vlserver_operation(&vc, cell, key))
|
|
return ERR_PTR(-ERESTARTSYS);
|
|
|
|
while (afs_select_vlserver(&vc)) {
|
|
vldb = afs_vl_get_entry_by_name_u(&vc, volname, volnamesz);
|
|
}
|
|
|
|
ret = afs_end_vlserver_operation(&vc);
|
|
return ret < 0 ? ERR_PTR(ret) : vldb;
|
|
}
|
|
|
|
/*
|
|
* Look up a volume in the VL server and create a candidate volume record for
|
|
* it.
|
|
*
|
|
* The volume name can be one of the following:
|
|
* "%[cell:]volume[.]" R/W volume
|
|
* "#[cell:]volume[.]" R/O or R/W volume (rwparent=0),
|
|
* or R/W (rwparent=1) volume
|
|
* "%[cell:]volume.readonly" R/O volume
|
|
* "#[cell:]volume.readonly" R/O volume
|
|
* "%[cell:]volume.backup" Backup volume
|
|
* "#[cell:]volume.backup" Backup volume
|
|
*
|
|
* The cell name is optional, and defaults to the current cell.
|
|
*
|
|
* See "The Rules of Mount Point Traversal" in Chapter 5 of the AFS SysAdmin
|
|
* Guide
|
|
* - Rule 1: Explicit type suffix forces access of that type or nothing
|
|
* (no suffix, then use Rule 2 & 3)
|
|
* - Rule 2: If parent volume is R/O, then mount R/O volume by preference, R/W
|
|
* if not available
|
|
* - Rule 3: If parent volume is R/W, then only mount R/W volume unless
|
|
* explicitly told otherwise
|
|
*/
|
|
struct afs_volume *afs_create_volume(struct afs_fs_context *params)
|
|
{
|
|
struct afs_vldb_entry *vldb;
|
|
struct afs_volume *volume;
|
|
unsigned long type_mask = 1UL << params->type;
|
|
|
|
vldb = afs_vl_lookup_vldb(params->cell, params->key,
|
|
params->volname, params->volnamesz);
|
|
if (IS_ERR(vldb))
|
|
return ERR_CAST(vldb);
|
|
|
|
if (test_bit(AFS_VLDB_QUERY_ERROR, &vldb->flags)) {
|
|
volume = ERR_PTR(vldb->error);
|
|
goto error;
|
|
}
|
|
|
|
/* Make the final decision on the type we want */
|
|
volume = ERR_PTR(-ENOMEDIUM);
|
|
if (params->force) {
|
|
if (!(vldb->flags & type_mask))
|
|
goto error;
|
|
} else if (test_bit(AFS_VLDB_HAS_RO, &vldb->flags)) {
|
|
params->type = AFSVL_ROVOL;
|
|
} else if (test_bit(AFS_VLDB_HAS_RW, &vldb->flags)) {
|
|
params->type = AFSVL_RWVOL;
|
|
} else {
|
|
goto error;
|
|
}
|
|
|
|
volume = afs_lookup_volume(params, vldb);
|
|
|
|
error:
|
|
kfree(vldb);
|
|
return volume;
|
|
}
|
|
|
|
/*
|
|
* Destroy a volume record
|
|
*/
|
|
static void afs_destroy_volume(struct work_struct *work)
|
|
{
|
|
struct afs_volume *volume = container_of(work, struct afs_volume, destructor);
|
|
struct afs_server_list *slist = rcu_access_pointer(volume->servers);
|
|
|
|
_enter("%p", volume);
|
|
|
|
#ifdef CONFIG_AFS_FSCACHE
|
|
ASSERTCMP(volume->cache, ==, NULL);
|
|
#endif
|
|
|
|
afs_detach_volume_from_servers(volume, slist);
|
|
afs_remove_volume_from_cell(volume);
|
|
afs_put_serverlist(volume->cell->net, slist);
|
|
afs_put_cell(volume->cell, afs_cell_trace_put_vol);
|
|
trace_afs_volume(volume->vid, refcount_read(&volume->ref),
|
|
afs_volume_trace_free);
|
|
kfree_rcu(volume, rcu);
|
|
|
|
_leave(" [destroyed]");
|
|
}
|
|
|
|
/*
|
|
* Try to get a reference on a volume record.
|
|
*/
|
|
bool afs_try_get_volume(struct afs_volume *volume, enum afs_volume_trace reason)
|
|
{
|
|
int r;
|
|
|
|
if (__refcount_inc_not_zero(&volume->ref, &r)) {
|
|
trace_afs_volume(volume->vid, r + 1, reason);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
* Get a reference on a volume record.
|
|
*/
|
|
struct afs_volume *afs_get_volume(struct afs_volume *volume,
|
|
enum afs_volume_trace reason)
|
|
{
|
|
if (volume) {
|
|
int r;
|
|
|
|
__refcount_inc(&volume->ref, &r);
|
|
trace_afs_volume(volume->vid, r + 1, reason);
|
|
}
|
|
return volume;
|
|
}
|
|
|
|
|
|
/*
|
|
* Drop a reference on a volume record.
|
|
*/
|
|
void afs_put_volume(struct afs_volume *volume, enum afs_volume_trace reason)
|
|
{
|
|
if (volume) {
|
|
afs_volid_t vid = volume->vid;
|
|
bool zero;
|
|
int r;
|
|
|
|
zero = __refcount_dec_and_test(&volume->ref, &r);
|
|
trace_afs_volume(vid, r - 1, reason);
|
|
if (zero)
|
|
schedule_work(&volume->destructor);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Activate a volume.
|
|
*/
|
|
int afs_activate_volume(struct afs_volume *volume)
|
|
{
|
|
#ifdef CONFIG_AFS_FSCACHE
|
|
struct fscache_volume *vcookie;
|
|
char *name;
|
|
|
|
name = kasprintf(GFP_KERNEL, "afs,%s,%llx",
|
|
volume->cell->name, volume->vid);
|
|
if (!name)
|
|
return -ENOMEM;
|
|
|
|
vcookie = fscache_acquire_volume(name, NULL, NULL, 0);
|
|
if (IS_ERR(vcookie)) {
|
|
if (vcookie != ERR_PTR(-EBUSY)) {
|
|
kfree(name);
|
|
return PTR_ERR(vcookie);
|
|
}
|
|
pr_err("AFS: Cache volume key already in use (%s)\n", name);
|
|
vcookie = NULL;
|
|
}
|
|
volume->cache = vcookie;
|
|
kfree(name);
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Deactivate a volume.
|
|
*/
|
|
void afs_deactivate_volume(struct afs_volume *volume)
|
|
{
|
|
_enter("%s", volume->name);
|
|
|
|
#ifdef CONFIG_AFS_FSCACHE
|
|
fscache_relinquish_volume(volume->cache, NULL,
|
|
test_bit(AFS_VOLUME_DELETED, &volume->flags));
|
|
volume->cache = NULL;
|
|
#endif
|
|
|
|
_leave("");
|
|
}
|
|
|
|
/*
|
|
* Query the VL service to update the volume status.
|
|
*/
|
|
static int afs_update_volume_status(struct afs_volume *volume, struct key *key)
|
|
{
|
|
struct afs_server_list *new, *old, *discard;
|
|
struct afs_vldb_entry *vldb;
|
|
char idbuf[16];
|
|
int ret, idsz;
|
|
|
|
_enter("");
|
|
|
|
/* We look up an ID by passing it as a decimal string in the
|
|
* operation's name parameter.
|
|
*/
|
|
idsz = sprintf(idbuf, "%llu", volume->vid);
|
|
|
|
vldb = afs_vl_lookup_vldb(volume->cell, key, idbuf, idsz);
|
|
if (IS_ERR(vldb)) {
|
|
ret = PTR_ERR(vldb);
|
|
goto error;
|
|
}
|
|
|
|
/* See if the volume got renamed. */
|
|
if (vldb->name_len != volume->name_len ||
|
|
memcmp(vldb->name, volume->name, vldb->name_len) != 0) {
|
|
/* TODO: Use RCU'd string. */
|
|
memcpy(volume->name, vldb->name, AFS_MAXVOLNAME);
|
|
volume->name_len = vldb->name_len;
|
|
}
|
|
|
|
/* See if the volume's server list got updated. */
|
|
new = afs_alloc_server_list(volume, key, vldb);
|
|
if (IS_ERR(new)) {
|
|
ret = PTR_ERR(new);
|
|
goto error_vldb;
|
|
}
|
|
|
|
write_lock(&volume->servers_lock);
|
|
|
|
discard = new;
|
|
old = rcu_dereference_protected(volume->servers,
|
|
lockdep_is_held(&volume->servers_lock));
|
|
if (afs_annotate_server_list(new, old)) {
|
|
new->seq = volume->servers_seq + 1;
|
|
rcu_assign_pointer(volume->servers, new);
|
|
smp_wmb();
|
|
volume->servers_seq++;
|
|
discard = old;
|
|
}
|
|
|
|
/* Check more often if replication is ongoing. */
|
|
if (new->ro_replicating)
|
|
volume->update_at = ktime_get_real_seconds() + 10 * 60;
|
|
else
|
|
volume->update_at = ktime_get_real_seconds() + afs_volume_record_life;
|
|
write_unlock(&volume->servers_lock);
|
|
|
|
if (discard == old)
|
|
afs_reattach_volume_to_servers(volume, new, old);
|
|
afs_put_serverlist(volume->cell->net, discard);
|
|
ret = 0;
|
|
error_vldb:
|
|
kfree(vldb);
|
|
error:
|
|
_leave(" = %d", ret);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Make sure the volume record is up to date.
|
|
*/
|
|
int afs_check_volume_status(struct afs_volume *volume, struct afs_operation *op)
|
|
{
|
|
int ret, retries = 0;
|
|
|
|
_enter("");
|
|
|
|
retry:
|
|
if (test_bit(AFS_VOLUME_WAIT, &volume->flags))
|
|
goto wait;
|
|
if (volume->update_at <= ktime_get_real_seconds() ||
|
|
test_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags))
|
|
goto update;
|
|
_leave(" = 0");
|
|
return 0;
|
|
|
|
update:
|
|
if (!test_and_set_bit_lock(AFS_VOLUME_UPDATING, &volume->flags)) {
|
|
clear_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags);
|
|
ret = afs_update_volume_status(volume, op->key);
|
|
if (ret < 0)
|
|
set_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags);
|
|
clear_bit_unlock(AFS_VOLUME_WAIT, &volume->flags);
|
|
clear_bit_unlock(AFS_VOLUME_UPDATING, &volume->flags);
|
|
wake_up_bit(&volume->flags, AFS_VOLUME_WAIT);
|
|
_leave(" = %d", ret);
|
|
return ret;
|
|
}
|
|
|
|
wait:
|
|
if (!test_bit(AFS_VOLUME_WAIT, &volume->flags)) {
|
|
_leave(" = 0 [no wait]");
|
|
return 0;
|
|
}
|
|
|
|
ret = wait_on_bit(&volume->flags, AFS_VOLUME_WAIT,
|
|
(op->flags & AFS_OPERATION_UNINTR) ?
|
|
TASK_UNINTERRUPTIBLE : TASK_INTERRUPTIBLE);
|
|
if (ret == -ERESTARTSYS) {
|
|
_leave(" = %d", ret);
|
|
return ret;
|
|
}
|
|
|
|
retries++;
|
|
if (retries == 4) {
|
|
_leave(" = -ESTALE");
|
|
return -ESTALE;
|
|
}
|
|
goto retry;
|
|
}
|