From b092dadd55fb242a480f81c421303a9e53302156 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 4 Nov 2018 21:55:35 -0500 Subject: [PATCH] bcachefs: Scale down number of writepoints when low on space this means we don't have to reserve space for them when calculating filesystem capacity Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 41 ++---------- fs/bcachefs/alloc_background.h | 2 +- fs/bcachefs/alloc_foreground.c | 113 +++++++++++++++++++++++++++++++-- fs/bcachefs/alloc_foreground.h | 11 +--- fs/bcachefs/alloc_types.h | 4 +- fs/bcachefs/bcachefs.h | 6 +- fs/bcachefs/buckets.c | 5 -- fs/bcachefs/buckets.h | 6 ++ fs/bcachefs/super.c | 3 +- 9 files changed, 133 insertions(+), 58 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 88be5f4be4b1..1eb39283e7e2 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -975,6 +975,7 @@ void bch2_recalc_capacity(struct bch_fs *c) { struct bch_dev *ca; u64 capacity = 0, reserved_sectors = 0, gc_reserve; + unsigned bucket_size_max = 0; unsigned long ra_pages = 0; unsigned i, j; @@ -1012,12 +1013,9 @@ void bch2_recalc_capacity(struct bch_fs *c) dev_reserve += ca->free_inc.size; - dev_reserve += ARRAY_SIZE(c->write_points); - dev_reserve += 1; /* btree write point */ dev_reserve += 1; /* copygc write point */ dev_reserve += 1; /* rebalance write point */ - dev_reserve += WRITE_POINT_COUNT; dev_reserve *= ca->mi.bucket_size; @@ -1027,6 +1025,9 @@ void bch2_recalc_capacity(struct bch_fs *c) ca->mi.first_bucket); reserved_sectors += dev_reserve * 2; + + bucket_size_max = max_t(unsigned, bucket_size_max, + ca->mi.bucket_size); } gc_reserve = c->opts.gc_reserve_bytes @@ -1039,6 +1040,8 @@ void bch2_recalc_capacity(struct bch_fs *c) c->capacity = capacity - reserved_sectors; + c->bucket_size_max = bucket_size_max; + if (c->capacity) { bch2_io_timer_add(&c->io_clock[READ], &c->bucket_clock[READ].rescale); @@ -1330,8 +1333,6 @@ not_enough: * invalidated on disk: */ if (invalidating_data) { - BUG(); - pr_info("holding writes"); pr_debug("invalidating existing data"); set_bit(BCH_FS_HOLD_BTREE_WRITES, &c->flags); } else { @@ -1391,40 +1392,12 @@ int bch2_fs_allocator_start(struct bch_fs *c) return bch2_alloc_write(c); } -void bch2_fs_allocator_init(struct bch_fs *c) +void bch2_fs_allocator_background_init(struct bch_fs *c) { - struct open_bucket *ob; - struct write_point *wp; - - mutex_init(&c->write_points_hash_lock); spin_lock_init(&c->freelist_lock); bch2_bucket_clock_init(c, READ); bch2_bucket_clock_init(c, WRITE); - /* open bucket 0 is a sentinal NULL: */ - spin_lock_init(&c->open_buckets[0].lock); - - for (ob = c->open_buckets + 1; - ob < c->open_buckets + ARRAY_SIZE(c->open_buckets); ob++) { - spin_lock_init(&ob->lock); - c->open_buckets_nr_free++; - - ob->freelist = c->open_buckets_freelist; - c->open_buckets_freelist = ob - c->open_buckets; - } - - writepoint_init(&c->btree_write_point, BCH_DATA_BTREE); - writepoint_init(&c->rebalance_write_point, BCH_DATA_USER); - - for (wp = c->write_points; - wp < c->write_points + ARRAY_SIZE(c->write_points); wp++) { - writepoint_init(wp, BCH_DATA_USER); - - wp->last_used = sched_clock(); - wp->write_point = (unsigned long) wp; - hlist_add_head_rcu(&wp->node, writepoint_hash(c, wp->write_point)); - } - c->pd_controllers_update_seconds = 5; INIT_DELAYED_WORK(&c->pd_controllers_update, pd_controllers_update); } diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 6dbabe83cab7..245e037fbaea 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -57,6 +57,6 @@ int bch2_dev_allocator_start(struct bch_dev *); int bch2_alloc_write(struct bch_fs *); int bch2_fs_allocator_start(struct bch_fs *); -void bch2_fs_allocator_init(struct bch_fs *); +void bch2_fs_allocator_background_init(struct bch_fs *); #endif /* _BCACHEFS_ALLOC_BACKGROUND_H */ diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 920d9ff3c53b..df74e41ec890 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -492,7 +492,7 @@ void bch2_writepoint_stop(struct bch_fs *c, struct bch_dev *ca, mutex_lock(&wp->lock); open_bucket_for_each(c, &wp->ptrs, ob, i) - if (ob->ptr.dev == ca->dev_idx) + if (!ca || ob->ptr.dev == ca->dev_idx) open_bucket_free_unused(c, wp, ob); else ob_push(c, &ptrs, ob); @@ -501,6 +501,15 @@ void bch2_writepoint_stop(struct bch_fs *c, struct bch_dev *ca, mutex_unlock(&wp->lock); } +static inline struct hlist_head *writepoint_hash(struct bch_fs *c, + unsigned long write_point) +{ + unsigned hash = + hash_long(write_point, ilog2(ARRAY_SIZE(c->write_points_hash))); + + return &c->write_points_hash[hash]; +} + static struct write_point *__writepoint_find(struct hlist_head *head, unsigned long write_point) { @@ -513,6 +522,53 @@ static struct write_point *__writepoint_find(struct hlist_head *head, return NULL; } +static inline bool too_many_writepoints(struct bch_fs *c, unsigned factor) +{ + u64 stranded = c->write_points_nr * c->bucket_size_max; + u64 free = bch2_fs_sectors_free(c, bch2_fs_usage_read(c)); + + return stranded * factor > free; +} + +static bool try_increase_writepoints(struct bch_fs *c) +{ + struct write_point *wp; + + if (c->write_points_nr == ARRAY_SIZE(c->write_points) || + too_many_writepoints(c, 32)) + return false; + + wp = c->write_points + c->write_points_nr++; + hlist_add_head_rcu(&wp->node, writepoint_hash(c, wp->write_point)); + return true; +} + +static bool try_decrease_writepoints(struct bch_fs *c, + unsigned old_nr) +{ + struct write_point *wp; + + mutex_lock(&c->write_points_hash_lock); + if (c->write_points_nr < old_nr) { + mutex_unlock(&c->write_points_hash_lock); + return true; + } + + if (c->write_points_nr == 1 || + !too_many_writepoints(c, 8)) { + mutex_unlock(&c->write_points_hash_lock); + return false; + } + + wp = c->write_points + --c->write_points_nr; + + hlist_del_rcu(&wp->node); + mutex_unlock(&c->write_points_hash_lock); + + bch2_writepoint_stop(c, NULL, wp); + return true; +} + static struct write_point *writepoint_find(struct bch_fs *c, unsigned long write_point) { @@ -536,16 +592,22 @@ lock_wp: mutex_unlock(&wp->lock); goto restart_find; } - +restart_find_oldest: oldest = NULL; for (wp = c->write_points; - wp < c->write_points + ARRAY_SIZE(c->write_points); - wp++) + wp < c->write_points + c->write_points_nr; wp++) if (!oldest || time_before64(wp->last_used, oldest->last_used)) oldest = wp; mutex_lock(&oldest->lock); mutex_lock(&c->write_points_hash_lock); + if (oldest >= c->write_points + c->write_points_nr || + try_increase_writepoints(c)) { + mutex_unlock(&c->write_points_hash_lock); + mutex_unlock(&oldest->lock); + goto restart_find_oldest; + } + wp = __writepoint_find(head, write_point); if (wp && wp != oldest) { mutex_unlock(&c->write_points_hash_lock); @@ -581,10 +643,12 @@ struct write_point *bch2_alloc_sectors_start(struct bch_fs *c, unsigned nr_effective = 0; struct open_buckets ptrs = { .nr = 0 }; bool have_cache = false; + unsigned write_points_nr; int ret = 0, i; BUG_ON(!nr_replicas || !nr_replicas_required); - +retry: + write_points_nr = c->write_points_nr; wp = writepoint_find(c, write_point.v); if (!target || (flags & BCH_WRITE_ONLY_SPECIFIED_DEVS)) { @@ -637,6 +701,11 @@ err: wp->ptrs = ptrs; mutex_unlock(&wp->lock); + + if (ret == -ENOSPC && + try_decrease_writepoints(c, write_points_nr)) + goto retry; + return ERR_PTR(ret); } @@ -688,3 +757,37 @@ void bch2_alloc_sectors_done(struct bch_fs *c, struct write_point *wp) bch2_open_buckets_put(c, &ptrs); } + +void bch2_fs_allocator_foreground_init(struct bch_fs *c) +{ + struct open_bucket *ob; + struct write_point *wp; + + mutex_init(&c->write_points_hash_lock); + c->write_points_nr = ARRAY_SIZE(c->write_points); + + /* open bucket 0 is a sentinal NULL: */ + spin_lock_init(&c->open_buckets[0].lock); + + for (ob = c->open_buckets + 1; + ob < c->open_buckets + ARRAY_SIZE(c->open_buckets); ob++) { + spin_lock_init(&ob->lock); + c->open_buckets_nr_free++; + + ob->freelist = c->open_buckets_freelist; + c->open_buckets_freelist = ob - c->open_buckets; + } + + writepoint_init(&c->btree_write_point, BCH_DATA_BTREE); + writepoint_init(&c->rebalance_write_point, BCH_DATA_USER); + + for (wp = c->write_points; + wp < c->write_points + c->write_points_nr; wp++) { + writepoint_init(wp, BCH_DATA_USER); + + wp->last_used = sched_clock(); + wp->write_point = (unsigned long) wp; + hlist_add_head_rcu(&wp->node, + writepoint_hash(c, wp->write_point)); + } +} diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h index 636fe686dc48..6672101cbe26 100644 --- a/fs/bcachefs/alloc_foreground.h +++ b/fs/bcachefs/alloc_foreground.h @@ -91,15 +91,6 @@ void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *); void bch2_writepoint_stop(struct bch_fs *, struct bch_dev *, struct write_point *); -static inline struct hlist_head *writepoint_hash(struct bch_fs *c, - unsigned long write_point) -{ - unsigned hash = - hash_long(write_point, ilog2(ARRAY_SIZE(c->write_points_hash))); - - return &c->write_points_hash[hash]; -} - static inline struct write_point_specifier writepoint_hashed(unsigned long v) { return (struct write_point_specifier) { .v = v | 1 }; @@ -117,4 +108,6 @@ static inline void writepoint_init(struct write_point *wp, wp->type = type; } +void bch2_fs_allocator_foreground_init(struct bch_fs *); + #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */ diff --git a/fs/bcachefs/alloc_types.h b/fs/bcachefs/alloc_types.h index e0306d68ae9f..2a9c6f0344ed 100644 --- a/fs/bcachefs/alloc_types.h +++ b/fs/bcachefs/alloc_types.h @@ -46,7 +46,9 @@ typedef FIFO(long) alloc_fifo; /* Enough for 16 cache devices, 2 tiers and some left over for pipelining */ #define OPEN_BUCKETS_COUNT 256 -#define WRITE_POINT_COUNT 32 + +#define WRITE_POINT_HASH_NR 32 +#define WRITE_POINT_MAX 32 struct open_bucket { spinlock_t lock; diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 95d505aaf82f..5665b93f200b 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -601,6 +601,7 @@ struct bch_fs { * and forces them to be revalidated */ u32 capacity_gen; + unsigned bucket_size_max; atomic64_t sectors_available; @@ -630,9 +631,10 @@ struct bch_fs { struct write_point btree_write_point; struct write_point rebalance_write_point; - struct write_point write_points[WRITE_POINT_COUNT]; - struct hlist_head write_points_hash[WRITE_POINT_COUNT]; + struct write_point write_points[WRITE_POINT_MAX]; + struct hlist_head write_points_hash[WRITE_POINT_HASH_NR]; struct mutex write_points_hash_lock; + unsigned write_points_nr; /* GARBAGE COLLECTION */ struct task_struct *gc_thread; diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 6f40c4bd16ec..cfbe3ed41d0e 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -300,11 +300,6 @@ u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage stats) return min(c->capacity, __bch2_fs_sectors_used(c, stats)); } -static u64 bch2_fs_sectors_free(struct bch_fs *c, struct bch_fs_usage stats) -{ - return c->capacity - bch2_fs_sectors_used(c, stats); -} - static inline int is_unavailable_bucket(struct bucket_mark m) { return !is_available_bucket(m); diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index e22c51972c31..c40ffe862a06 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -175,6 +175,12 @@ void bch2_fs_usage_apply(struct bch_fs *, struct bch_fs_usage *, u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage); +static inline u64 bch2_fs_sectors_free(struct bch_fs *c, + struct bch_fs_usage stats) +{ + return c->capacity - bch2_fs_sectors_used(c, stats); +} + static inline bool is_available_bucket(struct bucket_mark mark) { return (!mark.owned_by_allocator && diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 9d9d4fb8348b..a2ee698970a8 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -524,7 +524,8 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) for (i = 0; i < BCH_TIME_STAT_NR; i++) bch2_time_stats_init(&c->times[i]); - bch2_fs_allocator_init(c); + bch2_fs_allocator_background_init(c); + bch2_fs_allocator_foreground_init(c); bch2_fs_rebalance_init(c); bch2_fs_quota_init(c);