bcachefs: New bucket invalidate path

In the old allocator code, preparing an existing empty bucket was part
of the same code path that invalidated buckets containing cached data.
In the new allocator code this is no longer the case: the main allocator
path finds empty buckets (via the new freespace btree), and can't
allocate buckets that contain cached data.

We now need a separate code path to invalidate buckets containing cached
data when we're low on empty buckets, which this patch implements. When
the number of free buckets decreases that triggers the new invalidate
path to run, which uses the LRU btree to pick cached data buckets to
invalidate until we're above our watermark.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/alloc_background.c

@@ -718,6 +718,86 @@ void bch2_do_discards(struct bch_fs *c)
 		percpu_ref_put(&c->writes);
 }
 
+static int invalidate_one_bucket(struct btree_trans *trans, struct bch_dev *ca)
+{
+	struct bch_fs *c = trans->c;
+	struct btree_iter lru_iter, alloc_iter = { NULL };
+	struct bkey_s_c k;
+	struct bkey_i_alloc_v4 *a;
+	u64 bucket, idx;
+	int ret;
+
+	bch2_trans_iter_init(trans, &lru_iter, BTREE_ID_lru,
+			     POS(ca->dev_idx, 0), 0);
+	k = bch2_btree_iter_peek(&lru_iter);
+	ret = bkey_err(k);
+	if (ret)
+		goto out;
+
+	if (!k.k || k.k->p.inode != ca->dev_idx)
+		goto out;
+
+	if (bch2_fs_inconsistent_on(k.k->type != KEY_TYPE_lru, c,
+				    "non lru key in lru btree"))
+		goto out;
+
+	idx	= k.k->p.offset;
+	bucket	= le64_to_cpu(bkey_s_c_to_lru(k).v->idx);
+
+	a = bch2_trans_start_alloc_update(trans, &alloc_iter,
+					  POS(ca->dev_idx, bucket));
+	ret = PTR_ERR_OR_ZERO(a);
+	if (ret)
+		goto out;
+
+	if (bch2_fs_inconsistent_on(idx != alloc_lru_idx(a->v), c,
+			"invalidating bucket with wrong lru idx (got %llu should be %llu",
+			idx, alloc_lru_idx(a->v)))
+		goto out;
+
+	SET_BCH_ALLOC_V4_NEED_INC_GEN(&a->v, false);
+	a->v.gen++;
+	a->v.data_type		= 0;
+	a->v.dirty_sectors	= 0;
+	a->v.cached_sectors	= 0;
+	a->v.io_time[READ]	= atomic64_read(&c->io_clock[READ].now);
+	a->v.io_time[WRITE]	= atomic64_read(&c->io_clock[WRITE].now);
+
+	ret = bch2_trans_update(trans, &alloc_iter, &a->k_i,
+				BTREE_TRIGGER_BUCKET_INVALIDATE);
+out:
+	bch2_trans_iter_exit(trans, &alloc_iter);
+	bch2_trans_iter_exit(trans, &lru_iter);
+	return ret;
+}
+
+static void bch2_do_invalidates_work(struct work_struct *work)
+{
+	struct bch_fs *c = container_of(work, struct bch_fs, invalidate_work);
+	struct bch_dev *ca;
+	struct btree_trans trans;
+	unsigned i;
+	int ret = 0;
+
+	bch2_trans_init(&trans, c, 0, 0);
+
+	for_each_member_device(ca, c, i)
+		while (!ret && should_invalidate_buckets(ca))
+			ret = __bch2_trans_do(&trans, NULL, NULL,
+					      BTREE_INSERT_USE_RESERVE|
+					      BTREE_INSERT_NOFAIL,
+					invalidate_one_bucket(&trans, ca));
+
+	bch2_trans_exit(&trans);
+	percpu_ref_put(&c->writes);
+}
+
+void bch2_do_invalidates(struct bch_fs *c)
+{
+	if (percpu_ref_tryget(&c->writes))
+		queue_work(system_long_wq, &c->invalidate_work);
+}
+
 static int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca)
 {
 	struct btree_trans trans;
@@ -1002,4 +1082,5 @@ void bch2_fs_allocator_background_init(struct bch_fs *c)
 {
 	spin_lock_init(&c->freelist_lock);
 	INIT_WORK(&c->discard_work, bch2_do_discards_work);
+	INIT_WORK(&c->invalidate_work, bch2_do_invalidates_work);
 }

fs/bcachefs/alloc_background.h

@@ -115,6 +115,17 @@ int bch2_trans_mark_alloc(struct btree_trans *, struct bkey_s_c,
 			  struct bkey_i *, unsigned);
 void bch2_do_discards(struct bch_fs *);
 
+static inline bool should_invalidate_buckets(struct bch_dev *ca)
+{
+	struct bch_dev_usage u = bch2_dev_usage_read(ca);
+
+	return u.d[BCH_DATA_cached].buckets &&
+		u.buckets_unavailable + u.d[BCH_DATA_cached].buckets <
+		ca->mi.nbuckets >> 7;
+}
+
+void bch2_do_invalidates(struct bch_fs *);
+
 int bch2_fs_freespace_init(struct bch_fs *);
 
 void bch2_recalc_capacity(struct bch_fs *);

fs/bcachefs/bcachefs.h

@@ -759,6 +759,7 @@ struct bch_fs {
 
 	struct buckets_waiting_for_journal buckets_waiting_for_journal;
 	struct work_struct	discard_work;
+	struct work_struct	invalidate_work;
 
 	/* GARBAGE COLLECTION */
 	struct task_struct	*gc_thread;

fs/bcachefs/buckets.c

@@ -548,6 +548,11 @@ int bch2_mark_alloc(struct btree_trans *trans,
 	    !new_a.journal_seq)
 		bch2_do_discards(c);
 
+	if (!old_a.data_type &&
+	    new_a.data_type &&
+	    should_invalidate_buckets(ca))
+		bch2_do_invalidates(c);
+
 	if (bucket_state(new_a) == BUCKET_need_gc_gens) {
 		atomic_inc(&c->kick_gc);
 		wake_up_process(c->gc_thread);