0d2602ca30
This adds support for active queue tracking, meaning that the blk-mq tagging maintains a count of active users of a tag set. This allows us to maintain a notion of fairness between users, so that we can distribute the tag depth evenly without starving some users while allowing others to try unfair deep queues. If sharing of a tag set is detected, each hardware queue will track the depth of its own queue. And if this exceeds the total depth divided by the number of active queues, the user is actively throttled down. The active queue count is done lazily to avoid bouncing that data between submitter and completer. Each hardware queue gets marked active when it allocates its first tag, and gets marked inactive when 1) the last tag is cleared, and 2) the queue timeout grace period has passed. Signed-off-by: Jens Axboe <axboe@fb.com>
93 lines
2.3 KiB
C
93 lines
2.3 KiB
C
#ifndef INT_BLK_MQ_TAG_H
|
|
#define INT_BLK_MQ_TAG_H
|
|
|
|
enum {
|
|
BT_WAIT_QUEUES = 8,
|
|
BT_WAIT_BATCH = 8,
|
|
};
|
|
|
|
struct bt_wait_state {
|
|
atomic_t wait_cnt;
|
|
wait_queue_head_t wait;
|
|
} ____cacheline_aligned_in_smp;
|
|
|
|
#define TAG_TO_INDEX(bt, tag) ((tag) >> (bt)->bits_per_word)
|
|
#define TAG_TO_BIT(bt, tag) ((tag) & ((1 << (bt)->bits_per_word) - 1))
|
|
|
|
struct blk_mq_bitmap {
|
|
unsigned long word;
|
|
unsigned long depth;
|
|
} ____cacheline_aligned_in_smp;
|
|
|
|
struct blk_mq_bitmap_tags {
|
|
unsigned int depth;
|
|
unsigned int wake_cnt;
|
|
unsigned int bits_per_word;
|
|
|
|
unsigned int map_nr;
|
|
struct blk_mq_bitmap *map;
|
|
|
|
unsigned int wake_index;
|
|
struct bt_wait_state *bs;
|
|
};
|
|
|
|
/*
|
|
* Tag address space map.
|
|
*/
|
|
struct blk_mq_tags {
|
|
unsigned int nr_tags;
|
|
unsigned int nr_reserved_tags;
|
|
|
|
atomic_t active_queues;
|
|
|
|
struct blk_mq_bitmap_tags bitmap_tags;
|
|
struct blk_mq_bitmap_tags breserved_tags;
|
|
|
|
struct request **rqs;
|
|
struct list_head page_list;
|
|
};
|
|
|
|
|
|
extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node);
|
|
extern void blk_mq_free_tags(struct blk_mq_tags *tags);
|
|
|
|
extern unsigned int blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, unsigned int *last_tag, gfp_t gfp, bool reserved);
|
|
extern void blk_mq_wait_for_tags(struct blk_mq_hw_ctx *hctx, bool reserved);
|
|
extern void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag, unsigned int *last_tag);
|
|
extern void blk_mq_tag_busy_iter(struct blk_mq_tags *tags, void (*fn)(void *data, unsigned long *), void *data);
|
|
extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);
|
|
extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page);
|
|
extern void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *last_tag);
|
|
|
|
enum {
|
|
BLK_MQ_TAG_CACHE_MIN = 1,
|
|
BLK_MQ_TAG_CACHE_MAX = 64,
|
|
};
|
|
|
|
enum {
|
|
BLK_MQ_TAG_FAIL = -1U,
|
|
BLK_MQ_TAG_MIN = BLK_MQ_TAG_CACHE_MIN,
|
|
BLK_MQ_TAG_MAX = BLK_MQ_TAG_FAIL - 1,
|
|
};
|
|
|
|
extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *);
|
|
extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *);
|
|
|
|
static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
|
|
{
|
|
if (!(hctx->flags & BLK_MQ_F_TAG_SHARED))
|
|
return false;
|
|
|
|
return __blk_mq_tag_busy(hctx);
|
|
}
|
|
|
|
static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
|
|
{
|
|
if (!(hctx->flags & BLK_MQ_F_TAG_SHARED))
|
|
return;
|
|
|
|
__blk_mq_tag_idle(hctx);
|
|
}
|
|
|
|
#endif
|