Merge branch 'for-3.13/post-mq-drivers' into for-linus
This commit is contained in:
commit
1355b37f11
@ -39,15 +39,15 @@ Module configuration options
|
||||
============================
|
||||
|
||||
If you use the floppy driver as a module, use the following syntax:
|
||||
modprobe floppy <options>
|
||||
modprobe floppy floppy="<options>"
|
||||
|
||||
Example:
|
||||
modprobe floppy omnibook messages
|
||||
modprobe floppy floppy="omnibook messages"
|
||||
|
||||
If you need certain options enabled every time you load the floppy driver,
|
||||
you can put:
|
||||
|
||||
options floppy omnibook messages
|
||||
options floppy floppy="omnibook messages"
|
||||
|
||||
in a configuration file in /etc/modprobe.d/.
|
||||
|
||||
|
@ -110,7 +110,7 @@ source "drivers/block/mtip32xx/Kconfig"
|
||||
|
||||
config BLK_CPQ_DA
|
||||
tristate "Compaq SMART2 support"
|
||||
depends on PCI && VIRT_TO_BUS
|
||||
depends on PCI && VIRT_TO_BUS && 0
|
||||
help
|
||||
This is the driver for Compaq Smart Array controllers. Everyone
|
||||
using these boards should say Y here. See the file
|
||||
@ -319,6 +319,16 @@ config BLK_DEV_NVME
|
||||
To compile this driver as a module, choose M here: the
|
||||
module will be called nvme.
|
||||
|
||||
config BLK_DEV_SKD
|
||||
tristate "STEC S1120 Block Driver"
|
||||
depends on PCI
|
||||
depends on 64BIT
|
||||
---help---
|
||||
Saying Y or M here will enable support for the
|
||||
STEC, Inc. S1120 PCIe SSD.
|
||||
|
||||
Use device /dev/skd$N amd /dev/skd$Np$M.
|
||||
|
||||
config BLK_DEV_OSD
|
||||
tristate "OSD object-as-blkdev support"
|
||||
depends on SCSI_OSD_ULD
|
||||
|
@ -23,6 +23,7 @@ obj-$(CONFIG_CDROM_PKTCDVD) += pktcdvd.o
|
||||
obj-$(CONFIG_MG_DISK) += mg_disk.o
|
||||
obj-$(CONFIG_SUNVDC) += sunvdc.o
|
||||
obj-$(CONFIG_BLK_DEV_NVME) += nvme.o
|
||||
obj-$(CONFIG_BLK_DEV_SKD) += skd.o
|
||||
obj-$(CONFIG_BLK_DEV_OSD) += osdblk.o
|
||||
|
||||
obj-$(CONFIG_BLK_DEV_UMEM) += umem.o
|
||||
@ -44,4 +45,5 @@ obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
|
||||
obj-$(CONFIG_BLK_DEV_NULL_BLK) += null_blk.o
|
||||
|
||||
nvme-y := nvme-core.o nvme-scsi.o
|
||||
skd-y := skd_main.o
|
||||
swim_mod-y := swim.o swim_asm.o
|
||||
|
@ -5183,7 +5183,7 @@ reinit_after_soft_reset:
|
||||
rebuild_lun_table(h, 1, 0);
|
||||
cciss_engage_scsi(h);
|
||||
h->busy_initializing = 0;
|
||||
return 1;
|
||||
return 0;
|
||||
|
||||
clean4:
|
||||
cciss_free_cmd_pool(h);
|
||||
|
@ -1474,7 +1474,8 @@ enum determine_dev_size {
|
||||
DS_ERROR = -1,
|
||||
DS_UNCHANGED = 0,
|
||||
DS_SHRUNK = 1,
|
||||
DS_GREW = 2
|
||||
DS_GREW = 2,
|
||||
DS_GREW_FROM_ZERO = 3,
|
||||
};
|
||||
extern enum determine_dev_size
|
||||
drbd_determine_dev_size(struct drbd_conf *, enum dds_flags, struct resize_parms *) __must_hold(local);
|
||||
|
@ -2750,13 +2750,6 @@ int __init drbd_init(void)
|
||||
return err;
|
||||
}
|
||||
|
||||
err = drbd_genl_register();
|
||||
if (err) {
|
||||
printk(KERN_ERR "drbd: unable to register generic netlink family\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
|
||||
register_reboot_notifier(&drbd_notifier);
|
||||
|
||||
/*
|
||||
@ -2767,6 +2760,15 @@ int __init drbd_init(void)
|
||||
drbd_proc = NULL; /* play safe for drbd_cleanup */
|
||||
idr_init(&minors);
|
||||
|
||||
rwlock_init(&global_state_lock);
|
||||
INIT_LIST_HEAD(&drbd_tconns);
|
||||
|
||||
err = drbd_genl_register();
|
||||
if (err) {
|
||||
printk(KERN_ERR "drbd: unable to register generic netlink family\n");
|
||||
goto fail;
|
||||
}
|
||||
|
||||
err = drbd_create_mempools();
|
||||
if (err)
|
||||
goto fail;
|
||||
@ -2778,9 +2780,6 @@ int __init drbd_init(void)
|
||||
goto fail;
|
||||
}
|
||||
|
||||
rwlock_init(&global_state_lock);
|
||||
INIT_LIST_HEAD(&drbd_tconns);
|
||||
|
||||
retry.wq = create_singlethread_workqueue("drbd-reissue");
|
||||
if (!retry.wq) {
|
||||
printk(KERN_ERR "drbd: unable to create retry workqueue\n");
|
||||
|
@ -955,7 +955,7 @@ drbd_determine_dev_size(struct drbd_conf *mdev, enum dds_flags flags, struct res
|
||||
}
|
||||
|
||||
if (size > la_size_sect)
|
||||
rv = DS_GREW;
|
||||
rv = la_size_sect ? DS_GREW : DS_GREW_FROM_ZERO;
|
||||
if (size < la_size_sect)
|
||||
rv = DS_SHRUNK;
|
||||
|
||||
@ -1132,9 +1132,9 @@ void drbd_reconsider_max_bio_size(struct drbd_conf *mdev)
|
||||
/* We may ignore peer limits if the peer is modern enough.
|
||||
Because new from 8.3.8 onwards the peer can use multiple
|
||||
BIOs for a single peer_request */
|
||||
if (mdev->state.conn >= C_CONNECTED) {
|
||||
if (mdev->state.conn >= C_WF_REPORT_PARAMS) {
|
||||
if (mdev->tconn->agreed_pro_version < 94)
|
||||
peer = min( mdev->peer_max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
|
||||
peer = min(mdev->peer_max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
|
||||
/* Correct old drbd (up to 8.3.7) if it believes it can do more than 32KiB */
|
||||
else if (mdev->tconn->agreed_pro_version == 94)
|
||||
peer = DRBD_MAX_SIZE_H80_PACKET;
|
||||
|
@ -1890,29 +1890,11 @@ static u32 seq_max(u32 a, u32 b)
|
||||
return seq_greater(a, b) ? a : b;
|
||||
}
|
||||
|
||||
static bool need_peer_seq(struct drbd_conf *mdev)
|
||||
{
|
||||
struct drbd_tconn *tconn = mdev->tconn;
|
||||
int tp;
|
||||
|
||||
/*
|
||||
* We only need to keep track of the last packet_seq number of our peer
|
||||
* if we are in dual-primary mode and we have the resolve-conflicts flag set; see
|
||||
* handle_write_conflicts().
|
||||
*/
|
||||
|
||||
rcu_read_lock();
|
||||
tp = rcu_dereference(mdev->tconn->net_conf)->two_primaries;
|
||||
rcu_read_unlock();
|
||||
|
||||
return tp && test_bit(RESOLVE_CONFLICTS, &tconn->flags);
|
||||
}
|
||||
|
||||
static void update_peer_seq(struct drbd_conf *mdev, unsigned int peer_seq)
|
||||
{
|
||||
unsigned int newest_peer_seq;
|
||||
|
||||
if (need_peer_seq(mdev)) {
|
||||
if (test_bit(RESOLVE_CONFLICTS, &mdev->tconn->flags)) {
|
||||
spin_lock(&mdev->peer_seq_lock);
|
||||
newest_peer_seq = seq_max(mdev->peer_seq, peer_seq);
|
||||
mdev->peer_seq = newest_peer_seq;
|
||||
@ -1972,22 +1954,31 @@ static int wait_for_and_update_peer_seq(struct drbd_conf *mdev, const u32 peer_s
|
||||
{
|
||||
DEFINE_WAIT(wait);
|
||||
long timeout;
|
||||
int ret;
|
||||
int ret = 0, tp;
|
||||
|
||||
if (!need_peer_seq(mdev))
|
||||
if (!test_bit(RESOLVE_CONFLICTS, &mdev->tconn->flags))
|
||||
return 0;
|
||||
|
||||
spin_lock(&mdev->peer_seq_lock);
|
||||
for (;;) {
|
||||
if (!seq_greater(peer_seq - 1, mdev->peer_seq)) {
|
||||
mdev->peer_seq = seq_max(mdev->peer_seq, peer_seq);
|
||||
ret = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
if (signal_pending(current)) {
|
||||
ret = -ERESTARTSYS;
|
||||
break;
|
||||
}
|
||||
|
||||
rcu_read_lock();
|
||||
tp = rcu_dereference(mdev->tconn->net_conf)->two_primaries;
|
||||
rcu_read_unlock();
|
||||
|
||||
if (!tp)
|
||||
break;
|
||||
|
||||
/* Only need to wait if two_primaries is enabled */
|
||||
prepare_to_wait(&mdev->seq_wait, &wait, TASK_INTERRUPTIBLE);
|
||||
spin_unlock(&mdev->peer_seq_lock);
|
||||
rcu_read_lock();
|
||||
@ -2228,8 +2219,10 @@ static int receive_Data(struct drbd_tconn *tconn, struct packet_info *pi)
|
||||
}
|
||||
goto out_interrupted;
|
||||
}
|
||||
} else
|
||||
} else {
|
||||
update_peer_seq(mdev, peer_seq);
|
||||
spin_lock_irq(&mdev->tconn->req_lock);
|
||||
}
|
||||
list_add(&peer_req->w.list, &mdev->active_ee);
|
||||
spin_unlock_irq(&mdev->tconn->req_lock);
|
||||
|
||||
@ -4132,7 +4125,11 @@ recv_bm_rle_bits(struct drbd_conf *mdev,
|
||||
(unsigned int)bs.buf_len);
|
||||
return -EIO;
|
||||
}
|
||||
look_ahead >>= bits;
|
||||
/* if we consumed all 64 bits, assign 0; >> 64 is "undefined"; */
|
||||
if (likely(bits < 64))
|
||||
look_ahead >>= bits;
|
||||
else
|
||||
look_ahead = 0;
|
||||
have -= bits;
|
||||
|
||||
bits = bitstream_get_bits(&bs, &tmp, 64 - have);
|
||||
|
@ -1306,6 +1306,7 @@ int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct
|
||||
int backing_limit;
|
||||
|
||||
if (bio_size && get_ldev(mdev)) {
|
||||
unsigned int max_hw_sectors = queue_max_hw_sectors(q);
|
||||
struct request_queue * const b =
|
||||
mdev->ldev->backing_bdev->bd_disk->queue;
|
||||
if (b->merge_bvec_fn) {
|
||||
@ -1313,6 +1314,8 @@ int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct
|
||||
limit = min(limit, backing_limit);
|
||||
}
|
||||
put_ldev(mdev);
|
||||
if ((limit >> 9) > max_hw_sectors)
|
||||
limit = max_hw_sectors << 9;
|
||||
}
|
||||
return limit;
|
||||
}
|
||||
|
@ -894,13 +894,6 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
|
||||
|
||||
bio_list_init(&lo->lo_bio_list);
|
||||
|
||||
/*
|
||||
* set queue make_request_fn, and add limits based on lower level
|
||||
* device
|
||||
*/
|
||||
blk_queue_make_request(lo->lo_queue, loop_make_request);
|
||||
lo->lo_queue->queuedata = lo;
|
||||
|
||||
if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
|
||||
blk_queue_flush(lo->lo_queue, REQ_FLUSH);
|
||||
|
||||
@ -1618,6 +1611,8 @@ static int loop_add(struct loop_device **l, int i)
|
||||
if (!lo)
|
||||
goto out;
|
||||
|
||||
lo->lo_state = Lo_unbound;
|
||||
|
||||
/* allocate id, if @id >= 0, we're requesting that specific id */
|
||||
if (i >= 0) {
|
||||
err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
|
||||
@ -1635,6 +1630,12 @@ static int loop_add(struct loop_device **l, int i)
|
||||
if (!lo->lo_queue)
|
||||
goto out_free_idr;
|
||||
|
||||
/*
|
||||
* set queue make_request_fn
|
||||
*/
|
||||
blk_queue_make_request(lo->lo_queue, loop_make_request);
|
||||
lo->lo_queue->queuedata = lo;
|
||||
|
||||
disk = lo->lo_disk = alloc_disk(1 << part_shift);
|
||||
if (!disk)
|
||||
goto out_free_queue;
|
||||
|
@ -936,7 +936,7 @@ static int mg_probe(struct platform_device *plat_dev)
|
||||
goto probe_err_3b;
|
||||
}
|
||||
err = request_irq(host->irq, mg_irq,
|
||||
IRQF_DISABLED | IRQF_TRIGGER_RISING,
|
||||
IRQF_TRIGGER_RISING,
|
||||
MG_DEV_NAME, host);
|
||||
if (err) {
|
||||
printk(KERN_ERR "%s:%d fail (request_irq err=%d)\n",
|
||||
|
@ -126,64 +126,30 @@ struct mtip_compat_ide_task_request_s {
|
||||
static bool mtip_check_surprise_removal(struct pci_dev *pdev)
|
||||
{
|
||||
u16 vendor_id = 0;
|
||||
struct driver_data *dd = pci_get_drvdata(pdev);
|
||||
|
||||
if (dd->sr)
|
||||
return true;
|
||||
|
||||
/* Read the vendorID from the configuration space */
|
||||
pci_read_config_word(pdev, 0x00, &vendor_id);
|
||||
if (vendor_id == 0xFFFF)
|
||||
if (vendor_id == 0xFFFF) {
|
||||
dd->sr = true;
|
||||
if (dd->queue)
|
||||
set_bit(QUEUE_FLAG_DEAD, &dd->queue->queue_flags);
|
||||
else
|
||||
dev_warn(&dd->pdev->dev,
|
||||
"%s: dd->queue is NULL\n", __func__);
|
||||
if (dd->port) {
|
||||
set_bit(MTIP_PF_SR_CLEANUP_BIT, &dd->port->flags);
|
||||
wake_up_interruptible(&dd->port->svc_wait);
|
||||
} else
|
||||
dev_warn(&dd->pdev->dev,
|
||||
"%s: dd->port is NULL\n", __func__);
|
||||
return true; /* device removed */
|
||||
|
||||
return false; /* device present */
|
||||
}
|
||||
|
||||
/*
|
||||
* This function is called for clean the pending command in the
|
||||
* command slot during the surprise removal of device and return
|
||||
* error to the upper layer.
|
||||
*
|
||||
* @dd Pointer to the DRIVER_DATA structure.
|
||||
*
|
||||
* return value
|
||||
* None
|
||||
*/
|
||||
static void mtip_command_cleanup(struct driver_data *dd)
|
||||
{
|
||||
int group = 0, commandslot = 0, commandindex = 0;
|
||||
struct mtip_cmd *command;
|
||||
struct mtip_port *port = dd->port;
|
||||
static int in_progress;
|
||||
|
||||
if (in_progress)
|
||||
return;
|
||||
|
||||
in_progress = 1;
|
||||
|
||||
for (group = 0; group < 4; group++) {
|
||||
for (commandslot = 0; commandslot < 32; commandslot++) {
|
||||
if (!(port->allocated[group] & (1 << commandslot)))
|
||||
continue;
|
||||
|
||||
commandindex = group << 5 | commandslot;
|
||||
command = &port->commands[commandindex];
|
||||
|
||||
if (atomic_read(&command->active)
|
||||
&& (command->async_callback)) {
|
||||
command->async_callback(command->async_data,
|
||||
-ENODEV);
|
||||
command->async_callback = NULL;
|
||||
command->async_data = NULL;
|
||||
}
|
||||
|
||||
dma_unmap_sg(&port->dd->pdev->dev,
|
||||
command->sg,
|
||||
command->scatter_ents,
|
||||
command->direction);
|
||||
}
|
||||
}
|
||||
|
||||
up(&port->cmd_slot);
|
||||
|
||||
set_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag);
|
||||
in_progress = 0;
|
||||
return false; /* device present */
|
||||
}
|
||||
|
||||
/*
|
||||
@ -222,10 +188,7 @@ static int get_slot(struct mtip_port *port)
|
||||
}
|
||||
dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n");
|
||||
|
||||
if (mtip_check_surprise_removal(port->dd->pdev)) {
|
||||
/* Device not present, clean outstanding commands */
|
||||
mtip_command_cleanup(port->dd);
|
||||
}
|
||||
mtip_check_surprise_removal(port->dd->pdev);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -245,6 +208,107 @@ static inline void release_slot(struct mtip_port *port, int tag)
|
||||
smp_mb__after_clear_bit();
|
||||
}
|
||||
|
||||
/*
|
||||
* IO completion function.
|
||||
*
|
||||
* This completion function is called by the driver ISR when a
|
||||
* command that was issued by the kernel completes. It first calls the
|
||||
* asynchronous completion function which normally calls back into the block
|
||||
* layer passing the asynchronous callback data, then unmaps the
|
||||
* scatter list associated with the completed command, and finally
|
||||
* clears the allocated bit associated with the completed command.
|
||||
*
|
||||
* @port Pointer to the port data structure.
|
||||
* @tag Tag of the command.
|
||||
* @data Pointer to driver_data.
|
||||
* @status Completion status.
|
||||
*
|
||||
* return value
|
||||
* None
|
||||
*/
|
||||
static void mtip_async_complete(struct mtip_port *port,
|
||||
int tag,
|
||||
void *data,
|
||||
int status)
|
||||
{
|
||||
struct mtip_cmd *command;
|
||||
struct driver_data *dd = data;
|
||||
int cb_status = status ? -EIO : 0;
|
||||
|
||||
if (unlikely(!dd) || unlikely(!port))
|
||||
return;
|
||||
|
||||
command = &port->commands[tag];
|
||||
|
||||
if (unlikely(status == PORT_IRQ_TF_ERR)) {
|
||||
dev_warn(&port->dd->pdev->dev,
|
||||
"Command tag %d failed due to TFE\n", tag);
|
||||
}
|
||||
|
||||
/* Upper layer callback */
|
||||
if (likely(command->async_callback))
|
||||
command->async_callback(command->async_data, cb_status);
|
||||
|
||||
command->async_callback = NULL;
|
||||
command->comp_func = NULL;
|
||||
|
||||
/* Unmap the DMA scatter list entries */
|
||||
dma_unmap_sg(&dd->pdev->dev,
|
||||
command->sg,
|
||||
command->scatter_ents,
|
||||
command->direction);
|
||||
|
||||
/* Clear the allocated and active bits for the command */
|
||||
atomic_set(&port->commands[tag].active, 0);
|
||||
release_slot(port, tag);
|
||||
|
||||
up(&port->cmd_slot);
|
||||
}
|
||||
|
||||
/*
|
||||
* This function is called for clean the pending command in the
|
||||
* command slot during the surprise removal of device and return
|
||||
* error to the upper layer.
|
||||
*
|
||||
* @dd Pointer to the DRIVER_DATA structure.
|
||||
*
|
||||
* return value
|
||||
* None
|
||||
*/
|
||||
static void mtip_command_cleanup(struct driver_data *dd)
|
||||
{
|
||||
int tag = 0;
|
||||
struct mtip_cmd *cmd;
|
||||
struct mtip_port *port = dd->port;
|
||||
unsigned int num_cmd_slots = dd->slot_groups * 32;
|
||||
|
||||
if (!test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
|
||||
return;
|
||||
|
||||
if (!port)
|
||||
return;
|
||||
|
||||
cmd = &port->commands[MTIP_TAG_INTERNAL];
|
||||
if (atomic_read(&cmd->active))
|
||||
if (readl(port->cmd_issue[MTIP_TAG_INTERNAL]) &
|
||||
(1 << MTIP_TAG_INTERNAL))
|
||||
if (cmd->comp_func)
|
||||
cmd->comp_func(port, MTIP_TAG_INTERNAL,
|
||||
cmd->comp_data, -ENODEV);
|
||||
|
||||
while (1) {
|
||||
tag = find_next_bit(port->allocated, num_cmd_slots, tag);
|
||||
if (tag >= num_cmd_slots)
|
||||
break;
|
||||
|
||||
cmd = &port->commands[tag];
|
||||
if (atomic_read(&cmd->active))
|
||||
mtip_async_complete(port, tag, dd, -ENODEV);
|
||||
}
|
||||
|
||||
set_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag);
|
||||
}
|
||||
|
||||
/*
|
||||
* Reset the HBA (without sleeping)
|
||||
*
|
||||
@ -584,6 +648,9 @@ static void mtip_timeout_function(unsigned long int data)
|
||||
if (unlikely(!port))
|
||||
return;
|
||||
|
||||
if (unlikely(port->dd->sr))
|
||||
return;
|
||||
|
||||
if (test_bit(MTIP_DDF_RESUME_BIT, &port->dd->dd_flag)) {
|
||||
mod_timer(&port->cmd_timer,
|
||||
jiffies + msecs_to_jiffies(30000));
|
||||
@ -674,66 +741,6 @@ static void mtip_timeout_function(unsigned long int data)
|
||||
jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
|
||||
}
|
||||
|
||||
/*
|
||||
* IO completion function.
|
||||
*
|
||||
* This completion function is called by the driver ISR when a
|
||||
* command that was issued by the kernel completes. It first calls the
|
||||
* asynchronous completion function which normally calls back into the block
|
||||
* layer passing the asynchronous callback data, then unmaps the
|
||||
* scatter list associated with the completed command, and finally
|
||||
* clears the allocated bit associated with the completed command.
|
||||
*
|
||||
* @port Pointer to the port data structure.
|
||||
* @tag Tag of the command.
|
||||
* @data Pointer to driver_data.
|
||||
* @status Completion status.
|
||||
*
|
||||
* return value
|
||||
* None
|
||||
*/
|
||||
static void mtip_async_complete(struct mtip_port *port,
|
||||
int tag,
|
||||
void *data,
|
||||
int status)
|
||||
{
|
||||
struct mtip_cmd *command;
|
||||
struct driver_data *dd = data;
|
||||
int cb_status = status ? -EIO : 0;
|
||||
|
||||
if (unlikely(!dd) || unlikely(!port))
|
||||
return;
|
||||
|
||||
command = &port->commands[tag];
|
||||
|
||||
if (unlikely(status == PORT_IRQ_TF_ERR)) {
|
||||
dev_warn(&port->dd->pdev->dev,
|
||||
"Command tag %d failed due to TFE\n", tag);
|
||||
}
|
||||
|
||||
/* Upper layer callback */
|
||||
if (likely(command->async_callback))
|
||||
command->async_callback(command->async_data, cb_status);
|
||||
|
||||
command->async_callback = NULL;
|
||||
command->comp_func = NULL;
|
||||
|
||||
/* Unmap the DMA scatter list entries */
|
||||
dma_unmap_sg(&dd->pdev->dev,
|
||||
command->sg,
|
||||
command->scatter_ents,
|
||||
command->direction);
|
||||
|
||||
/* Clear the allocated and active bits for the command */
|
||||
atomic_set(&port->commands[tag].active, 0);
|
||||
release_slot(port, tag);
|
||||
|
||||
if (unlikely(command->unaligned))
|
||||
up(&port->cmd_slot_unal);
|
||||
else
|
||||
up(&port->cmd_slot);
|
||||
}
|
||||
|
||||
/*
|
||||
* Internal command completion callback function.
|
||||
*
|
||||
@ -854,7 +861,6 @@ static void mtip_handle_tfe(struct driver_data *dd)
|
||||
"Missing completion func for tag %d",
|
||||
tag);
|
||||
if (mtip_check_surprise_removal(dd->pdev)) {
|
||||
mtip_command_cleanup(dd);
|
||||
/* don't proceed further */
|
||||
return;
|
||||
}
|
||||
@ -1018,14 +1024,12 @@ static inline void mtip_workq_sdbfx(struct mtip_port *port, int group,
|
||||
command->comp_data,
|
||||
0);
|
||||
} else {
|
||||
dev_warn(&dd->pdev->dev,
|
||||
"Null completion "
|
||||
"for tag %d",
|
||||
dev_dbg(&dd->pdev->dev,
|
||||
"Null completion for tag %d",
|
||||
tag);
|
||||
|
||||
if (mtip_check_surprise_removal(
|
||||
dd->pdev)) {
|
||||
mtip_command_cleanup(dd);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -1145,7 +1149,6 @@ static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
|
||||
|
||||
if (unlikely(port_stat & PORT_IRQ_ERR)) {
|
||||
if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
|
||||
mtip_command_cleanup(dd);
|
||||
/* don't proceed further */
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
@ -2806,34 +2809,51 @@ static ssize_t show_device_status(struct device_driver *drv, char *buf)
|
||||
static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf,
|
||||
size_t len, loff_t *offset)
|
||||
{
|
||||
struct driver_data *dd = (struct driver_data *)f->private_data;
|
||||
int size = *offset;
|
||||
char buf[MTIP_DFS_MAX_BUF_SIZE];
|
||||
char *buf;
|
||||
int rv = 0;
|
||||
|
||||
if (!len || *offset)
|
||||
return 0;
|
||||
|
||||
buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
|
||||
if (!buf) {
|
||||
dev_err(&dd->pdev->dev,
|
||||
"Memory allocation: status buffer\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
size += show_device_status(NULL, buf);
|
||||
|
||||
*offset = size <= len ? size : len;
|
||||
size = copy_to_user(ubuf, buf, *offset);
|
||||
if (size)
|
||||
return -EFAULT;
|
||||
rv = -EFAULT;
|
||||
|
||||
return *offset;
|
||||
kfree(buf);
|
||||
return rv ? rv : *offset;
|
||||
}
|
||||
|
||||
static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
|
||||
size_t len, loff_t *offset)
|
||||
{
|
||||
struct driver_data *dd = (struct driver_data *)f->private_data;
|
||||
char buf[MTIP_DFS_MAX_BUF_SIZE];
|
||||
char *buf;
|
||||
u32 group_allocated;
|
||||
int size = *offset;
|
||||
int n;
|
||||
int n, rv = 0;
|
||||
|
||||
if (!len || size)
|
||||
return 0;
|
||||
|
||||
buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
|
||||
if (!buf) {
|
||||
dev_err(&dd->pdev->dev,
|
||||
"Memory allocation: register buffer\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
size += sprintf(&buf[size], "H/ S ACTive : [ 0x");
|
||||
|
||||
for (n = dd->slot_groups-1; n >= 0; n--)
|
||||
@ -2888,21 +2908,30 @@ static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
|
||||
*offset = size <= len ? size : len;
|
||||
size = copy_to_user(ubuf, buf, *offset);
|
||||
if (size)
|
||||
return -EFAULT;
|
||||
rv = -EFAULT;
|
||||
|
||||
return *offset;
|
||||
kfree(buf);
|
||||
return rv ? rv : *offset;
|
||||
}
|
||||
|
||||
static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
|
||||
size_t len, loff_t *offset)
|
||||
{
|
||||
struct driver_data *dd = (struct driver_data *)f->private_data;
|
||||
char buf[MTIP_DFS_MAX_BUF_SIZE];
|
||||
char *buf;
|
||||
int size = *offset;
|
||||
int rv = 0;
|
||||
|
||||
if (!len || size)
|
||||
return 0;
|
||||
|
||||
buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
|
||||
if (!buf) {
|
||||
dev_err(&dd->pdev->dev,
|
||||
"Memory allocation: flag buffer\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n",
|
||||
dd->port->flags);
|
||||
size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n",
|
||||
@ -2911,9 +2940,10 @@ static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
|
||||
*offset = size <= len ? size : len;
|
||||
size = copy_to_user(ubuf, buf, *offset);
|
||||
if (size)
|
||||
return -EFAULT;
|
||||
rv = -EFAULT;
|
||||
|
||||
return *offset;
|
||||
kfree(buf);
|
||||
return rv ? rv : *offset;
|
||||
}
|
||||
|
||||
static const struct file_operations mtip_device_status_fops = {
|
||||
@ -3006,6 +3036,46 @@ static void mtip_hw_debugfs_exit(struct driver_data *dd)
|
||||
debugfs_remove_recursive(dd->dfs_node);
|
||||
}
|
||||
|
||||
static int mtip_free_orphan(struct driver_data *dd)
|
||||
{
|
||||
struct kobject *kobj;
|
||||
|
||||
if (dd->bdev) {
|
||||
if (dd->bdev->bd_holders >= 1)
|
||||
return -2;
|
||||
|
||||
bdput(dd->bdev);
|
||||
dd->bdev = NULL;
|
||||
}
|
||||
|
||||
mtip_hw_debugfs_exit(dd);
|
||||
|
||||
spin_lock(&rssd_index_lock);
|
||||
ida_remove(&rssd_index_ida, dd->index);
|
||||
spin_unlock(&rssd_index_lock);
|
||||
|
||||
if (!test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag) &&
|
||||
test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) {
|
||||
put_disk(dd->disk);
|
||||
} else {
|
||||
if (dd->disk) {
|
||||
kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
|
||||
if (kobj) {
|
||||
mtip_hw_sysfs_exit(dd, kobj);
|
||||
kobject_put(kobj);
|
||||
}
|
||||
del_gendisk(dd->disk);
|
||||
dd->disk = NULL;
|
||||
}
|
||||
if (dd->queue) {
|
||||
dd->queue->queuedata = NULL;
|
||||
blk_cleanup_queue(dd->queue);
|
||||
dd->queue = NULL;
|
||||
}
|
||||
}
|
||||
kfree(dd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform any init/resume time hardware setup
|
||||
@ -3154,6 +3224,7 @@ static int mtip_service_thread(void *data)
|
||||
unsigned long slot, slot_start, slot_wrap;
|
||||
unsigned int num_cmd_slots = dd->slot_groups * 32;
|
||||
struct mtip_port *port = dd->port;
|
||||
int ret;
|
||||
|
||||
while (1) {
|
||||
/*
|
||||
@ -3164,13 +3235,18 @@ static int mtip_service_thread(void *data)
|
||||
!(port->flags & MTIP_PF_PAUSE_IO));
|
||||
|
||||
if (kthread_should_stop())
|
||||
goto st_out;
|
||||
|
||||
set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
|
||||
|
||||
/* If I am an orphan, start self cleanup */
|
||||
if (test_bit(MTIP_PF_SR_CLEANUP_BIT, &port->flags))
|
||||
break;
|
||||
|
||||
if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
|
||||
&dd->dd_flag)))
|
||||
break;
|
||||
goto st_out;
|
||||
|
||||
set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
|
||||
if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
|
||||
slot = 1;
|
||||
/* used to restrict the loop to one iteration */
|
||||
@ -3201,7 +3277,7 @@ static int mtip_service_thread(void *data)
|
||||
|
||||
clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
|
||||
} else if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) {
|
||||
if (!mtip_ftl_rebuild_poll(dd))
|
||||
if (mtip_ftl_rebuild_poll(dd) < 0)
|
||||
set_bit(MTIP_DDF_REBUILD_FAILED_BIT,
|
||||
&dd->dd_flag);
|
||||
clear_bit(MTIP_PF_REBUILD_BIT, &port->flags);
|
||||
@ -3209,8 +3285,30 @@ static int mtip_service_thread(void *data)
|
||||
clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
|
||||
|
||||
if (test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
|
||||
break;
|
||||
goto st_out;
|
||||
}
|
||||
|
||||
/* wait for pci remove to exit */
|
||||
while (1) {
|
||||
if (test_bit(MTIP_DDF_REMOVE_DONE_BIT, &dd->dd_flag))
|
||||
break;
|
||||
msleep_interruptible(1000);
|
||||
if (kthread_should_stop())
|
||||
goto st_out;
|
||||
}
|
||||
|
||||
while (1) {
|
||||
ret = mtip_free_orphan(dd);
|
||||
if (!ret) {
|
||||
/* NOTE: All data structures are invalid, do not
|
||||
* access any here */
|
||||
return 0;
|
||||
}
|
||||
msleep_interruptible(1000);
|
||||
if (kthread_should_stop())
|
||||
goto st_out;
|
||||
}
|
||||
st_out:
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -3437,13 +3535,13 @@ static int mtip_hw_init(struct driver_data *dd)
|
||||
rv = -EFAULT;
|
||||
goto out3;
|
||||
}
|
||||
mtip_dump_identify(dd->port);
|
||||
|
||||
if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
|
||||
MTIP_FTL_REBUILD_MAGIC) {
|
||||
set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags);
|
||||
return MTIP_FTL_REBUILD_MAGIC;
|
||||
}
|
||||
mtip_dump_identify(dd->port);
|
||||
|
||||
/* check write protect, over temp and rebuild statuses */
|
||||
rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
|
||||
@ -3467,8 +3565,8 @@ static int mtip_hw_init(struct driver_data *dd)
|
||||
}
|
||||
if (buf[288] == 0xBF) {
|
||||
dev_info(&dd->pdev->dev,
|
||||
"Drive indicates rebuild has failed.\n");
|
||||
/* TODO */
|
||||
"Drive is in security locked state.\n");
|
||||
set_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag);
|
||||
}
|
||||
}
|
||||
|
||||
@ -3523,9 +3621,8 @@ static int mtip_hw_exit(struct driver_data *dd)
|
||||
* Send standby immediate (E0h) to the drive so that it
|
||||
* saves its state.
|
||||
*/
|
||||
if (!test_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag)) {
|
||||
|
||||
if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags))
|
||||
if (!dd->sr) {
|
||||
if (!test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))
|
||||
if (mtip_standby_immediate(dd->port))
|
||||
dev_warn(&dd->pdev->dev,
|
||||
"STANDBY IMMEDIATE failed\n");
|
||||
@ -3551,6 +3648,7 @@ static int mtip_hw_exit(struct driver_data *dd)
|
||||
dd->port->command_list_dma);
|
||||
/* Free the memory allocated for the for structure. */
|
||||
kfree(dd->port);
|
||||
dd->port = NULL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -3572,7 +3670,8 @@ static int mtip_hw_shutdown(struct driver_data *dd)
|
||||
* Send standby immediate (E0h) to the drive so that it
|
||||
* saves its state.
|
||||
*/
|
||||
mtip_standby_immediate(dd->port);
|
||||
if (!dd->sr && dd->port)
|
||||
mtip_standby_immediate(dd->port);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -3887,6 +3986,10 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio)
|
||||
bio_endio(bio, -ENODATA);
|
||||
return;
|
||||
}
|
||||
if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) {
|
||||
bio_endio(bio, -ENXIO);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (unlikely(bio->bi_rw & REQ_DISCARD)) {
|
||||
@ -4010,6 +4113,8 @@ static int mtip_block_initialize(struct driver_data *dd)
|
||||
dd->disk->private_data = dd;
|
||||
dd->index = index;
|
||||
|
||||
mtip_hw_debugfs_init(dd);
|
||||
|
||||
/*
|
||||
* if rebuild pending, start the service thread, and delay the block
|
||||
* queue creation and add_disk()
|
||||
@ -4068,6 +4173,7 @@ skip_create_disk:
|
||||
/* Enable the block device and add it to /dev */
|
||||
add_disk(dd->disk);
|
||||
|
||||
dd->bdev = bdget_disk(dd->disk, 0);
|
||||
/*
|
||||
* Now that the disk is active, initialize any sysfs attributes
|
||||
* managed by the protocol layer.
|
||||
@ -4077,7 +4183,6 @@ skip_create_disk:
|
||||
mtip_hw_sysfs_init(dd, kobj);
|
||||
kobject_put(kobj);
|
||||
}
|
||||
mtip_hw_debugfs_init(dd);
|
||||
|
||||
if (dd->mtip_svc_handler) {
|
||||
set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
|
||||
@ -4103,7 +4208,8 @@ start_service_thread:
|
||||
return rv;
|
||||
|
||||
kthread_run_error:
|
||||
mtip_hw_debugfs_exit(dd);
|
||||
bdput(dd->bdev);
|
||||
dd->bdev = NULL;
|
||||
|
||||
/* Delete our gendisk. This also removes the device from /dev */
|
||||
del_gendisk(dd->disk);
|
||||
@ -4112,6 +4218,7 @@ read_capacity_error:
|
||||
blk_cleanup_queue(dd->queue);
|
||||
|
||||
block_queue_alloc_init_error:
|
||||
mtip_hw_debugfs_exit(dd);
|
||||
disk_index_error:
|
||||
spin_lock(&rssd_index_lock);
|
||||
ida_remove(&rssd_index_ida, index);
|
||||
@ -4141,40 +4248,48 @@ static int mtip_block_remove(struct driver_data *dd)
|
||||
{
|
||||
struct kobject *kobj;
|
||||
|
||||
if (dd->mtip_svc_handler) {
|
||||
set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
|
||||
wake_up_interruptible(&dd->port->svc_wait);
|
||||
kthread_stop(dd->mtip_svc_handler);
|
||||
}
|
||||
if (!dd->sr) {
|
||||
mtip_hw_debugfs_exit(dd);
|
||||
|
||||
/* Clean up the sysfs attributes, if created */
|
||||
if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) {
|
||||
kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
|
||||
if (kobj) {
|
||||
mtip_hw_sysfs_exit(dd, kobj);
|
||||
kobject_put(kobj);
|
||||
if (dd->mtip_svc_handler) {
|
||||
set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
|
||||
wake_up_interruptible(&dd->port->svc_wait);
|
||||
kthread_stop(dd->mtip_svc_handler);
|
||||
}
|
||||
|
||||
/* Clean up the sysfs attributes, if created */
|
||||
if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) {
|
||||
kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
|
||||
if (kobj) {
|
||||
mtip_hw_sysfs_exit(dd, kobj);
|
||||
kobject_put(kobj);
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Delete our gendisk structure. This also removes the device
|
||||
* from /dev
|
||||
*/
|
||||
if (dd->bdev) {
|
||||
bdput(dd->bdev);
|
||||
dd->bdev = NULL;
|
||||
}
|
||||
if (dd->disk) {
|
||||
if (dd->disk->queue) {
|
||||
del_gendisk(dd->disk);
|
||||
blk_cleanup_queue(dd->queue);
|
||||
dd->queue = NULL;
|
||||
} else
|
||||
put_disk(dd->disk);
|
||||
}
|
||||
dd->disk = NULL;
|
||||
|
||||
spin_lock(&rssd_index_lock);
|
||||
ida_remove(&rssd_index_ida, dd->index);
|
||||
spin_unlock(&rssd_index_lock);
|
||||
} else {
|
||||
dev_info(&dd->pdev->dev, "device %s surprise removal\n",
|
||||
dd->disk->disk_name);
|
||||
}
|
||||
mtip_hw_debugfs_exit(dd);
|
||||
|
||||
/*
|
||||
* Delete our gendisk structure. This also removes the device
|
||||
* from /dev
|
||||
*/
|
||||
if (dd->disk) {
|
||||
if (dd->disk->queue)
|
||||
del_gendisk(dd->disk);
|
||||
else
|
||||
put_disk(dd->disk);
|
||||
}
|
||||
|
||||
spin_lock(&rssd_index_lock);
|
||||
ida_remove(&rssd_index_ida, dd->index);
|
||||
spin_unlock(&rssd_index_lock);
|
||||
|
||||
blk_cleanup_queue(dd->queue);
|
||||
dd->disk = NULL;
|
||||
dd->queue = NULL;
|
||||
|
||||
/* De-initialize the protocol layer. */
|
||||
mtip_hw_exit(dd);
|
||||
@ -4490,8 +4605,7 @@ done:
|
||||
static void mtip_pci_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct driver_data *dd = pci_get_drvdata(pdev);
|
||||
int counter = 0;
|
||||
unsigned long flags;
|
||||
unsigned long flags, to;
|
||||
|
||||
set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
|
||||
|
||||
@ -4500,17 +4614,22 @@ static void mtip_pci_remove(struct pci_dev *pdev)
|
||||
list_add(&dd->remove_list, &removing_list);
|
||||
spin_unlock_irqrestore(&dev_lock, flags);
|
||||
|
||||
if (mtip_check_surprise_removal(pdev)) {
|
||||
while (!test_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag)) {
|
||||
counter++;
|
||||
msleep(20);
|
||||
if (counter == 10) {
|
||||
/* Cleanup the outstanding commands */
|
||||
mtip_command_cleanup(dd);
|
||||
break;
|
||||
}
|
||||
}
|
||||
mtip_check_surprise_removal(pdev);
|
||||
synchronize_irq(dd->pdev->irq);
|
||||
|
||||
/* Spin until workers are done */
|
||||
to = jiffies + msecs_to_jiffies(4000);
|
||||
do {
|
||||
msleep(20);
|
||||
} while (atomic_read(&dd->irq_workers_active) != 0 &&
|
||||
time_before(jiffies, to));
|
||||
|
||||
if (atomic_read(&dd->irq_workers_active) != 0) {
|
||||
dev_warn(&dd->pdev->dev,
|
||||
"Completion workers still active!\n");
|
||||
}
|
||||
/* Cleanup the outstanding commands */
|
||||
mtip_command_cleanup(dd);
|
||||
|
||||
/* Clean up the block layer. */
|
||||
mtip_block_remove(dd);
|
||||
@ -4529,8 +4648,15 @@ static void mtip_pci_remove(struct pci_dev *pdev)
|
||||
list_del_init(&dd->remove_list);
|
||||
spin_unlock_irqrestore(&dev_lock, flags);
|
||||
|
||||
kfree(dd);
|
||||
if (!dd->sr)
|
||||
kfree(dd);
|
||||
else
|
||||
set_bit(MTIP_DDF_REMOVE_DONE_BIT, &dd->dd_flag);
|
||||
|
||||
pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
|
||||
pci_set_drvdata(pdev, NULL);
|
||||
pci_dev_put(pdev);
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -140,6 +140,7 @@ enum {
|
||||
MTIP_PF_SVC_THD_ACTIVE_BIT = 4,
|
||||
MTIP_PF_ISSUE_CMDS_BIT = 5,
|
||||
MTIP_PF_REBUILD_BIT = 6,
|
||||
MTIP_PF_SR_CLEANUP_BIT = 7,
|
||||
MTIP_PF_SVC_THD_STOP_BIT = 8,
|
||||
|
||||
/* below are bit numbers in 'dd_flag' defined in driver_data */
|
||||
@ -147,15 +148,18 @@ enum {
|
||||
MTIP_DDF_REMOVE_PENDING_BIT = 1,
|
||||
MTIP_DDF_OVER_TEMP_BIT = 2,
|
||||
MTIP_DDF_WRITE_PROTECT_BIT = 3,
|
||||
MTIP_DDF_STOP_IO = ((1 << MTIP_DDF_REMOVE_PENDING_BIT) |
|
||||
(1 << MTIP_DDF_SEC_LOCK_BIT) |
|
||||
(1 << MTIP_DDF_OVER_TEMP_BIT) |
|
||||
(1 << MTIP_DDF_WRITE_PROTECT_BIT)),
|
||||
|
||||
MTIP_DDF_REMOVE_DONE_BIT = 4,
|
||||
MTIP_DDF_CLEANUP_BIT = 5,
|
||||
MTIP_DDF_RESUME_BIT = 6,
|
||||
MTIP_DDF_INIT_DONE_BIT = 7,
|
||||
MTIP_DDF_REBUILD_FAILED_BIT = 8,
|
||||
|
||||
MTIP_DDF_STOP_IO = ((1 << MTIP_DDF_REMOVE_PENDING_BIT) |
|
||||
(1 << MTIP_DDF_SEC_LOCK_BIT) |
|
||||
(1 << MTIP_DDF_OVER_TEMP_BIT) |
|
||||
(1 << MTIP_DDF_WRITE_PROTECT_BIT) |
|
||||
(1 << MTIP_DDF_REBUILD_FAILED_BIT)),
|
||||
|
||||
};
|
||||
|
||||
struct smart_attr {
|
||||
@ -499,6 +503,8 @@ struct driver_data {
|
||||
|
||||
bool trim_supp; /* flag indicating trim support */
|
||||
|
||||
bool sr;
|
||||
|
||||
int numa_node; /* NUMA support */
|
||||
|
||||
char workq_name[32];
|
||||
@ -511,6 +517,8 @@ struct driver_data {
|
||||
|
||||
int isr_binding;
|
||||
|
||||
struct block_device *bdev;
|
||||
|
||||
int unal_qdepth; /* qdepth of unaligned IO queue */
|
||||
|
||||
struct list_head online_list; /* linkage for online list */
|
||||
|
@ -473,45 +473,31 @@ static void pkt_debugfs_dev_new(struct pktcdvd_device *pd)
|
||||
{
|
||||
if (!pkt_debugfs_root)
|
||||
return;
|
||||
pd->dfs_f_info = NULL;
|
||||
pd->dfs_d_root = debugfs_create_dir(pd->name, pkt_debugfs_root);
|
||||
if (IS_ERR(pd->dfs_d_root)) {
|
||||
pd->dfs_d_root = NULL;
|
||||
if (!pd->dfs_d_root)
|
||||
return;
|
||||
}
|
||||
|
||||
pd->dfs_f_info = debugfs_create_file("info", S_IRUGO,
|
||||
pd->dfs_d_root, pd, &debug_fops);
|
||||
if (IS_ERR(pd->dfs_f_info)) {
|
||||
pd->dfs_f_info = NULL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
static void pkt_debugfs_dev_remove(struct pktcdvd_device *pd)
|
||||
{
|
||||
if (!pkt_debugfs_root)
|
||||
return;
|
||||
if (pd->dfs_f_info)
|
||||
debugfs_remove(pd->dfs_f_info);
|
||||
debugfs_remove(pd->dfs_f_info);
|
||||
debugfs_remove(pd->dfs_d_root);
|
||||
pd->dfs_f_info = NULL;
|
||||
if (pd->dfs_d_root)
|
||||
debugfs_remove(pd->dfs_d_root);
|
||||
pd->dfs_d_root = NULL;
|
||||
}
|
||||
|
||||
static void pkt_debugfs_init(void)
|
||||
{
|
||||
pkt_debugfs_root = debugfs_create_dir(DRIVER_NAME, NULL);
|
||||
if (IS_ERR(pkt_debugfs_root)) {
|
||||
pkt_debugfs_root = NULL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
static void pkt_debugfs_cleanup(void)
|
||||
{
|
||||
if (!pkt_debugfs_root)
|
||||
return;
|
||||
debugfs_remove(pkt_debugfs_root);
|
||||
pkt_debugfs_root = NULL;
|
||||
}
|
||||
|
@ -654,7 +654,8 @@ static void rsxx_eeh_failure(struct pci_dev *dev)
|
||||
for (i = 0; i < card->n_targets; i++) {
|
||||
spin_lock_bh(&card->ctrl[i].queue_lock);
|
||||
cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
|
||||
&card->ctrl[i].queue);
|
||||
&card->ctrl[i].queue,
|
||||
COMPLETE_DMA);
|
||||
spin_unlock_bh(&card->ctrl[i].queue_lock);
|
||||
|
||||
cnt += rsxx_dma_cancel(&card->ctrl[i]);
|
||||
@ -748,10 +749,6 @@ static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
|
||||
|
||||
card->eeh_state = 0;
|
||||
|
||||
st = rsxx_eeh_remap_dmas(card);
|
||||
if (st)
|
||||
goto failed_remap_dmas;
|
||||
|
||||
spin_lock_irqsave(&card->irq_lock, flags);
|
||||
if (card->n_targets & RSXX_MAX_TARGETS)
|
||||
rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
|
||||
@ -778,7 +775,6 @@ static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
|
||||
return PCI_ERS_RESULT_RECOVERED;
|
||||
|
||||
failed_hw_buffers_init:
|
||||
failed_remap_dmas:
|
||||
for (i = 0; i < card->n_targets; i++) {
|
||||
if (card->ctrl[i].status.buf)
|
||||
pci_free_consistent(card->dev,
|
||||
|
@ -295,13 +295,15 @@ int rsxx_setup_dev(struct rsxx_cardinfo *card)
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
blk_size = card->config.data.block_size;
|
||||
if (card->config_valid) {
|
||||
blk_size = card->config.data.block_size;
|
||||
blk_queue_dma_alignment(card->queue, blk_size - 1);
|
||||
blk_queue_logical_block_size(card->queue, blk_size);
|
||||
}
|
||||
|
||||
blk_queue_make_request(card->queue, rsxx_make_request);
|
||||
blk_queue_bounce_limit(card->queue, BLK_BOUNCE_ANY);
|
||||
blk_queue_dma_alignment(card->queue, blk_size - 1);
|
||||
blk_queue_max_hw_sectors(card->queue, blkdev_max_hw_sectors);
|
||||
blk_queue_logical_block_size(card->queue, blk_size);
|
||||
blk_queue_physical_block_size(card->queue, RSXX_HW_BLK_SIZE);
|
||||
|
||||
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, card->queue);
|
||||
|
@ -221,6 +221,21 @@ static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
|
||||
}
|
||||
|
||||
/*----------------- RSXX DMA Handling -------------------*/
|
||||
static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma)
|
||||
{
|
||||
if (dma->cmd != HW_CMD_BLK_DISCARD) {
|
||||
if (!pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) {
|
||||
pci_unmap_page(ctrl->card->dev, dma->dma_addr,
|
||||
get_dma_size(dma),
|
||||
dma->cmd == HW_CMD_BLK_WRITE ?
|
||||
PCI_DMA_TODEVICE :
|
||||
PCI_DMA_FROMDEVICE);
|
||||
}
|
||||
}
|
||||
|
||||
kmem_cache_free(rsxx_dma_pool, dma);
|
||||
}
|
||||
|
||||
static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
|
||||
struct rsxx_dma *dma,
|
||||
unsigned int status)
|
||||
@ -232,21 +247,14 @@ static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
|
||||
if (status & DMA_CANCELLED)
|
||||
ctrl->stats.dma_cancelled++;
|
||||
|
||||
if (dma->dma_addr)
|
||||
pci_unmap_page(ctrl->card->dev, dma->dma_addr,
|
||||
get_dma_size(dma),
|
||||
dma->cmd == HW_CMD_BLK_WRITE ?
|
||||
PCI_DMA_TODEVICE :
|
||||
PCI_DMA_FROMDEVICE);
|
||||
|
||||
if (dma->cb)
|
||||
dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
|
||||
|
||||
kmem_cache_free(rsxx_dma_pool, dma);
|
||||
rsxx_free_dma(ctrl, dma);
|
||||
}
|
||||
|
||||
int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
|
||||
struct list_head *q)
|
||||
struct list_head *q, unsigned int done)
|
||||
{
|
||||
struct rsxx_dma *dma;
|
||||
struct rsxx_dma *tmp;
|
||||
@ -254,7 +262,10 @@ int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
|
||||
|
||||
list_for_each_entry_safe(dma, tmp, q, list) {
|
||||
list_del(&dma->list);
|
||||
rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
|
||||
if (done & COMPLETE_DMA)
|
||||
rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
|
||||
else
|
||||
rsxx_free_dma(ctrl, dma);
|
||||
cnt++;
|
||||
}
|
||||
|
||||
@ -370,7 +381,7 @@ static void dma_engine_stalled(unsigned long data)
|
||||
|
||||
/* Clean up the DMA queue */
|
||||
spin_lock(&ctrl->queue_lock);
|
||||
cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
|
||||
cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
|
||||
spin_unlock(&ctrl->queue_lock);
|
||||
|
||||
cnt += rsxx_dma_cancel(ctrl);
|
||||
@ -388,6 +399,7 @@ static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
|
||||
int tag;
|
||||
int cmds_pending = 0;
|
||||
struct hw_cmd *hw_cmd_buf;
|
||||
int dir;
|
||||
|
||||
hw_cmd_buf = ctrl->cmd.buf;
|
||||
|
||||
@ -424,6 +436,31 @@ static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dma->cmd != HW_CMD_BLK_DISCARD) {
|
||||
if (dma->cmd == HW_CMD_BLK_WRITE)
|
||||
dir = PCI_DMA_TODEVICE;
|
||||
else
|
||||
dir = PCI_DMA_FROMDEVICE;
|
||||
|
||||
/*
|
||||
* The function pci_map_page is placed here because we
|
||||
* can only, by design, issue up to 255 commands to the
|
||||
* hardware at one time per DMA channel. So the maximum
|
||||
* amount of mapped memory would be 255 * 4 channels *
|
||||
* 4096 Bytes which is less than 2GB, the limit of a x8
|
||||
* Non-HWWD PCIe slot. This way the pci_map_page
|
||||
* function should never fail because of a lack of
|
||||
* mappable memory.
|
||||
*/
|
||||
dma->dma_addr = pci_map_page(ctrl->card->dev, dma->page,
|
||||
dma->pg_off, dma->sub_page.cnt << 9, dir);
|
||||
if (pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) {
|
||||
push_tracker(ctrl->trackers, tag);
|
||||
rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
set_tracker_dma(ctrl->trackers, tag, dma);
|
||||
hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
|
||||
hw_cmd_buf[ctrl->cmd.idx].tag = tag;
|
||||
@ -620,14 +657,6 @@ static int rsxx_queue_dma(struct rsxx_cardinfo *card,
|
||||
if (!dma)
|
||||
return -ENOMEM;
|
||||
|
||||
dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
|
||||
dir ? PCI_DMA_TODEVICE :
|
||||
PCI_DMA_FROMDEVICE);
|
||||
if (!dma->dma_addr) {
|
||||
kmem_cache_free(rsxx_dma_pool, dma);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
|
||||
dma->laddr = laddr;
|
||||
dma->sub_page.off = (dma_off >> 9);
|
||||
@ -736,11 +765,9 @@ int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
|
||||
return 0;
|
||||
|
||||
bvec_err:
|
||||
for (i = 0; i < card->n_targets; i++) {
|
||||
spin_lock_bh(&card->ctrl[i].queue_lock);
|
||||
rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]);
|
||||
spin_unlock_bh(&card->ctrl[i].queue_lock);
|
||||
}
|
||||
for (i = 0; i < card->n_targets; i++)
|
||||
rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
|
||||
FREE_DMA);
|
||||
|
||||
return st;
|
||||
}
|
||||
@ -990,7 +1017,7 @@ void rsxx_dma_destroy(struct rsxx_cardinfo *card)
|
||||
|
||||
/* Clean up the DMA queue */
|
||||
spin_lock_bh(&ctrl->queue_lock);
|
||||
rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
|
||||
rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
|
||||
spin_unlock_bh(&ctrl->queue_lock);
|
||||
|
||||
rsxx_dma_cancel(ctrl);
|
||||
@ -1032,6 +1059,14 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
|
||||
else
|
||||
card->ctrl[i].stats.reads_issued--;
|
||||
|
||||
if (dma->cmd != HW_CMD_BLK_DISCARD) {
|
||||
pci_unmap_page(card->dev, dma->dma_addr,
|
||||
get_dma_size(dma),
|
||||
dma->cmd == HW_CMD_BLK_WRITE ?
|
||||
PCI_DMA_TODEVICE :
|
||||
PCI_DMA_FROMDEVICE);
|
||||
}
|
||||
|
||||
list_add_tail(&dma->list, &issued_dmas[i]);
|
||||
push_tracker(card->ctrl[i].trackers, j);
|
||||
cnt++;
|
||||
@ -1043,15 +1078,6 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
|
||||
atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
|
||||
card->ctrl[i].stats.sw_q_depth += cnt;
|
||||
card->ctrl[i].e_cnt = 0;
|
||||
|
||||
list_for_each_entry(dma, &card->ctrl[i].queue, list) {
|
||||
if (dma->dma_addr)
|
||||
pci_unmap_page(card->dev, dma->dma_addr,
|
||||
get_dma_size(dma),
|
||||
dma->cmd == HW_CMD_BLK_WRITE ?
|
||||
PCI_DMA_TODEVICE :
|
||||
PCI_DMA_FROMDEVICE);
|
||||
}
|
||||
spin_unlock_bh(&card->ctrl[i].queue_lock);
|
||||
}
|
||||
|
||||
@ -1060,31 +1086,6 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
|
||||
{
|
||||
struct rsxx_dma *dma;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < card->n_targets; i++) {
|
||||
spin_lock_bh(&card->ctrl[i].queue_lock);
|
||||
list_for_each_entry(dma, &card->ctrl[i].queue, list) {
|
||||
dma->dma_addr = pci_map_page(card->dev, dma->page,
|
||||
dma->pg_off, get_dma_size(dma),
|
||||
dma->cmd == HW_CMD_BLK_WRITE ?
|
||||
PCI_DMA_TODEVICE :
|
||||
PCI_DMA_FROMDEVICE);
|
||||
if (!dma->dma_addr) {
|
||||
spin_unlock_bh(&card->ctrl[i].queue_lock);
|
||||
kmem_cache_free(rsxx_dma_pool, dma);
|
||||
return -ENOMEM;
|
||||
}
|
||||
}
|
||||
spin_unlock_bh(&card->ctrl[i].queue_lock);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rsxx_dma_init(void)
|
||||
{
|
||||
rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
|
||||
|
@ -52,7 +52,7 @@ struct proc_cmd;
|
||||
#define RS70_PCI_REV_SUPPORTED 4
|
||||
|
||||
#define DRIVER_NAME "rsxx"
|
||||
#define DRIVER_VERSION "4.0"
|
||||
#define DRIVER_VERSION "4.0.3.2516"
|
||||
|
||||
/* Block size is 4096 */
|
||||
#define RSXX_HW_BLK_SHIFT 12
|
||||
@ -345,6 +345,11 @@ enum rsxx_creg_stat {
|
||||
CREG_STAT_TAG_MASK = 0x0000ff00,
|
||||
};
|
||||
|
||||
enum rsxx_dma_finish {
|
||||
FREE_DMA = 0x0,
|
||||
COMPLETE_DMA = 0x1,
|
||||
};
|
||||
|
||||
static inline unsigned int CREG_DATA(int N)
|
||||
{
|
||||
return CREG_DATA0 + (N << 2);
|
||||
@ -379,7 +384,9 @@ typedef void (*rsxx_dma_cb)(struct rsxx_cardinfo *card,
|
||||
int rsxx_dma_setup(struct rsxx_cardinfo *card);
|
||||
void rsxx_dma_destroy(struct rsxx_cardinfo *card);
|
||||
int rsxx_dma_init(void);
|
||||
int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl, struct list_head *q);
|
||||
int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
|
||||
struct list_head *q,
|
||||
unsigned int done);
|
||||
int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl);
|
||||
void rsxx_dma_cleanup(void);
|
||||
void rsxx_dma_queue_reset(struct rsxx_cardinfo *card);
|
||||
|
5432
drivers/block/skd_main.c
Normal file
5432
drivers/block/skd_main.c
Normal file
File diff suppressed because it is too large
Load Diff
330
drivers/block/skd_s1120.h
Normal file
330
drivers/block/skd_s1120.h
Normal file
@ -0,0 +1,330 @@
|
||||
/* Copyright 2012 STEC, Inc.
|
||||
*
|
||||
* This file is licensed under the terms of the 3-clause
|
||||
* BSD License (http://opensource.org/licenses/BSD-3-Clause)
|
||||
* or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
|
||||
* at your option. Both licenses are also available in the LICENSE file
|
||||
* distributed with this project. This file may not be copied, modified,
|
||||
* or distributed except in accordance with those terms.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef SKD_S1120_H
|
||||
#define SKD_S1120_H
|
||||
|
||||
#pragma pack(push, s1120_h, 1)
|
||||
|
||||
/*
|
||||
* Q-channel, 64-bit r/w
|
||||
*/
|
||||
#define FIT_Q_COMMAND 0x400u
|
||||
#define FIT_QCMD_QID_MASK (0x3 << 1)
|
||||
#define FIT_QCMD_QID0 (0x0 << 1)
|
||||
#define FIT_QCMD_QID_NORMAL FIT_QCMD_QID0
|
||||
#define FIT_QCMD_QID1 (0x1 << 1)
|
||||
#define FIT_QCMD_QID2 (0x2 << 1)
|
||||
#define FIT_QCMD_QID3 (0x3 << 1)
|
||||
#define FIT_QCMD_FLUSH_QUEUE (0ull) /* add QID */
|
||||
#define FIT_QCMD_MSGSIZE_MASK (0x3 << 4)
|
||||
#define FIT_QCMD_MSGSIZE_64 (0x0 << 4)
|
||||
#define FIT_QCMD_MSGSIZE_128 (0x1 << 4)
|
||||
#define FIT_QCMD_MSGSIZE_256 (0x2 << 4)
|
||||
#define FIT_QCMD_MSGSIZE_512 (0x3 << 4)
|
||||
#define FIT_QCMD_BASE_ADDRESS_MASK (0xFFFFFFFFFFFFFFC0ull)
|
||||
|
||||
/*
|
||||
* Control, 32-bit r/w
|
||||
*/
|
||||
#define FIT_CONTROL 0x500u
|
||||
#define FIT_CR_HARD_RESET (1u << 0u)
|
||||
#define FIT_CR_SOFT_RESET (1u << 1u)
|
||||
#define FIT_CR_DIS_TIMESTAMPS (1u << 6u)
|
||||
#define FIT_CR_ENABLE_INTERRUPTS (1u << 7u)
|
||||
|
||||
/*
|
||||
* Status, 32-bit, r/o
|
||||
*/
|
||||
#define FIT_STATUS 0x510u
|
||||
#define FIT_SR_DRIVE_STATE_MASK 0x000000FFu
|
||||
#define FIT_SR_SIGNATURE (0xFF << 8)
|
||||
#define FIT_SR_PIO_DMA (1 << 16)
|
||||
#define FIT_SR_DRIVE_OFFLINE 0x00
|
||||
#define FIT_SR_DRIVE_INIT 0x01
|
||||
/* #define FIT_SR_DRIVE_READY 0x02 */
|
||||
#define FIT_SR_DRIVE_ONLINE 0x03
|
||||
#define FIT_SR_DRIVE_BUSY 0x04
|
||||
#define FIT_SR_DRIVE_FAULT 0x05
|
||||
#define FIT_SR_DRIVE_DEGRADED 0x06
|
||||
#define FIT_SR_PCIE_LINK_DOWN 0x07
|
||||
#define FIT_SR_DRIVE_SOFT_RESET 0x08
|
||||
#define FIT_SR_DRIVE_INIT_FAULT 0x09
|
||||
#define FIT_SR_DRIVE_BUSY_SANITIZE 0x0A
|
||||
#define FIT_SR_DRIVE_BUSY_ERASE 0x0B
|
||||
#define FIT_SR_DRIVE_FW_BOOTING 0x0C
|
||||
#define FIT_SR_DRIVE_NEED_FW_DOWNLOAD 0xFE
|
||||
#define FIT_SR_DEVICE_MISSING 0xFF
|
||||
#define FIT_SR__RESERVED 0xFFFFFF00u
|
||||
|
||||
/*
|
||||
* FIT_STATUS - Status register data definition
|
||||
*/
|
||||
#define FIT_SR_STATE_MASK (0xFF << 0)
|
||||
#define FIT_SR_SIGNATURE (0xFF << 8)
|
||||
#define FIT_SR_PIO_DMA (1 << 16)
|
||||
|
||||
/*
|
||||
* Interrupt status, 32-bit r/w1c (w1c ==> write 1 to clear)
|
||||
*/
|
||||
#define FIT_INT_STATUS_HOST 0x520u
|
||||
#define FIT_ISH_FW_STATE_CHANGE (1u << 0u)
|
||||
#define FIT_ISH_COMPLETION_POSTED (1u << 1u)
|
||||
#define FIT_ISH_MSG_FROM_DEV (1u << 2u)
|
||||
#define FIT_ISH_UNDEFINED_3 (1u << 3u)
|
||||
#define FIT_ISH_UNDEFINED_4 (1u << 4u)
|
||||
#define FIT_ISH_Q0_FULL (1u << 5u)
|
||||
#define FIT_ISH_Q1_FULL (1u << 6u)
|
||||
#define FIT_ISH_Q2_FULL (1u << 7u)
|
||||
#define FIT_ISH_Q3_FULL (1u << 8u)
|
||||
#define FIT_ISH_QCMD_FIFO_OVERRUN (1u << 9u)
|
||||
#define FIT_ISH_BAD_EXP_ROM_READ (1u << 10u)
|
||||
|
||||
#define FIT_INT_DEF_MASK \
|
||||
(FIT_ISH_FW_STATE_CHANGE | \
|
||||
FIT_ISH_COMPLETION_POSTED | \
|
||||
FIT_ISH_MSG_FROM_DEV | \
|
||||
FIT_ISH_Q0_FULL | \
|
||||
FIT_ISH_Q1_FULL | \
|
||||
FIT_ISH_Q2_FULL | \
|
||||
FIT_ISH_Q3_FULL | \
|
||||
FIT_ISH_QCMD_FIFO_OVERRUN | \
|
||||
FIT_ISH_BAD_EXP_ROM_READ)
|
||||
|
||||
#define FIT_INT_QUEUE_FULL \
|
||||
(FIT_ISH_Q0_FULL | \
|
||||
FIT_ISH_Q1_FULL | \
|
||||
FIT_ISH_Q2_FULL | \
|
||||
FIT_ISH_Q3_FULL)
|
||||
|
||||
#define MSI_MSG_NWL_ERROR_0 0x00000000
|
||||
#define MSI_MSG_NWL_ERROR_1 0x00000001
|
||||
#define MSI_MSG_NWL_ERROR_2 0x00000002
|
||||
#define MSI_MSG_NWL_ERROR_3 0x00000003
|
||||
#define MSI_MSG_STATE_CHANGE 0x00000004
|
||||
#define MSI_MSG_COMPLETION_POSTED 0x00000005
|
||||
#define MSI_MSG_MSG_FROM_DEV 0x00000006
|
||||
#define MSI_MSG_RESERVED_0 0x00000007
|
||||
#define MSI_MSG_RESERVED_1 0x00000008
|
||||
#define MSI_MSG_QUEUE_0_FULL 0x00000009
|
||||
#define MSI_MSG_QUEUE_1_FULL 0x0000000A
|
||||
#define MSI_MSG_QUEUE_2_FULL 0x0000000B
|
||||
#define MSI_MSG_QUEUE_3_FULL 0x0000000C
|
||||
|
||||
#define FIT_INT_RESERVED_MASK \
|
||||
(FIT_ISH_UNDEFINED_3 | \
|
||||
FIT_ISH_UNDEFINED_4)
|
||||
|
||||
/*
|
||||
* Interrupt mask, 32-bit r/w
|
||||
* Bit definitions are the same as FIT_INT_STATUS_HOST
|
||||
*/
|
||||
#define FIT_INT_MASK_HOST 0x528u
|
||||
|
||||
/*
|
||||
* Message to device, 32-bit r/w
|
||||
*/
|
||||
#define FIT_MSG_TO_DEVICE 0x540u
|
||||
|
||||
/*
|
||||
* Message from device, 32-bit, r/o
|
||||
*/
|
||||
#define FIT_MSG_FROM_DEVICE 0x548u
|
||||
|
||||
/*
|
||||
* 32-bit messages to/from device, composition/extraction macros
|
||||
*/
|
||||
#define FIT_MXD_CONS(TYPE, PARAM, DATA) \
|
||||
((((TYPE) & 0xFFu) << 24u) | \
|
||||
(((PARAM) & 0xFFu) << 16u) | \
|
||||
(((DATA) & 0xFFFFu) << 0u))
|
||||
#define FIT_MXD_TYPE(MXD) (((MXD) >> 24u) & 0xFFu)
|
||||
#define FIT_MXD_PARAM(MXD) (((MXD) >> 16u) & 0xFFu)
|
||||
#define FIT_MXD_DATA(MXD) (((MXD) >> 0u) & 0xFFFFu)
|
||||
|
||||
/*
|
||||
* Types of messages to/from device
|
||||
*/
|
||||
#define FIT_MTD_FITFW_INIT 0x01u
|
||||
#define FIT_MTD_GET_CMDQ_DEPTH 0x02u
|
||||
#define FIT_MTD_SET_COMPQ_DEPTH 0x03u
|
||||
#define FIT_MTD_SET_COMPQ_ADDR 0x04u
|
||||
#define FIT_MTD_ARM_QUEUE 0x05u
|
||||
#define FIT_MTD_CMD_LOG_HOST_ID 0x07u
|
||||
#define FIT_MTD_CMD_LOG_TIME_STAMP_LO 0x08u
|
||||
#define FIT_MTD_CMD_LOG_TIME_STAMP_HI 0x09u
|
||||
#define FIT_MFD_SMART_EXCEEDED 0x10u
|
||||
#define FIT_MFD_POWER_DOWN 0x11u
|
||||
#define FIT_MFD_OFFLINE 0x12u
|
||||
#define FIT_MFD_ONLINE 0x13u
|
||||
#define FIT_MFD_FW_RESTARTING 0x14u
|
||||
#define FIT_MFD_PM_ACTIVE 0x15u
|
||||
#define FIT_MFD_PM_STANDBY 0x16u
|
||||
#define FIT_MFD_PM_SLEEP 0x17u
|
||||
#define FIT_MFD_CMD_PROGRESS 0x18u
|
||||
|
||||
#define FIT_MTD_DEBUG 0xFEu
|
||||
#define FIT_MFD_DEBUG 0xFFu
|
||||
|
||||
#define FIT_MFD_MASK (0xFFu)
|
||||
#define FIT_MFD_DATA_MASK (0xFFu)
|
||||
#define FIT_MFD_MSG(x) (((x) >> 24) & FIT_MFD_MASK)
|
||||
#define FIT_MFD_DATA(x) ((x) & FIT_MFD_MASK)
|
||||
|
||||
/*
|
||||
* Extra arg to FIT_MSG_TO_DEVICE, 64-bit r/w
|
||||
* Used to set completion queue address (FIT_MTD_SET_COMPQ_ADDR)
|
||||
* (was Response buffer in docs)
|
||||
*/
|
||||
#define FIT_MSG_TO_DEVICE_ARG 0x580u
|
||||
|
||||
/*
|
||||
* Hardware (ASIC) version, 32-bit r/o
|
||||
*/
|
||||
#define FIT_HW_VERSION 0x588u
|
||||
|
||||
/*
|
||||
* Scatter/gather list descriptor.
|
||||
* 32-bytes and must be aligned on a 32-byte boundary.
|
||||
* All fields are in little endian order.
|
||||
*/
|
||||
struct fit_sg_descriptor {
|
||||
uint32_t control;
|
||||
uint32_t byte_count;
|
||||
uint64_t host_side_addr;
|
||||
uint64_t dev_side_addr;
|
||||
uint64_t next_desc_ptr;
|
||||
};
|
||||
|
||||
#define FIT_SGD_CONTROL_NOT_LAST 0x000u
|
||||
#define FIT_SGD_CONTROL_LAST 0x40Eu
|
||||
|
||||
/*
|
||||
* Header at the beginning of a FIT message. The header
|
||||
* is followed by SSDI requests each 64 bytes.
|
||||
* A FIT message can be up to 512 bytes long and must start
|
||||
* on a 64-byte boundary.
|
||||
*/
|
||||
struct fit_msg_hdr {
|
||||
uint8_t protocol_id;
|
||||
uint8_t num_protocol_cmds_coalesced;
|
||||
uint8_t _reserved[62];
|
||||
};
|
||||
|
||||
#define FIT_PROTOCOL_ID_FIT 1
|
||||
#define FIT_PROTOCOL_ID_SSDI 2
|
||||
#define FIT_PROTOCOL_ID_SOFIT 3
|
||||
|
||||
|
||||
#define FIT_PROTOCOL_MINOR_VER(mtd_val) ((mtd_val >> 16) & 0xF)
|
||||
#define FIT_PROTOCOL_MAJOR_VER(mtd_val) ((mtd_val >> 20) & 0xF)
|
||||
|
||||
/*
|
||||
* Format of a completion entry. The completion queue is circular
|
||||
* and must have at least as many entries as the maximum number
|
||||
* of commands that may be issued to the device.
|
||||
*
|
||||
* There are no head/tail pointers. The cycle value is used to
|
||||
* infer the presence of new completion records.
|
||||
* Initially the cycle in all entries is 0, the index is 0, and
|
||||
* the cycle value to expect is 1. When completions are added
|
||||
* their cycle values are set to 1. When the index wraps the
|
||||
* cycle value to expect is incremented.
|
||||
*
|
||||
* Command_context is opaque and taken verbatim from the SSDI command.
|
||||
* All other fields are big endian.
|
||||
*/
|
||||
#define FIT_PROTOCOL_VERSION_0 0
|
||||
|
||||
/*
|
||||
* Protocol major version 1 completion entry.
|
||||
* The major protocol version is found in bits
|
||||
* 20-23 of the FIT_MTD_FITFW_INIT response.
|
||||
*/
|
||||
struct fit_completion_entry_v1 {
|
||||
uint32_t num_returned_bytes;
|
||||
uint16_t tag;
|
||||
uint8_t status; /* SCSI status */
|
||||
uint8_t cycle;
|
||||
};
|
||||
#define FIT_PROTOCOL_VERSION_1 1
|
||||
#define FIT_PROTOCOL_VERSION_CURRENT FIT_PROTOCOL_VERSION_1
|
||||
|
||||
struct fit_comp_error_info {
|
||||
uint8_t type:7; /* 00: Bits0-6 indicates the type of sense data. */
|
||||
uint8_t valid:1; /* 00: Bit 7 := 1 ==> info field is valid. */
|
||||
uint8_t reserved0; /* 01: Obsolete field */
|
||||
uint8_t key:4; /* 02: Bits0-3 indicate the sense key. */
|
||||
uint8_t reserved2:1; /* 02: Reserved bit. */
|
||||
uint8_t bad_length:1; /* 02: Incorrect Length Indicator */
|
||||
uint8_t end_medium:1; /* 02: End of Medium */
|
||||
uint8_t file_mark:1; /* 02: Filemark */
|
||||
uint8_t info[4]; /* 03: */
|
||||
uint8_t reserved1; /* 07: Additional Sense Length */
|
||||
uint8_t cmd_spec[4]; /* 08: Command Specific Information */
|
||||
uint8_t code; /* 0C: Additional Sense Code */
|
||||
uint8_t qual; /* 0D: Additional Sense Code Qualifier */
|
||||
uint8_t fruc; /* 0E: Field Replaceable Unit Code */
|
||||
uint8_t sks_high:7; /* 0F: Sense Key Specific (MSB) */
|
||||
uint8_t sks_valid:1; /* 0F: Sense Key Specific Valid */
|
||||
uint16_t sks_low; /* 10: Sense Key Specific (LSW) */
|
||||
uint16_t reserved3; /* 12: Part of additional sense bytes (unused) */
|
||||
uint16_t uec; /* 14: Additional Sense Bytes */
|
||||
uint64_t per; /* 16: Additional Sense Bytes */
|
||||
uint8_t reserved4[2]; /* 1E: Additional Sense Bytes (unused) */
|
||||
};
|
||||
|
||||
|
||||
/* Task management constants */
|
||||
#define SOFT_TASK_SIMPLE 0x00
|
||||
#define SOFT_TASK_HEAD_OF_QUEUE 0x01
|
||||
#define SOFT_TASK_ORDERED 0x02
|
||||
|
||||
/* Version zero has the last 32 bits reserved,
|
||||
* Version one has the last 32 bits sg_list_len_bytes;
|
||||
*/
|
||||
struct skd_command_header {
|
||||
uint64_t sg_list_dma_address;
|
||||
uint16_t tag;
|
||||
uint8_t attribute;
|
||||
uint8_t add_cdb_len; /* In 32 bit words */
|
||||
uint32_t sg_list_len_bytes;
|
||||
};
|
||||
|
||||
struct skd_scsi_request {
|
||||
struct skd_command_header hdr;
|
||||
unsigned char cdb[16];
|
||||
/* unsigned char _reserved[16]; */
|
||||
};
|
||||
|
||||
struct driver_inquiry_data {
|
||||
uint8_t peripheral_device_type:5;
|
||||
uint8_t qualifier:3;
|
||||
uint8_t page_code;
|
||||
uint16_t page_length;
|
||||
uint16_t pcie_bus_number;
|
||||
uint8_t pcie_device_number;
|
||||
uint8_t pcie_function_number;
|
||||
uint8_t pcie_link_speed;
|
||||
uint8_t pcie_link_lanes;
|
||||
uint16_t pcie_vendor_id;
|
||||
uint16_t pcie_device_id;
|
||||
uint16_t pcie_subsystem_vendor_id;
|
||||
uint16_t pcie_subsystem_device_id;
|
||||
uint8_t reserved1[2];
|
||||
uint8_t reserved2[3];
|
||||
uint8_t driver_version_length;
|
||||
uint8_t driver_version[0x14];
|
||||
};
|
||||
|
||||
#pragma pack(pop, s1120_h)
|
||||
|
||||
#endif /* SKD_S1120_H */
|
@ -887,6 +887,8 @@ static int dispatch_discard_io(struct xen_blkif *blkif,
|
||||
unsigned long secure;
|
||||
struct phys_req preq;
|
||||
|
||||
xen_blkif_get(blkif);
|
||||
|
||||
preq.sector_number = req->u.discard.sector_number;
|
||||
preq.nr_sects = req->u.discard.nr_sectors;
|
||||
|
||||
@ -899,7 +901,6 @@ static int dispatch_discard_io(struct xen_blkif *blkif,
|
||||
}
|
||||
blkif->st_ds_req++;
|
||||
|
||||
xen_blkif_get(blkif);
|
||||
secure = (blkif->vbd.discard_secure &&
|
||||
(req->u.discard.flag & BLKIF_DISCARD_SECURE)) ?
|
||||
BLKDEV_DISCARD_SECURE : 0;
|
||||
|
@ -121,7 +121,8 @@ struct blkfront_info
|
||||
struct work_struct work;
|
||||
struct gnttab_free_callback callback;
|
||||
struct blk_shadow shadow[BLK_RING_SIZE];
|
||||
struct list_head persistent_gnts;
|
||||
struct list_head grants;
|
||||
struct list_head indirect_pages;
|
||||
unsigned int persistent_gnts_c;
|
||||
unsigned long shadow_free;
|
||||
unsigned int feature_flush;
|
||||
@ -200,15 +201,17 @@ static int fill_grant_buffer(struct blkfront_info *info, int num)
|
||||
if (!gnt_list_entry)
|
||||
goto out_of_memory;
|
||||
|
||||
granted_page = alloc_page(GFP_NOIO);
|
||||
if (!granted_page) {
|
||||
kfree(gnt_list_entry);
|
||||
goto out_of_memory;
|
||||
if (info->feature_persistent) {
|
||||
granted_page = alloc_page(GFP_NOIO);
|
||||
if (!granted_page) {
|
||||
kfree(gnt_list_entry);
|
||||
goto out_of_memory;
|
||||
}
|
||||
gnt_list_entry->pfn = page_to_pfn(granted_page);
|
||||
}
|
||||
|
||||
gnt_list_entry->pfn = page_to_pfn(granted_page);
|
||||
gnt_list_entry->gref = GRANT_INVALID_REF;
|
||||
list_add(&gnt_list_entry->node, &info->persistent_gnts);
|
||||
list_add(&gnt_list_entry->node, &info->grants);
|
||||
i++;
|
||||
}
|
||||
|
||||
@ -216,9 +219,10 @@ static int fill_grant_buffer(struct blkfront_info *info, int num)
|
||||
|
||||
out_of_memory:
|
||||
list_for_each_entry_safe(gnt_list_entry, n,
|
||||
&info->persistent_gnts, node) {
|
||||
&info->grants, node) {
|
||||
list_del(&gnt_list_entry->node);
|
||||
__free_page(pfn_to_page(gnt_list_entry->pfn));
|
||||
if (info->feature_persistent)
|
||||
__free_page(pfn_to_page(gnt_list_entry->pfn));
|
||||
kfree(gnt_list_entry);
|
||||
i--;
|
||||
}
|
||||
@ -227,13 +231,14 @@ out_of_memory:
|
||||
}
|
||||
|
||||
static struct grant *get_grant(grant_ref_t *gref_head,
|
||||
unsigned long pfn,
|
||||
struct blkfront_info *info)
|
||||
{
|
||||
struct grant *gnt_list_entry;
|
||||
unsigned long buffer_mfn;
|
||||
|
||||
BUG_ON(list_empty(&info->persistent_gnts));
|
||||
gnt_list_entry = list_first_entry(&info->persistent_gnts, struct grant,
|
||||
BUG_ON(list_empty(&info->grants));
|
||||
gnt_list_entry = list_first_entry(&info->grants, struct grant,
|
||||
node);
|
||||
list_del(&gnt_list_entry->node);
|
||||
|
||||
@ -245,6 +250,10 @@ static struct grant *get_grant(grant_ref_t *gref_head,
|
||||
/* Assign a gref to this page */
|
||||
gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
|
||||
BUG_ON(gnt_list_entry->gref == -ENOSPC);
|
||||
if (!info->feature_persistent) {
|
||||
BUG_ON(!pfn);
|
||||
gnt_list_entry->pfn = pfn;
|
||||
}
|
||||
buffer_mfn = pfn_to_mfn(gnt_list_entry->pfn);
|
||||
gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
|
||||
info->xbdev->otherend_id,
|
||||
@ -400,10 +409,13 @@ static int blkif_queue_request(struct request *req)
|
||||
if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
|
||||
return 1;
|
||||
|
||||
max_grefs = info->max_indirect_segments ?
|
||||
info->max_indirect_segments +
|
||||
INDIRECT_GREFS(info->max_indirect_segments) :
|
||||
BLKIF_MAX_SEGMENTS_PER_REQUEST;
|
||||
max_grefs = req->nr_phys_segments;
|
||||
if (max_grefs > BLKIF_MAX_SEGMENTS_PER_REQUEST)
|
||||
/*
|
||||
* If we are using indirect segments we need to account
|
||||
* for the indirect grefs used in the request.
|
||||
*/
|
||||
max_grefs += INDIRECT_GREFS(req->nr_phys_segments);
|
||||
|
||||
/* Check if we have enough grants to allocate a requests */
|
||||
if (info->persistent_gnts_c < max_grefs) {
|
||||
@ -477,22 +489,34 @@ static int blkif_queue_request(struct request *req)
|
||||
|
||||
if ((ring_req->operation == BLKIF_OP_INDIRECT) &&
|
||||
(i % SEGS_PER_INDIRECT_FRAME == 0)) {
|
||||
unsigned long pfn;
|
||||
|
||||
if (segments)
|
||||
kunmap_atomic(segments);
|
||||
|
||||
n = i / SEGS_PER_INDIRECT_FRAME;
|
||||
gnt_list_entry = get_grant(&gref_head, info);
|
||||
if (!info->feature_persistent) {
|
||||
struct page *indirect_page;
|
||||
|
||||
/* Fetch a pre-allocated page to use for indirect grefs */
|
||||
BUG_ON(list_empty(&info->indirect_pages));
|
||||
indirect_page = list_first_entry(&info->indirect_pages,
|
||||
struct page, lru);
|
||||
list_del(&indirect_page->lru);
|
||||
pfn = page_to_pfn(indirect_page);
|
||||
}
|
||||
gnt_list_entry = get_grant(&gref_head, pfn, info);
|
||||
info->shadow[id].indirect_grants[n] = gnt_list_entry;
|
||||
segments = kmap_atomic(pfn_to_page(gnt_list_entry->pfn));
|
||||
ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
|
||||
}
|
||||
|
||||
gnt_list_entry = get_grant(&gref_head, info);
|
||||
gnt_list_entry = get_grant(&gref_head, page_to_pfn(sg_page(sg)), info);
|
||||
ref = gnt_list_entry->gref;
|
||||
|
||||
info->shadow[id].grants_used[i] = gnt_list_entry;
|
||||
|
||||
if (rq_data_dir(req)) {
|
||||
if (rq_data_dir(req) && info->feature_persistent) {
|
||||
char *bvec_data;
|
||||
void *shared_data;
|
||||
|
||||
@ -904,21 +928,36 @@ static void blkif_free(struct blkfront_info *info, int suspend)
|
||||
blk_stop_queue(info->rq);
|
||||
|
||||
/* Remove all persistent grants */
|
||||
if (!list_empty(&info->persistent_gnts)) {
|
||||
if (!list_empty(&info->grants)) {
|
||||
list_for_each_entry_safe(persistent_gnt, n,
|
||||
&info->persistent_gnts, node) {
|
||||
&info->grants, node) {
|
||||
list_del(&persistent_gnt->node);
|
||||
if (persistent_gnt->gref != GRANT_INVALID_REF) {
|
||||
gnttab_end_foreign_access(persistent_gnt->gref,
|
||||
0, 0UL);
|
||||
info->persistent_gnts_c--;
|
||||
}
|
||||
__free_page(pfn_to_page(persistent_gnt->pfn));
|
||||
if (info->feature_persistent)
|
||||
__free_page(pfn_to_page(persistent_gnt->pfn));
|
||||
kfree(persistent_gnt);
|
||||
}
|
||||
}
|
||||
BUG_ON(info->persistent_gnts_c != 0);
|
||||
|
||||
/*
|
||||
* Remove indirect pages, this only happens when using indirect
|
||||
* descriptors but not persistent grants
|
||||
*/
|
||||
if (!list_empty(&info->indirect_pages)) {
|
||||
struct page *indirect_page, *n;
|
||||
|
||||
BUG_ON(info->feature_persistent);
|
||||
list_for_each_entry_safe(indirect_page, n, &info->indirect_pages, lru) {
|
||||
list_del(&indirect_page->lru);
|
||||
__free_page(indirect_page);
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < BLK_RING_SIZE; i++) {
|
||||
/*
|
||||
* Clear persistent grants present in requests already
|
||||
@ -933,7 +972,8 @@ static void blkif_free(struct blkfront_info *info, int suspend)
|
||||
for (j = 0; j < segs; j++) {
|
||||
persistent_gnt = info->shadow[i].grants_used[j];
|
||||
gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
|
||||
__free_page(pfn_to_page(persistent_gnt->pfn));
|
||||
if (info->feature_persistent)
|
||||
__free_page(pfn_to_page(persistent_gnt->pfn));
|
||||
kfree(persistent_gnt);
|
||||
}
|
||||
|
||||
@ -992,7 +1032,7 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
|
||||
nseg = s->req.operation == BLKIF_OP_INDIRECT ?
|
||||
s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
|
||||
|
||||
if (bret->operation == BLKIF_OP_READ) {
|
||||
if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
|
||||
/*
|
||||
* Copy the data received from the backend into the bvec.
|
||||
* Since bv_offset can be different than 0, and bv_len different
|
||||
@ -1013,13 +1053,51 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
|
||||
}
|
||||
/* Add the persistent grant into the list of free grants */
|
||||
for (i = 0; i < nseg; i++) {
|
||||
list_add(&s->grants_used[i]->node, &info->persistent_gnts);
|
||||
info->persistent_gnts_c++;
|
||||
if (gnttab_query_foreign_access(s->grants_used[i]->gref)) {
|
||||
/*
|
||||
* If the grant is still mapped by the backend (the
|
||||
* backend has chosen to make this grant persistent)
|
||||
* we add it at the head of the list, so it will be
|
||||
* reused first.
|
||||
*/
|
||||
if (!info->feature_persistent)
|
||||
pr_alert_ratelimited("backed has not unmapped grant: %u\n",
|
||||
s->grants_used[i]->gref);
|
||||
list_add(&s->grants_used[i]->node, &info->grants);
|
||||
info->persistent_gnts_c++;
|
||||
} else {
|
||||
/*
|
||||
* If the grant is not mapped by the backend we end the
|
||||
* foreign access and add it to the tail of the list,
|
||||
* so it will not be picked again unless we run out of
|
||||
* persistent grants.
|
||||
*/
|
||||
gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
|
||||
s->grants_used[i]->gref = GRANT_INVALID_REF;
|
||||
list_add_tail(&s->grants_used[i]->node, &info->grants);
|
||||
}
|
||||
}
|
||||
if (s->req.operation == BLKIF_OP_INDIRECT) {
|
||||
for (i = 0; i < INDIRECT_GREFS(nseg); i++) {
|
||||
list_add(&s->indirect_grants[i]->node, &info->persistent_gnts);
|
||||
info->persistent_gnts_c++;
|
||||
if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) {
|
||||
if (!info->feature_persistent)
|
||||
pr_alert_ratelimited("backed has not unmapped grant: %u\n",
|
||||
s->indirect_grants[i]->gref);
|
||||
list_add(&s->indirect_grants[i]->node, &info->grants);
|
||||
info->persistent_gnts_c++;
|
||||
} else {
|
||||
struct page *indirect_page;
|
||||
|
||||
gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL);
|
||||
/*
|
||||
* Add the used indirect page back to the list of
|
||||
* available pages for indirect grefs.
|
||||
*/
|
||||
indirect_page = pfn_to_page(s->indirect_grants[i]->pfn);
|
||||
list_add(&indirect_page->lru, &info->indirect_pages);
|
||||
s->indirect_grants[i]->gref = GRANT_INVALID_REF;
|
||||
list_add_tail(&s->indirect_grants[i]->node, &info->grants);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1313,7 +1391,8 @@ static int blkfront_probe(struct xenbus_device *dev,
|
||||
spin_lock_init(&info->io_lock);
|
||||
info->xbdev = dev;
|
||||
info->vdevice = vdevice;
|
||||
INIT_LIST_HEAD(&info->persistent_gnts);
|
||||
INIT_LIST_HEAD(&info->grants);
|
||||
INIT_LIST_HEAD(&info->indirect_pages);
|
||||
info->persistent_gnts_c = 0;
|
||||
info->connected = BLKIF_STATE_DISCONNECTED;
|
||||
INIT_WORK(&info->work, blkif_restart_queue);
|
||||
@ -1609,6 +1688,23 @@ static int blkfront_setup_indirect(struct blkfront_info *info)
|
||||
if (err)
|
||||
goto out_of_memory;
|
||||
|
||||
if (!info->feature_persistent && info->max_indirect_segments) {
|
||||
/*
|
||||
* We are using indirect descriptors but not persistent
|
||||
* grants, we need to allocate a set of pages that can be
|
||||
* used for mapping indirect grefs
|
||||
*/
|
||||
int num = INDIRECT_GREFS(segs) * BLK_RING_SIZE;
|
||||
|
||||
BUG_ON(!list_empty(&info->indirect_pages));
|
||||
for (i = 0; i < num; i++) {
|
||||
struct page *indirect_page = alloc_page(GFP_NOIO);
|
||||
if (!indirect_page)
|
||||
goto out_of_memory;
|
||||
list_add(&indirect_page->lru, &info->indirect_pages);
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < BLK_RING_SIZE; i++) {
|
||||
info->shadow[i].grants_used = kzalloc(
|
||||
sizeof(info->shadow[i].grants_used[0]) * segs,
|
||||
@ -1639,6 +1735,13 @@ out_of_memory:
|
||||
kfree(info->shadow[i].indirect_grants);
|
||||
info->shadow[i].indirect_grants = NULL;
|
||||
}
|
||||
if (!list_empty(&info->indirect_pages)) {
|
||||
struct page *indirect_page, *n;
|
||||
list_for_each_entry_safe(indirect_page, n, &info->indirect_pages, lru) {
|
||||
list_del(&indirect_page->lru);
|
||||
__free_page(indirect_page);
|
||||
}
|
||||
}
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
|
@ -13,15 +13,8 @@ config BCACHE_DEBUG
|
||||
---help---
|
||||
Don't select this option unless you're a developer
|
||||
|
||||
Enables extra debugging tools (primarily a fuzz tester)
|
||||
|
||||
config BCACHE_EDEBUG
|
||||
bool "Extended runtime checks"
|
||||
depends on BCACHE
|
||||
---help---
|
||||
Don't select this option unless you're a developer
|
||||
|
||||
Enables extra runtime checks which significantly affect performance
|
||||
Enables extra debugging tools, allows expensive runtime checks to be
|
||||
turned on.
|
||||
|
||||
config BCACHE_CLOSURES_DEBUG
|
||||
bool "Debug closures"
|
||||
|
@ -63,13 +63,12 @@
|
||||
#include "bcache.h"
|
||||
#include "btree.h"
|
||||
|
||||
#include <linux/blkdev.h>
|
||||
#include <linux/freezer.h>
|
||||
#include <linux/kthread.h>
|
||||
#include <linux/random.h>
|
||||
#include <trace/events/bcache.h>
|
||||
|
||||
#define MAX_IN_FLIGHT_DISCARDS 8U
|
||||
|
||||
/* Bucket heap / gen */
|
||||
|
||||
uint8_t bch_inc_gen(struct cache *ca, struct bucket *b)
|
||||
@ -121,75 +120,6 @@ void bch_rescale_priorities(struct cache_set *c, int sectors)
|
||||
mutex_unlock(&c->bucket_lock);
|
||||
}
|
||||
|
||||
/* Discard/TRIM */
|
||||
|
||||
struct discard {
|
||||
struct list_head list;
|
||||
struct work_struct work;
|
||||
struct cache *ca;
|
||||
long bucket;
|
||||
|
||||
struct bio bio;
|
||||
struct bio_vec bv;
|
||||
};
|
||||
|
||||
static void discard_finish(struct work_struct *w)
|
||||
{
|
||||
struct discard *d = container_of(w, struct discard, work);
|
||||
struct cache *ca = d->ca;
|
||||
char buf[BDEVNAME_SIZE];
|
||||
|
||||
if (!test_bit(BIO_UPTODATE, &d->bio.bi_flags)) {
|
||||
pr_notice("discard error on %s, disabling",
|
||||
bdevname(ca->bdev, buf));
|
||||
d->ca->discard = 0;
|
||||
}
|
||||
|
||||
mutex_lock(&ca->set->bucket_lock);
|
||||
|
||||
fifo_push(&ca->free, d->bucket);
|
||||
list_add(&d->list, &ca->discards);
|
||||
atomic_dec(&ca->discards_in_flight);
|
||||
|
||||
mutex_unlock(&ca->set->bucket_lock);
|
||||
|
||||
closure_wake_up(&ca->set->bucket_wait);
|
||||
wake_up_process(ca->alloc_thread);
|
||||
|
||||
closure_put(&ca->set->cl);
|
||||
}
|
||||
|
||||
static void discard_endio(struct bio *bio, int error)
|
||||
{
|
||||
struct discard *d = container_of(bio, struct discard, bio);
|
||||
schedule_work(&d->work);
|
||||
}
|
||||
|
||||
static void do_discard(struct cache *ca, long bucket)
|
||||
{
|
||||
struct discard *d = list_first_entry(&ca->discards,
|
||||
struct discard, list);
|
||||
|
||||
list_del(&d->list);
|
||||
d->bucket = bucket;
|
||||
|
||||
atomic_inc(&ca->discards_in_flight);
|
||||
closure_get(&ca->set->cl);
|
||||
|
||||
bio_init(&d->bio);
|
||||
|
||||
d->bio.bi_sector = bucket_to_sector(ca->set, d->bucket);
|
||||
d->bio.bi_bdev = ca->bdev;
|
||||
d->bio.bi_rw = REQ_WRITE|REQ_DISCARD;
|
||||
d->bio.bi_max_vecs = 1;
|
||||
d->bio.bi_io_vec = d->bio.bi_inline_vecs;
|
||||
d->bio.bi_size = bucket_bytes(ca);
|
||||
d->bio.bi_end_io = discard_endio;
|
||||
bio_set_prio(&d->bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
|
||||
|
||||
submit_bio(0, &d->bio);
|
||||
}
|
||||
|
||||
/* Allocation */
|
||||
|
||||
static inline bool can_inc_bucket_gen(struct bucket *b)
|
||||
@ -280,7 +210,7 @@ static void invalidate_buckets_lru(struct cache *ca)
|
||||
* multiple times when it can't do anything
|
||||
*/
|
||||
ca->invalidate_needs_gc = 1;
|
||||
bch_queue_gc(ca->set);
|
||||
wake_up_gc(ca->set);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -305,7 +235,7 @@ static void invalidate_buckets_fifo(struct cache *ca)
|
||||
|
||||
if (++checked >= ca->sb.nbuckets) {
|
||||
ca->invalidate_needs_gc = 1;
|
||||
bch_queue_gc(ca->set);
|
||||
wake_up_gc(ca->set);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -330,7 +260,7 @@ static void invalidate_buckets_random(struct cache *ca)
|
||||
|
||||
if (++checked >= ca->sb.nbuckets / 2) {
|
||||
ca->invalidate_needs_gc = 1;
|
||||
bch_queue_gc(ca->set);
|
||||
wake_up_gc(ca->set);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -398,16 +328,18 @@ static int bch_allocator_thread(void *arg)
|
||||
else
|
||||
break;
|
||||
|
||||
allocator_wait(ca, (int) fifo_free(&ca->free) >
|
||||
atomic_read(&ca->discards_in_flight));
|
||||
|
||||
if (ca->discard) {
|
||||
allocator_wait(ca, !list_empty(&ca->discards));
|
||||
do_discard(ca, bucket);
|
||||
} else {
|
||||
fifo_push(&ca->free, bucket);
|
||||
closure_wake_up(&ca->set->bucket_wait);
|
||||
mutex_unlock(&ca->set->bucket_lock);
|
||||
blkdev_issue_discard(ca->bdev,
|
||||
bucket_to_sector(ca->set, bucket),
|
||||
ca->sb.block_size, GFP_KERNEL, 0);
|
||||
mutex_lock(&ca->set->bucket_lock);
|
||||
}
|
||||
|
||||
allocator_wait(ca, !fifo_full(&ca->free));
|
||||
|
||||
fifo_push(&ca->free, bucket);
|
||||
wake_up(&ca->set->bucket_wait);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -433,16 +365,40 @@ static int bch_allocator_thread(void *arg)
|
||||
}
|
||||
}
|
||||
|
||||
long bch_bucket_alloc(struct cache *ca, unsigned watermark, struct closure *cl)
|
||||
long bch_bucket_alloc(struct cache *ca, unsigned watermark, bool wait)
|
||||
{
|
||||
long r = -1;
|
||||
again:
|
||||
DEFINE_WAIT(w);
|
||||
struct bucket *b;
|
||||
long r;
|
||||
|
||||
/* fastpath */
|
||||
if (fifo_used(&ca->free) > ca->watermark[watermark]) {
|
||||
fifo_pop(&ca->free, r);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (!wait)
|
||||
return -1;
|
||||
|
||||
while (1) {
|
||||
if (fifo_used(&ca->free) > ca->watermark[watermark]) {
|
||||
fifo_pop(&ca->free, r);
|
||||
break;
|
||||
}
|
||||
|
||||
prepare_to_wait(&ca->set->bucket_wait, &w,
|
||||
TASK_UNINTERRUPTIBLE);
|
||||
|
||||
mutex_unlock(&ca->set->bucket_lock);
|
||||
schedule();
|
||||
mutex_lock(&ca->set->bucket_lock);
|
||||
}
|
||||
|
||||
finish_wait(&ca->set->bucket_wait, &w);
|
||||
out:
|
||||
wake_up_process(ca->alloc_thread);
|
||||
|
||||
if (fifo_used(&ca->free) > ca->watermark[watermark] &&
|
||||
fifo_pop(&ca->free, r)) {
|
||||
struct bucket *b = ca->buckets + r;
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
if (expensive_debug_checks(ca->set)) {
|
||||
size_t iter;
|
||||
long i;
|
||||
|
||||
@ -455,36 +411,23 @@ again:
|
||||
BUG_ON(i == r);
|
||||
fifo_for_each(i, &ca->unused, iter)
|
||||
BUG_ON(i == r);
|
||||
#endif
|
||||
BUG_ON(atomic_read(&b->pin) != 1);
|
||||
|
||||
SET_GC_SECTORS_USED(b, ca->sb.bucket_size);
|
||||
|
||||
if (watermark <= WATERMARK_METADATA) {
|
||||
SET_GC_MARK(b, GC_MARK_METADATA);
|
||||
b->prio = BTREE_PRIO;
|
||||
} else {
|
||||
SET_GC_MARK(b, GC_MARK_RECLAIMABLE);
|
||||
b->prio = INITIAL_PRIO;
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
trace_bcache_alloc_fail(ca);
|
||||
b = ca->buckets + r;
|
||||
|
||||
if (cl) {
|
||||
closure_wait(&ca->set->bucket_wait, cl);
|
||||
BUG_ON(atomic_read(&b->pin) != 1);
|
||||
|
||||
if (closure_blocking(cl)) {
|
||||
mutex_unlock(&ca->set->bucket_lock);
|
||||
closure_sync(cl);
|
||||
mutex_lock(&ca->set->bucket_lock);
|
||||
goto again;
|
||||
}
|
||||
SET_GC_SECTORS_USED(b, ca->sb.bucket_size);
|
||||
|
||||
if (watermark <= WATERMARK_METADATA) {
|
||||
SET_GC_MARK(b, GC_MARK_METADATA);
|
||||
b->prio = BTREE_PRIO;
|
||||
} else {
|
||||
SET_GC_MARK(b, GC_MARK_RECLAIMABLE);
|
||||
b->prio = INITIAL_PRIO;
|
||||
}
|
||||
|
||||
return -1;
|
||||
return r;
|
||||
}
|
||||
|
||||
void bch_bucket_free(struct cache_set *c, struct bkey *k)
|
||||
@ -501,7 +444,7 @@ void bch_bucket_free(struct cache_set *c, struct bkey *k)
|
||||
}
|
||||
|
||||
int __bch_bucket_alloc_set(struct cache_set *c, unsigned watermark,
|
||||
struct bkey *k, int n, struct closure *cl)
|
||||
struct bkey *k, int n, bool wait)
|
||||
{
|
||||
int i;
|
||||
|
||||
@ -514,7 +457,7 @@ int __bch_bucket_alloc_set(struct cache_set *c, unsigned watermark,
|
||||
|
||||
for (i = 0; i < n; i++) {
|
||||
struct cache *ca = c->cache_by_alloc[i];
|
||||
long b = bch_bucket_alloc(ca, watermark, cl);
|
||||
long b = bch_bucket_alloc(ca, watermark, wait);
|
||||
|
||||
if (b == -1)
|
||||
goto err;
|
||||
@ -529,22 +472,202 @@ int __bch_bucket_alloc_set(struct cache_set *c, unsigned watermark,
|
||||
return 0;
|
||||
err:
|
||||
bch_bucket_free(c, k);
|
||||
__bkey_put(c, k);
|
||||
bkey_put(c, k);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int bch_bucket_alloc_set(struct cache_set *c, unsigned watermark,
|
||||
struct bkey *k, int n, struct closure *cl)
|
||||
struct bkey *k, int n, bool wait)
|
||||
{
|
||||
int ret;
|
||||
mutex_lock(&c->bucket_lock);
|
||||
ret = __bch_bucket_alloc_set(c, watermark, k, n, cl);
|
||||
ret = __bch_bucket_alloc_set(c, watermark, k, n, wait);
|
||||
mutex_unlock(&c->bucket_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Sector allocator */
|
||||
|
||||
struct open_bucket {
|
||||
struct list_head list;
|
||||
unsigned last_write_point;
|
||||
unsigned sectors_free;
|
||||
BKEY_PADDED(key);
|
||||
};
|
||||
|
||||
/*
|
||||
* We keep multiple buckets open for writes, and try to segregate different
|
||||
* write streams for better cache utilization: first we look for a bucket where
|
||||
* the last write to it was sequential with the current write, and failing that
|
||||
* we look for a bucket that was last used by the same task.
|
||||
*
|
||||
* The ideas is if you've got multiple tasks pulling data into the cache at the
|
||||
* same time, you'll get better cache utilization if you try to segregate their
|
||||
* data and preserve locality.
|
||||
*
|
||||
* For example, say you've starting Firefox at the same time you're copying a
|
||||
* bunch of files. Firefox will likely end up being fairly hot and stay in the
|
||||
* cache awhile, but the data you copied might not be; if you wrote all that
|
||||
* data to the same buckets it'd get invalidated at the same time.
|
||||
*
|
||||
* Both of those tasks will be doing fairly random IO so we can't rely on
|
||||
* detecting sequential IO to segregate their data, but going off of the task
|
||||
* should be a sane heuristic.
|
||||
*/
|
||||
static struct open_bucket *pick_data_bucket(struct cache_set *c,
|
||||
const struct bkey *search,
|
||||
unsigned write_point,
|
||||
struct bkey *alloc)
|
||||
{
|
||||
struct open_bucket *ret, *ret_task = NULL;
|
||||
|
||||
list_for_each_entry_reverse(ret, &c->data_buckets, list)
|
||||
if (!bkey_cmp(&ret->key, search))
|
||||
goto found;
|
||||
else if (ret->last_write_point == write_point)
|
||||
ret_task = ret;
|
||||
|
||||
ret = ret_task ?: list_first_entry(&c->data_buckets,
|
||||
struct open_bucket, list);
|
||||
found:
|
||||
if (!ret->sectors_free && KEY_PTRS(alloc)) {
|
||||
ret->sectors_free = c->sb.bucket_size;
|
||||
bkey_copy(&ret->key, alloc);
|
||||
bkey_init(alloc);
|
||||
}
|
||||
|
||||
if (!ret->sectors_free)
|
||||
ret = NULL;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocates some space in the cache to write to, and k to point to the newly
|
||||
* allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
|
||||
* end of the newly allocated space).
|
||||
*
|
||||
* May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
|
||||
* sectors were actually allocated.
|
||||
*
|
||||
* If s->writeback is true, will not fail.
|
||||
*/
|
||||
bool bch_alloc_sectors(struct cache_set *c, struct bkey *k, unsigned sectors,
|
||||
unsigned write_point, unsigned write_prio, bool wait)
|
||||
{
|
||||
struct open_bucket *b;
|
||||
BKEY_PADDED(key) alloc;
|
||||
unsigned i;
|
||||
|
||||
/*
|
||||
* We might have to allocate a new bucket, which we can't do with a
|
||||
* spinlock held. So if we have to allocate, we drop the lock, allocate
|
||||
* and then retry. KEY_PTRS() indicates whether alloc points to
|
||||
* allocated bucket(s).
|
||||
*/
|
||||
|
||||
bkey_init(&alloc.key);
|
||||
spin_lock(&c->data_bucket_lock);
|
||||
|
||||
while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
|
||||
unsigned watermark = write_prio
|
||||
? WATERMARK_MOVINGGC
|
||||
: WATERMARK_NONE;
|
||||
|
||||
spin_unlock(&c->data_bucket_lock);
|
||||
|
||||
if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
|
||||
return false;
|
||||
|
||||
spin_lock(&c->data_bucket_lock);
|
||||
}
|
||||
|
||||
/*
|
||||
* If we had to allocate, we might race and not need to allocate the
|
||||
* second time we call find_data_bucket(). If we allocated a bucket but
|
||||
* didn't use it, drop the refcount bch_bucket_alloc_set() took:
|
||||
*/
|
||||
if (KEY_PTRS(&alloc.key))
|
||||
bkey_put(c, &alloc.key);
|
||||
|
||||
for (i = 0; i < KEY_PTRS(&b->key); i++)
|
||||
EBUG_ON(ptr_stale(c, &b->key, i));
|
||||
|
||||
/* Set up the pointer to the space we're allocating: */
|
||||
|
||||
for (i = 0; i < KEY_PTRS(&b->key); i++)
|
||||
k->ptr[i] = b->key.ptr[i];
|
||||
|
||||
sectors = min(sectors, b->sectors_free);
|
||||
|
||||
SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
|
||||
SET_KEY_SIZE(k, sectors);
|
||||
SET_KEY_PTRS(k, KEY_PTRS(&b->key));
|
||||
|
||||
/*
|
||||
* Move b to the end of the lru, and keep track of what this bucket was
|
||||
* last used for:
|
||||
*/
|
||||
list_move_tail(&b->list, &c->data_buckets);
|
||||
bkey_copy_key(&b->key, k);
|
||||
b->last_write_point = write_point;
|
||||
|
||||
b->sectors_free -= sectors;
|
||||
|
||||
for (i = 0; i < KEY_PTRS(&b->key); i++) {
|
||||
SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
|
||||
|
||||
atomic_long_add(sectors,
|
||||
&PTR_CACHE(c, &b->key, i)->sectors_written);
|
||||
}
|
||||
|
||||
if (b->sectors_free < c->sb.block_size)
|
||||
b->sectors_free = 0;
|
||||
|
||||
/*
|
||||
* k takes refcounts on the buckets it points to until it's inserted
|
||||
* into the btree, but if we're done with this bucket we just transfer
|
||||
* get_data_bucket()'s refcount.
|
||||
*/
|
||||
if (b->sectors_free)
|
||||
for (i = 0; i < KEY_PTRS(&b->key); i++)
|
||||
atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
|
||||
|
||||
spin_unlock(&c->data_bucket_lock);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Init */
|
||||
|
||||
void bch_open_buckets_free(struct cache_set *c)
|
||||
{
|
||||
struct open_bucket *b;
|
||||
|
||||
while (!list_empty(&c->data_buckets)) {
|
||||
b = list_first_entry(&c->data_buckets,
|
||||
struct open_bucket, list);
|
||||
list_del(&b->list);
|
||||
kfree(b);
|
||||
}
|
||||
}
|
||||
|
||||
int bch_open_buckets_alloc(struct cache_set *c)
|
||||
{
|
||||
int i;
|
||||
|
||||
spin_lock_init(&c->data_bucket_lock);
|
||||
|
||||
for (i = 0; i < 6; i++) {
|
||||
struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
|
||||
if (!b)
|
||||
return -ENOMEM;
|
||||
|
||||
list_add(&b->list, &c->data_buckets);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bch_cache_allocator_start(struct cache *ca)
|
||||
{
|
||||
struct task_struct *k = kthread_run(bch_allocator_thread,
|
||||
@ -556,22 +679,8 @@ int bch_cache_allocator_start(struct cache *ca)
|
||||
return 0;
|
||||
}
|
||||
|
||||
void bch_cache_allocator_exit(struct cache *ca)
|
||||
{
|
||||
struct discard *d;
|
||||
|
||||
while (!list_empty(&ca->discards)) {
|
||||
d = list_first_entry(&ca->discards, struct discard, list);
|
||||
cancel_work_sync(&d->work);
|
||||
list_del(&d->list);
|
||||
kfree(d);
|
||||
}
|
||||
}
|
||||
|
||||
int bch_cache_allocator_init(struct cache *ca)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
/*
|
||||
* Reserve:
|
||||
* Prio/gen writes first
|
||||
@ -589,15 +698,5 @@ int bch_cache_allocator_init(struct cache *ca)
|
||||
ca->watermark[WATERMARK_NONE] = ca->free.size / 2 +
|
||||
ca->watermark[WATERMARK_MOVINGGC];
|
||||
|
||||
for (i = 0; i < MAX_IN_FLIGHT_DISCARDS; i++) {
|
||||
struct discard *d = kzalloc(sizeof(*d), GFP_KERNEL);
|
||||
if (!d)
|
||||
return -ENOMEM;
|
||||
|
||||
d->ca = ca;
|
||||
INIT_WORK(&d->work, discard_finish);
|
||||
list_add(&d->list, &ca->discards);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -177,6 +177,7 @@
|
||||
|
||||
#define pr_fmt(fmt) "bcache: %s() " fmt "\n", __func__
|
||||
|
||||
#include <linux/bcache.h>
|
||||
#include <linux/bio.h>
|
||||
#include <linux/kobject.h>
|
||||
#include <linux/list.h>
|
||||
@ -210,168 +211,6 @@ BITMASK(GC_MARK, struct bucket, gc_mark, 0, 2);
|
||||
#define GC_MARK_METADATA 2
|
||||
BITMASK(GC_SECTORS_USED, struct bucket, gc_mark, 2, 14);
|
||||
|
||||
struct bkey {
|
||||
uint64_t high;
|
||||
uint64_t low;
|
||||
uint64_t ptr[];
|
||||
};
|
||||
|
||||
/* Enough for a key with 6 pointers */
|
||||
#define BKEY_PAD 8
|
||||
|
||||
#define BKEY_PADDED(key) \
|
||||
union { struct bkey key; uint64_t key ## _pad[BKEY_PAD]; }
|
||||
|
||||
/* Version 0: Cache device
|
||||
* Version 1: Backing device
|
||||
* Version 2: Seed pointer into btree node checksum
|
||||
* Version 3: Cache device with new UUID format
|
||||
* Version 4: Backing device with data offset
|
||||
*/
|
||||
#define BCACHE_SB_VERSION_CDEV 0
|
||||
#define BCACHE_SB_VERSION_BDEV 1
|
||||
#define BCACHE_SB_VERSION_CDEV_WITH_UUID 3
|
||||
#define BCACHE_SB_VERSION_BDEV_WITH_OFFSET 4
|
||||
#define BCACHE_SB_MAX_VERSION 4
|
||||
|
||||
#define SB_SECTOR 8
|
||||
#define SB_SIZE 4096
|
||||
#define SB_LABEL_SIZE 32
|
||||
#define SB_JOURNAL_BUCKETS 256U
|
||||
/* SB_JOURNAL_BUCKETS must be divisible by BITS_PER_LONG */
|
||||
#define MAX_CACHES_PER_SET 8
|
||||
|
||||
#define BDEV_DATA_START_DEFAULT 16 /* sectors */
|
||||
|
||||
struct cache_sb {
|
||||
uint64_t csum;
|
||||
uint64_t offset; /* sector where this sb was written */
|
||||
uint64_t version;
|
||||
|
||||
uint8_t magic[16];
|
||||
|
||||
uint8_t uuid[16];
|
||||
union {
|
||||
uint8_t set_uuid[16];
|
||||
uint64_t set_magic;
|
||||
};
|
||||
uint8_t label[SB_LABEL_SIZE];
|
||||
|
||||
uint64_t flags;
|
||||
uint64_t seq;
|
||||
uint64_t pad[8];
|
||||
|
||||
union {
|
||||
struct {
|
||||
/* Cache devices */
|
||||
uint64_t nbuckets; /* device size */
|
||||
|
||||
uint16_t block_size; /* sectors */
|
||||
uint16_t bucket_size; /* sectors */
|
||||
|
||||
uint16_t nr_in_set;
|
||||
uint16_t nr_this_dev;
|
||||
};
|
||||
struct {
|
||||
/* Backing devices */
|
||||
uint64_t data_offset;
|
||||
|
||||
/*
|
||||
* block_size from the cache device section is still used by
|
||||
* backing devices, so don't add anything here until we fix
|
||||
* things to not need it for backing devices anymore
|
||||
*/
|
||||
};
|
||||
};
|
||||
|
||||
uint32_t last_mount; /* time_t */
|
||||
|
||||
uint16_t first_bucket;
|
||||
union {
|
||||
uint16_t njournal_buckets;
|
||||
uint16_t keys;
|
||||
};
|
||||
uint64_t d[SB_JOURNAL_BUCKETS]; /* journal buckets */
|
||||
};
|
||||
|
||||
BITMASK(CACHE_SYNC, struct cache_sb, flags, 0, 1);
|
||||
BITMASK(CACHE_DISCARD, struct cache_sb, flags, 1, 1);
|
||||
BITMASK(CACHE_REPLACEMENT, struct cache_sb, flags, 2, 3);
|
||||
#define CACHE_REPLACEMENT_LRU 0U
|
||||
#define CACHE_REPLACEMENT_FIFO 1U
|
||||
#define CACHE_REPLACEMENT_RANDOM 2U
|
||||
|
||||
BITMASK(BDEV_CACHE_MODE, struct cache_sb, flags, 0, 4);
|
||||
#define CACHE_MODE_WRITETHROUGH 0U
|
||||
#define CACHE_MODE_WRITEBACK 1U
|
||||
#define CACHE_MODE_WRITEAROUND 2U
|
||||
#define CACHE_MODE_NONE 3U
|
||||
BITMASK(BDEV_STATE, struct cache_sb, flags, 61, 2);
|
||||
#define BDEV_STATE_NONE 0U
|
||||
#define BDEV_STATE_CLEAN 1U
|
||||
#define BDEV_STATE_DIRTY 2U
|
||||
#define BDEV_STATE_STALE 3U
|
||||
|
||||
/* Version 1: Seed pointer into btree node checksum
|
||||
*/
|
||||
#define BCACHE_BSET_VERSION 1
|
||||
|
||||
/*
|
||||
* This is the on disk format for btree nodes - a btree node on disk is a list
|
||||
* of these; within each set the keys are sorted
|
||||
*/
|
||||
struct bset {
|
||||
uint64_t csum;
|
||||
uint64_t magic;
|
||||
uint64_t seq;
|
||||
uint32_t version;
|
||||
uint32_t keys;
|
||||
|
||||
union {
|
||||
struct bkey start[0];
|
||||
uint64_t d[0];
|
||||
};
|
||||
};
|
||||
|
||||
/*
|
||||
* On disk format for priorities and gens - see super.c near prio_write() for
|
||||
* more.
|
||||
*/
|
||||
struct prio_set {
|
||||
uint64_t csum;
|
||||
uint64_t magic;
|
||||
uint64_t seq;
|
||||
uint32_t version;
|
||||
uint32_t pad;
|
||||
|
||||
uint64_t next_bucket;
|
||||
|
||||
struct bucket_disk {
|
||||
uint16_t prio;
|
||||
uint8_t gen;
|
||||
} __attribute((packed)) data[];
|
||||
};
|
||||
|
||||
struct uuid_entry {
|
||||
union {
|
||||
struct {
|
||||
uint8_t uuid[16];
|
||||
uint8_t label[32];
|
||||
uint32_t first_reg;
|
||||
uint32_t last_reg;
|
||||
uint32_t invalidated;
|
||||
|
||||
uint32_t flags;
|
||||
/* Size of flash only volumes */
|
||||
uint64_t sectors;
|
||||
};
|
||||
|
||||
uint8_t pad[128];
|
||||
};
|
||||
};
|
||||
|
||||
BITMASK(UUID_FLASH_ONLY, struct uuid_entry, flags, 0, 1);
|
||||
|
||||
#include "journal.h"
|
||||
#include "stats.h"
|
||||
struct search;
|
||||
@ -384,8 +223,6 @@ struct keybuf_key {
|
||||
void *private;
|
||||
};
|
||||
|
||||
typedef bool (keybuf_pred_fn)(struct keybuf *, struct bkey *);
|
||||
|
||||
struct keybuf {
|
||||
struct bkey last_scanned;
|
||||
spinlock_t lock;
|
||||
@ -400,7 +237,7 @@ struct keybuf {
|
||||
|
||||
struct rb_root keys;
|
||||
|
||||
#define KEYBUF_NR 100
|
||||
#define KEYBUF_NR 500
|
||||
DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR);
|
||||
};
|
||||
|
||||
@ -429,16 +266,15 @@ struct bcache_device {
|
||||
|
||||
struct gendisk *disk;
|
||||
|
||||
/* If nonzero, we're closing */
|
||||
atomic_t closing;
|
||||
unsigned long flags;
|
||||
#define BCACHE_DEV_CLOSING 0
|
||||
#define BCACHE_DEV_DETACHING 1
|
||||
#define BCACHE_DEV_UNLINK_DONE 2
|
||||
|
||||
/* If nonzero, we're detaching/unregistering from cache set */
|
||||
atomic_t detaching;
|
||||
int flush_done;
|
||||
|
||||
uint64_t nr_stripes;
|
||||
unsigned stripe_size_bits;
|
||||
unsigned nr_stripes;
|
||||
unsigned stripe_size;
|
||||
atomic_t *stripe_sectors_dirty;
|
||||
unsigned long *full_dirty_stripes;
|
||||
|
||||
unsigned long sectors_dirty_last;
|
||||
long sectors_dirty_derivative;
|
||||
@ -509,7 +345,7 @@ struct cached_dev {
|
||||
|
||||
/* Limit number of writeback bios in flight */
|
||||
struct semaphore in_flight;
|
||||
struct closure_with_timer writeback;
|
||||
struct task_struct *writeback_thread;
|
||||
|
||||
struct keybuf writeback_keys;
|
||||
|
||||
@ -527,8 +363,8 @@ struct cached_dev {
|
||||
unsigned sequential_cutoff;
|
||||
unsigned readahead;
|
||||
|
||||
unsigned sequential_merge:1;
|
||||
unsigned verify:1;
|
||||
unsigned bypass_torture_test:1;
|
||||
|
||||
unsigned partial_stripes_expensive:1;
|
||||
unsigned writeback_metadata:1;
|
||||
@ -620,15 +456,6 @@ struct cache {
|
||||
|
||||
bool discard; /* Get rid of? */
|
||||
|
||||
/*
|
||||
* We preallocate structs for issuing discards to buckets, and keep them
|
||||
* on this list when they're not in use; do_discard() issues discards
|
||||
* whenever there's work to do and is called by free_some_buckets() and
|
||||
* when a discard finishes.
|
||||
*/
|
||||
atomic_t discards_in_flight;
|
||||
struct list_head discards;
|
||||
|
||||
struct journal_device journal;
|
||||
|
||||
/* The rest of this all shows up in sysfs */
|
||||
@ -649,7 +476,6 @@ struct gc_stat {
|
||||
|
||||
size_t nkeys;
|
||||
uint64_t data; /* sectors */
|
||||
uint64_t dirty; /* sectors */
|
||||
unsigned in_use; /* percent */
|
||||
};
|
||||
|
||||
@ -744,8 +570,8 @@ struct cache_set {
|
||||
* basically a lock for this that we can wait on asynchronously. The
|
||||
* btree_root() macro releases the lock when it returns.
|
||||
*/
|
||||
struct closure *try_harder;
|
||||
struct closure_waitlist try_wait;
|
||||
struct task_struct *try_harder;
|
||||
wait_queue_head_t try_wait;
|
||||
uint64_t try_harder_start;
|
||||
|
||||
/*
|
||||
@ -759,7 +585,7 @@ struct cache_set {
|
||||
* written.
|
||||
*/
|
||||
atomic_t prio_blocked;
|
||||
struct closure_waitlist bucket_wait;
|
||||
wait_queue_head_t bucket_wait;
|
||||
|
||||
/*
|
||||
* For any bio we don't skip we subtract the number of sectors from
|
||||
@ -782,7 +608,7 @@ struct cache_set {
|
||||
struct gc_stat gc_stats;
|
||||
size_t nbuckets;
|
||||
|
||||
struct closure_with_waitlist gc;
|
||||
struct task_struct *gc_thread;
|
||||
/* Where in the btree gc currently is */
|
||||
struct bkey gc_done;
|
||||
|
||||
@ -795,11 +621,10 @@ struct cache_set {
|
||||
/* Counts how many sectors bio_insert has added to the cache */
|
||||
atomic_t sectors_to_gc;
|
||||
|
||||
struct closure moving_gc;
|
||||
struct closure_waitlist moving_gc_wait;
|
||||
wait_queue_head_t moving_gc_wait;
|
||||
struct keybuf moving_gc_keys;
|
||||
/* Number of moving GC bios in flight */
|
||||
atomic_t in_flight;
|
||||
struct semaphore moving_in_flight;
|
||||
|
||||
struct btree *root;
|
||||
|
||||
@ -841,22 +666,27 @@ struct cache_set {
|
||||
unsigned congested_read_threshold_us;
|
||||
unsigned congested_write_threshold_us;
|
||||
|
||||
spinlock_t sort_time_lock;
|
||||
struct time_stats sort_time;
|
||||
struct time_stats btree_gc_time;
|
||||
struct time_stats btree_split_time;
|
||||
spinlock_t btree_read_time_lock;
|
||||
struct time_stats btree_read_time;
|
||||
struct time_stats try_harder_time;
|
||||
|
||||
atomic_long_t cache_read_races;
|
||||
atomic_long_t writeback_keys_done;
|
||||
atomic_long_t writeback_keys_failed;
|
||||
|
||||
enum {
|
||||
ON_ERROR_UNREGISTER,
|
||||
ON_ERROR_PANIC,
|
||||
} on_error;
|
||||
unsigned error_limit;
|
||||
unsigned error_decay;
|
||||
|
||||
unsigned short journal_delay_ms;
|
||||
unsigned verify:1;
|
||||
unsigned key_merging_disabled:1;
|
||||
unsigned expensive_debug_checks:1;
|
||||
unsigned gc_always_rewrite:1;
|
||||
unsigned shrinker_disabled:1;
|
||||
unsigned copy_gc_enabled:1;
|
||||
@ -865,21 +695,6 @@ struct cache_set {
|
||||
struct hlist_head bucket_hash[1 << BUCKET_HASH_BITS];
|
||||
};
|
||||
|
||||
static inline bool key_merging_disabled(struct cache_set *c)
|
||||
{
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
return c->key_merging_disabled;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline bool SB_IS_BDEV(const struct cache_sb *sb)
|
||||
{
|
||||
return sb->version == BCACHE_SB_VERSION_BDEV
|
||||
|| sb->version == BCACHE_SB_VERSION_BDEV_WITH_OFFSET;
|
||||
}
|
||||
|
||||
struct bbio {
|
||||
unsigned submit_time_us;
|
||||
union {
|
||||
@ -933,59 +748,6 @@ static inline unsigned local_clock_us(void)
|
||||
#define prio_buckets(c) \
|
||||
DIV_ROUND_UP((size_t) (c)->sb.nbuckets, prios_per_bucket(c))
|
||||
|
||||
#define JSET_MAGIC 0x245235c1a3625032ULL
|
||||
#define PSET_MAGIC 0x6750e15f87337f91ULL
|
||||
#define BSET_MAGIC 0x90135c78b99e07f5ULL
|
||||
|
||||
#define jset_magic(c) ((c)->sb.set_magic ^ JSET_MAGIC)
|
||||
#define pset_magic(c) ((c)->sb.set_magic ^ PSET_MAGIC)
|
||||
#define bset_magic(c) ((c)->sb.set_magic ^ BSET_MAGIC)
|
||||
|
||||
/* Bkey fields: all units are in sectors */
|
||||
|
||||
#define KEY_FIELD(name, field, offset, size) \
|
||||
BITMASK(name, struct bkey, field, offset, size)
|
||||
|
||||
#define PTR_FIELD(name, offset, size) \
|
||||
static inline uint64_t name(const struct bkey *k, unsigned i) \
|
||||
{ return (k->ptr[i] >> offset) & ~(((uint64_t) ~0) << size); } \
|
||||
\
|
||||
static inline void SET_##name(struct bkey *k, unsigned i, uint64_t v)\
|
||||
{ \
|
||||
k->ptr[i] &= ~(~((uint64_t) ~0 << size) << offset); \
|
||||
k->ptr[i] |= v << offset; \
|
||||
}
|
||||
|
||||
KEY_FIELD(KEY_PTRS, high, 60, 3)
|
||||
KEY_FIELD(HEADER_SIZE, high, 58, 2)
|
||||
KEY_FIELD(KEY_CSUM, high, 56, 2)
|
||||
KEY_FIELD(KEY_PINNED, high, 55, 1)
|
||||
KEY_FIELD(KEY_DIRTY, high, 36, 1)
|
||||
|
||||
KEY_FIELD(KEY_SIZE, high, 20, 16)
|
||||
KEY_FIELD(KEY_INODE, high, 0, 20)
|
||||
|
||||
/* Next time I change the on disk format, KEY_OFFSET() won't be 64 bits */
|
||||
|
||||
static inline uint64_t KEY_OFFSET(const struct bkey *k)
|
||||
{
|
||||
return k->low;
|
||||
}
|
||||
|
||||
static inline void SET_KEY_OFFSET(struct bkey *k, uint64_t v)
|
||||
{
|
||||
k->low = v;
|
||||
}
|
||||
|
||||
PTR_FIELD(PTR_DEV, 51, 12)
|
||||
PTR_FIELD(PTR_OFFSET, 8, 43)
|
||||
PTR_FIELD(PTR_GEN, 0, 8)
|
||||
|
||||
#define PTR_CHECK_DEV ((1 << 12) - 1)
|
||||
|
||||
#define PTR(gen, offset, dev) \
|
||||
((((uint64_t) dev) << 51) | ((uint64_t) offset) << 8 | gen)
|
||||
|
||||
static inline size_t sector_to_bucket(struct cache_set *c, sector_t s)
|
||||
{
|
||||
return s >> c->bucket_bits;
|
||||
@ -1024,27 +786,11 @@ static inline struct bucket *PTR_BUCKET(struct cache_set *c,
|
||||
|
||||
/* Btree key macros */
|
||||
|
||||
/*
|
||||
* The high bit being set is a relic from when we used it to do binary
|
||||
* searches - it told you where a key started. It's not used anymore,
|
||||
* and can probably be safely dropped.
|
||||
*/
|
||||
#define KEY(dev, sector, len) \
|
||||
((struct bkey) { \
|
||||
.high = (1ULL << 63) | ((uint64_t) (len) << 20) | (dev), \
|
||||
.low = (sector) \
|
||||
})
|
||||
|
||||
static inline void bkey_init(struct bkey *k)
|
||||
{
|
||||
*k = KEY(0, 0, 0);
|
||||
*k = ZERO_KEY;
|
||||
}
|
||||
|
||||
#define KEY_START(k) (KEY_OFFSET(k) - KEY_SIZE(k))
|
||||
#define START_KEY(k) KEY(KEY_INODE(k), KEY_START(k), 0)
|
||||
#define MAX_KEY KEY(~(~0 << 20), ((uint64_t) ~0) >> 1, 0)
|
||||
#define ZERO_KEY KEY(0, 0, 0)
|
||||
|
||||
/*
|
||||
* This is used for various on disk data structures - cache_sb, prio_set, bset,
|
||||
* jset: The checksum is _always_ the first 8 bytes of these structs
|
||||
@ -1094,14 +840,6 @@ do { \
|
||||
for (b = (ca)->buckets + (ca)->sb.first_bucket; \
|
||||
b < (ca)->buckets + (ca)->sb.nbuckets; b++)
|
||||
|
||||
static inline void __bkey_put(struct cache_set *c, struct bkey *k)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
for (i = 0; i < KEY_PTRS(k); i++)
|
||||
atomic_dec_bug(&PTR_BUCKET(c, k, i)->pin);
|
||||
}
|
||||
|
||||
static inline void cached_dev_put(struct cached_dev *dc)
|
||||
{
|
||||
if (atomic_dec_and_test(&dc->count))
|
||||
@ -1173,13 +911,15 @@ uint8_t bch_inc_gen(struct cache *, struct bucket *);
|
||||
void bch_rescale_priorities(struct cache_set *, int);
|
||||
bool bch_bucket_add_unused(struct cache *, struct bucket *);
|
||||
|
||||
long bch_bucket_alloc(struct cache *, unsigned, struct closure *);
|
||||
long bch_bucket_alloc(struct cache *, unsigned, bool);
|
||||
void bch_bucket_free(struct cache_set *, struct bkey *);
|
||||
|
||||
int __bch_bucket_alloc_set(struct cache_set *, unsigned,
|
||||
struct bkey *, int, struct closure *);
|
||||
struct bkey *, int, bool);
|
||||
int bch_bucket_alloc_set(struct cache_set *, unsigned,
|
||||
struct bkey *, int, struct closure *);
|
||||
struct bkey *, int, bool);
|
||||
bool bch_alloc_sectors(struct cache_set *, struct bkey *, unsigned,
|
||||
unsigned, unsigned, bool);
|
||||
|
||||
__printf(2, 3)
|
||||
bool bch_cache_set_error(struct cache_set *, const char *, ...);
|
||||
@ -1187,7 +927,7 @@ bool bch_cache_set_error(struct cache_set *, const char *, ...);
|
||||
void bch_prio_write(struct cache *);
|
||||
void bch_write_bdev_super(struct cached_dev *, struct closure *);
|
||||
|
||||
extern struct workqueue_struct *bcache_wq, *bch_gc_wq;
|
||||
extern struct workqueue_struct *bcache_wq;
|
||||
extern const char * const bch_cache_modes[];
|
||||
extern struct mutex bch_register_lock;
|
||||
extern struct list_head bch_cache_sets;
|
||||
@ -1220,15 +960,14 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *);
|
||||
void bch_btree_cache_free(struct cache_set *);
|
||||
int bch_btree_cache_alloc(struct cache_set *);
|
||||
void bch_moving_init_cache_set(struct cache_set *);
|
||||
int bch_open_buckets_alloc(struct cache_set *);
|
||||
void bch_open_buckets_free(struct cache_set *);
|
||||
|
||||
int bch_cache_allocator_start(struct cache *ca);
|
||||
void bch_cache_allocator_exit(struct cache *ca);
|
||||
int bch_cache_allocator_init(struct cache *ca);
|
||||
|
||||
void bch_debug_exit(void);
|
||||
int bch_debug_init(struct kobject *);
|
||||
void bch_writeback_exit(void);
|
||||
int bch_writeback_init(void);
|
||||
void bch_request_exit(void);
|
||||
int bch_request_init(void);
|
||||
void bch_btree_exit(void);
|
||||
|
@ -14,22 +14,12 @@
|
||||
|
||||
/* Keylists */
|
||||
|
||||
void bch_keylist_copy(struct keylist *dest, struct keylist *src)
|
||||
{
|
||||
*dest = *src;
|
||||
|
||||
if (src->list == src->d) {
|
||||
size_t n = (uint64_t *) src->top - src->d;
|
||||
dest->top = (struct bkey *) &dest->d[n];
|
||||
dest->list = dest->d;
|
||||
}
|
||||
}
|
||||
|
||||
int bch_keylist_realloc(struct keylist *l, int nptrs, struct cache_set *c)
|
||||
{
|
||||
unsigned oldsize = (uint64_t *) l->top - l->list;
|
||||
unsigned newsize = oldsize + 2 + nptrs;
|
||||
uint64_t *new;
|
||||
size_t oldsize = bch_keylist_nkeys(l);
|
||||
size_t newsize = oldsize + 2 + nptrs;
|
||||
uint64_t *old_keys = l->keys_p == l->inline_keys ? NULL : l->keys_p;
|
||||
uint64_t *new_keys;
|
||||
|
||||
/* The journalling code doesn't handle the case where the keys to insert
|
||||
* is bigger than an empty write: If we just return -ENOMEM here,
|
||||
@ -45,24 +35,23 @@ int bch_keylist_realloc(struct keylist *l, int nptrs, struct cache_set *c)
|
||||
roundup_pow_of_two(oldsize) == newsize)
|
||||
return 0;
|
||||
|
||||
new = krealloc(l->list == l->d ? NULL : l->list,
|
||||
sizeof(uint64_t) * newsize, GFP_NOIO);
|
||||
new_keys = krealloc(old_keys, sizeof(uint64_t) * newsize, GFP_NOIO);
|
||||
|
||||
if (!new)
|
||||
if (!new_keys)
|
||||
return -ENOMEM;
|
||||
|
||||
if (l->list == l->d)
|
||||
memcpy(new, l->list, sizeof(uint64_t) * KEYLIST_INLINE);
|
||||
if (!old_keys)
|
||||
memcpy(new_keys, l->inline_keys, sizeof(uint64_t) * oldsize);
|
||||
|
||||
l->list = new;
|
||||
l->top = (struct bkey *) (&l->list[oldsize]);
|
||||
l->keys_p = new_keys;
|
||||
l->top_p = new_keys + oldsize;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct bkey *bch_keylist_pop(struct keylist *l)
|
||||
{
|
||||
struct bkey *k = l->bottom;
|
||||
struct bkey *k = l->keys;
|
||||
|
||||
if (k == l->top)
|
||||
return NULL;
|
||||
@ -73,21 +62,20 @@ struct bkey *bch_keylist_pop(struct keylist *l)
|
||||
return l->top = k;
|
||||
}
|
||||
|
||||
void bch_keylist_pop_front(struct keylist *l)
|
||||
{
|
||||
l->top_p -= bkey_u64s(l->keys);
|
||||
|
||||
memmove(l->keys,
|
||||
bkey_next(l->keys),
|
||||
bch_keylist_bytes(l));
|
||||
}
|
||||
|
||||
/* Pointer validation */
|
||||
|
||||
bool __bch_ptr_invalid(struct cache_set *c, int level, const struct bkey *k)
|
||||
static bool __ptr_invalid(struct cache_set *c, const struct bkey *k)
|
||||
{
|
||||
unsigned i;
|
||||
char buf[80];
|
||||
|
||||
if (level && (!KEY_PTRS(k) || !KEY_SIZE(k) || KEY_DIRTY(k)))
|
||||
goto bad;
|
||||
|
||||
if (!level && KEY_SIZE(k) > KEY_OFFSET(k))
|
||||
goto bad;
|
||||
|
||||
if (!KEY_SIZE(k))
|
||||
return true;
|
||||
|
||||
for (i = 0; i < KEY_PTRS(k); i++)
|
||||
if (ptr_available(c, k, i)) {
|
||||
@ -98,13 +86,83 @@ bool __bch_ptr_invalid(struct cache_set *c, int level, const struct bkey *k)
|
||||
if (KEY_SIZE(k) + r > c->sb.bucket_size ||
|
||||
bucket < ca->sb.first_bucket ||
|
||||
bucket >= ca->sb.nbuckets)
|
||||
goto bad;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool bch_btree_ptr_invalid(struct cache_set *c, const struct bkey *k)
|
||||
{
|
||||
char buf[80];
|
||||
|
||||
if (!KEY_PTRS(k) || !KEY_SIZE(k) || KEY_DIRTY(k))
|
||||
goto bad;
|
||||
|
||||
if (__ptr_invalid(c, k))
|
||||
goto bad;
|
||||
|
||||
return false;
|
||||
bad:
|
||||
bch_bkey_to_text(buf, sizeof(buf), k);
|
||||
cache_bug(c, "spotted bad key %s: %s", buf, bch_ptr_status(c, k));
|
||||
cache_bug(c, "spotted btree ptr %s: %s", buf, bch_ptr_status(c, k));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool bch_extent_ptr_invalid(struct cache_set *c, const struct bkey *k)
|
||||
{
|
||||
char buf[80];
|
||||
|
||||
if (!KEY_SIZE(k))
|
||||
return true;
|
||||
|
||||
if (KEY_SIZE(k) > KEY_OFFSET(k))
|
||||
goto bad;
|
||||
|
||||
if (__ptr_invalid(c, k))
|
||||
goto bad;
|
||||
|
||||
return false;
|
||||
bad:
|
||||
bch_bkey_to_text(buf, sizeof(buf), k);
|
||||
cache_bug(c, "spotted extent %s: %s", buf, bch_ptr_status(c, k));
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ptr_bad_expensive_checks(struct btree *b, const struct bkey *k,
|
||||
unsigned ptr)
|
||||
{
|
||||
struct bucket *g = PTR_BUCKET(b->c, k, ptr);
|
||||
char buf[80];
|
||||
|
||||
if (mutex_trylock(&b->c->bucket_lock)) {
|
||||
if (b->level) {
|
||||
if (KEY_DIRTY(k) ||
|
||||
g->prio != BTREE_PRIO ||
|
||||
(b->c->gc_mark_valid &&
|
||||
GC_MARK(g) != GC_MARK_METADATA))
|
||||
goto err;
|
||||
|
||||
} else {
|
||||
if (g->prio == BTREE_PRIO)
|
||||
goto err;
|
||||
|
||||
if (KEY_DIRTY(k) &&
|
||||
b->c->gc_mark_valid &&
|
||||
GC_MARK(g) != GC_MARK_DIRTY)
|
||||
goto err;
|
||||
}
|
||||
mutex_unlock(&b->c->bucket_lock);
|
||||
}
|
||||
|
||||
return false;
|
||||
err:
|
||||
mutex_unlock(&b->c->bucket_lock);
|
||||
bch_bkey_to_text(buf, sizeof(buf), k);
|
||||
btree_bug(b,
|
||||
"inconsistent pointer %s: bucket %zu pin %i prio %i gen %i last_gc %i mark %llu gc_gen %i",
|
||||
buf, PTR_BUCKET_NR(b->c, k, ptr), atomic_read(&g->pin),
|
||||
g->prio, g->gen, g->last_gc, GC_MARK(g), g->gc_gen);
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -118,64 +176,29 @@ bool bch_ptr_bad(struct btree *b, const struct bkey *k)
|
||||
bch_ptr_invalid(b, k))
|
||||
return true;
|
||||
|
||||
if (KEY_PTRS(k) && PTR_DEV(k, 0) == PTR_CHECK_DEV)
|
||||
return true;
|
||||
for (i = 0; i < KEY_PTRS(k); i++) {
|
||||
if (!ptr_available(b->c, k, i))
|
||||
return true;
|
||||
|
||||
for (i = 0; i < KEY_PTRS(k); i++)
|
||||
if (ptr_available(b->c, k, i)) {
|
||||
g = PTR_BUCKET(b->c, k, i);
|
||||
stale = ptr_stale(b->c, k, i);
|
||||
g = PTR_BUCKET(b->c, k, i);
|
||||
stale = ptr_stale(b->c, k, i);
|
||||
|
||||
btree_bug_on(stale > 96, b,
|
||||
"key too stale: %i, need_gc %u",
|
||||
stale, b->c->need_gc);
|
||||
btree_bug_on(stale > 96, b,
|
||||
"key too stale: %i, need_gc %u",
|
||||
stale, b->c->need_gc);
|
||||
|
||||
btree_bug_on(stale && KEY_DIRTY(k) && KEY_SIZE(k),
|
||||
b, "stale dirty pointer");
|
||||
btree_bug_on(stale && KEY_DIRTY(k) && KEY_SIZE(k),
|
||||
b, "stale dirty pointer");
|
||||
|
||||
if (stale)
|
||||
return true;
|
||||
if (stale)
|
||||
return true;
|
||||
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
if (!mutex_trylock(&b->c->bucket_lock))
|
||||
continue;
|
||||
|
||||
if (b->level) {
|
||||
if (KEY_DIRTY(k) ||
|
||||
g->prio != BTREE_PRIO ||
|
||||
(b->c->gc_mark_valid &&
|
||||
GC_MARK(g) != GC_MARK_METADATA))
|
||||
goto bug;
|
||||
|
||||
} else {
|
||||
if (g->prio == BTREE_PRIO)
|
||||
goto bug;
|
||||
|
||||
if (KEY_DIRTY(k) &&
|
||||
b->c->gc_mark_valid &&
|
||||
GC_MARK(g) != GC_MARK_DIRTY)
|
||||
goto bug;
|
||||
}
|
||||
mutex_unlock(&b->c->bucket_lock);
|
||||
#endif
|
||||
}
|
||||
if (expensive_debug_checks(b->c) &&
|
||||
ptr_bad_expensive_checks(b, k, i))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
bug:
|
||||
mutex_unlock(&b->c->bucket_lock);
|
||||
|
||||
{
|
||||
char buf[80];
|
||||
|
||||
bch_bkey_to_text(buf, sizeof(buf), k);
|
||||
btree_bug(b,
|
||||
"inconsistent pointer %s: bucket %zu pin %i prio %i gen %i last_gc %i mark %llu gc_gen %i",
|
||||
buf, PTR_BUCKET_NR(b->c, k, i), atomic_read(&g->pin),
|
||||
g->prio, g->gen, g->last_gc, GC_MARK(g), g->gc_gen);
|
||||
}
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Key/pointer manipulation */
|
||||
@ -458,16 +481,8 @@ static struct bkey *table_to_bkey(struct bset_tree *t, unsigned cacheline)
|
||||
|
||||
static inline uint64_t shrd128(uint64_t high, uint64_t low, uint8_t shift)
|
||||
{
|
||||
#ifdef CONFIG_X86_64
|
||||
asm("shrd %[shift],%[high],%[low]"
|
||||
: [low] "+Rm" (low)
|
||||
: [high] "R" (high),
|
||||
[shift] "ci" (shift)
|
||||
: "cc");
|
||||
#else
|
||||
low >>= shift;
|
||||
low |= (high << 1) << (63U - shift);
|
||||
#endif
|
||||
return low;
|
||||
}
|
||||
|
||||
@ -686,7 +701,7 @@ void bch_bset_init_next(struct btree *b)
|
||||
} else
|
||||
get_random_bytes(&i->seq, sizeof(uint64_t));
|
||||
|
||||
i->magic = bset_magic(b->c);
|
||||
i->magic = bset_magic(&b->c->sb);
|
||||
i->version = 0;
|
||||
i->keys = 0;
|
||||
|
||||
@ -824,16 +839,16 @@ struct bkey *__bch_bset_search(struct btree *b, struct bset_tree *t,
|
||||
} else
|
||||
i = bset_search_write_set(b, t, search);
|
||||
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
BUG_ON(bset_written(b, t) &&
|
||||
i.l != t->data->start &&
|
||||
bkey_cmp(tree_to_prev_bkey(t,
|
||||
inorder_to_tree(bkey_to_cacheline(t, i.l), t)),
|
||||
search) > 0);
|
||||
if (expensive_debug_checks(b->c)) {
|
||||
BUG_ON(bset_written(b, t) &&
|
||||
i.l != t->data->start &&
|
||||
bkey_cmp(tree_to_prev_bkey(t,
|
||||
inorder_to_tree(bkey_to_cacheline(t, i.l), t)),
|
||||
search) > 0);
|
||||
|
||||
BUG_ON(i.r != end(t->data) &&
|
||||
bkey_cmp(i.r, search) <= 0);
|
||||
#endif
|
||||
BUG_ON(i.r != end(t->data) &&
|
||||
bkey_cmp(i.r, search) <= 0);
|
||||
}
|
||||
|
||||
while (likely(i.l != i.r) &&
|
||||
bkey_cmp(i.l, search) <= 0)
|
||||
@ -844,6 +859,13 @@ struct bkey *__bch_bset_search(struct btree *b, struct bset_tree *t,
|
||||
|
||||
/* Btree iterator */
|
||||
|
||||
/*
|
||||
* Returns true if l > r - unless l == r, in which case returns true if l is
|
||||
* older than r.
|
||||
*
|
||||
* Necessary for btree_sort_fixup() - if there are multiple keys that compare
|
||||
* equal in different sets, we have to process them newest to oldest.
|
||||
*/
|
||||
static inline bool btree_iter_cmp(struct btree_iter_set l,
|
||||
struct btree_iter_set r)
|
||||
{
|
||||
@ -867,12 +889,16 @@ void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k,
|
||||
}
|
||||
|
||||
struct bkey *__bch_btree_iter_init(struct btree *b, struct btree_iter *iter,
|
||||
struct bkey *search, struct bset_tree *start)
|
||||
struct bkey *search, struct bset_tree *start)
|
||||
{
|
||||
struct bkey *ret = NULL;
|
||||
iter->size = ARRAY_SIZE(iter->data);
|
||||
iter->used = 0;
|
||||
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
iter->b = b;
|
||||
#endif
|
||||
|
||||
for (; start <= &b->sets[b->nsets]; start++) {
|
||||
ret = bch_bset_search(b, start, search);
|
||||
bch_btree_iter_push(iter, ret, end(start->data));
|
||||
@ -887,6 +913,8 @@ struct bkey *bch_btree_iter_next(struct btree_iter *iter)
|
||||
struct bkey *ret = NULL;
|
||||
|
||||
if (!btree_iter_end(iter)) {
|
||||
bch_btree_iter_next_check(iter);
|
||||
|
||||
ret = iter->data->k;
|
||||
iter->data->k = bkey_next(iter->data->k);
|
||||
|
||||
@ -916,14 +944,6 @@ struct bkey *bch_btree_iter_next_filter(struct btree_iter *iter,
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct bkey *bch_next_recurse_key(struct btree *b, struct bkey *search)
|
||||
{
|
||||
struct btree_iter iter;
|
||||
|
||||
bch_btree_iter_init(b, &iter, search);
|
||||
return bch_btree_iter_next_filter(&iter, b, bch_ptr_bad);
|
||||
}
|
||||
|
||||
/* Mergesort */
|
||||
|
||||
static void sort_key_next(struct btree_iter *iter,
|
||||
@ -998,7 +1018,6 @@ static void btree_mergesort(struct btree *b, struct bset *out,
|
||||
out->keys = last ? (uint64_t *) bkey_next(last) - out->d : 0;
|
||||
|
||||
pr_debug("sorted %i keys", out->keys);
|
||||
bch_check_key_order(b, out);
|
||||
}
|
||||
|
||||
static void __btree_sort(struct btree *b, struct btree_iter *iter,
|
||||
@ -1029,7 +1048,7 @@ static void __btree_sort(struct btree *b, struct btree_iter *iter,
|
||||
* memcpy()
|
||||
*/
|
||||
|
||||
out->magic = bset_magic(b->c);
|
||||
out->magic = bset_magic(&b->c->sb);
|
||||
out->seq = b->sets[0].data->seq;
|
||||
out->version = b->sets[0].data->version;
|
||||
swap(out, b->sets[0].data);
|
||||
@ -1050,24 +1069,21 @@ static void __btree_sort(struct btree *b, struct btree_iter *iter,
|
||||
if (b->written)
|
||||
bset_build_written_tree(b);
|
||||
|
||||
if (!start) {
|
||||
spin_lock(&b->c->sort_time_lock);
|
||||
if (!start)
|
||||
bch_time_stats_update(&b->c->sort_time, start_time);
|
||||
spin_unlock(&b->c->sort_time_lock);
|
||||
}
|
||||
}
|
||||
|
||||
void bch_btree_sort_partial(struct btree *b, unsigned start)
|
||||
{
|
||||
size_t oldsize = 0, order = b->page_order, keys = 0;
|
||||
size_t order = b->page_order, keys = 0;
|
||||
struct btree_iter iter;
|
||||
int oldsize = bch_count_data(b);
|
||||
|
||||
__bch_btree_iter_init(b, &iter, NULL, &b->sets[start]);
|
||||
|
||||
BUG_ON(b->sets[b->nsets].data == write_block(b) &&
|
||||
(b->sets[b->nsets].size || b->nsets));
|
||||
|
||||
if (b->written)
|
||||
oldsize = bch_count_data(b);
|
||||
|
||||
if (start) {
|
||||
unsigned i;
|
||||
@ -1083,7 +1099,7 @@ void bch_btree_sort_partial(struct btree *b, unsigned start)
|
||||
|
||||
__btree_sort(b, &iter, start, order, false);
|
||||
|
||||
EBUG_ON(b->written && bch_count_data(b) != oldsize);
|
||||
EBUG_ON(b->written && oldsize >= 0 && bch_count_data(b) != oldsize);
|
||||
}
|
||||
|
||||
void bch_btree_sort_and_fix_extents(struct btree *b, struct btree_iter *iter)
|
||||
@ -1101,9 +1117,7 @@ void bch_btree_sort_into(struct btree *b, struct btree *new)
|
||||
|
||||
btree_mergesort(b, new->sets->data, &iter, false, true);
|
||||
|
||||
spin_lock(&b->c->sort_time_lock);
|
||||
bch_time_stats_update(&b->c->sort_time, start_time);
|
||||
spin_unlock(&b->c->sort_time_lock);
|
||||
|
||||
bkey_copy_key(&new->key, &b->key);
|
||||
new->sets->size = 0;
|
||||
@ -1148,16 +1162,16 @@ out:
|
||||
/* Sysfs stuff */
|
||||
|
||||
struct bset_stats {
|
||||
struct btree_op op;
|
||||
size_t nodes;
|
||||
size_t sets_written, sets_unwritten;
|
||||
size_t bytes_written, bytes_unwritten;
|
||||
size_t floats, failed;
|
||||
};
|
||||
|
||||
static int bch_btree_bset_stats(struct btree *b, struct btree_op *op,
|
||||
struct bset_stats *stats)
|
||||
static int btree_bset_stats(struct btree_op *op, struct btree *b)
|
||||
{
|
||||
struct bkey *k;
|
||||
struct bset_stats *stats = container_of(op, struct bset_stats, op);
|
||||
unsigned i;
|
||||
|
||||
stats->nodes++;
|
||||
@ -1182,30 +1196,19 @@ static int bch_btree_bset_stats(struct btree *b, struct btree_op *op,
|
||||
}
|
||||
}
|
||||
|
||||
if (b->level) {
|
||||
struct btree_iter iter;
|
||||
|
||||
for_each_key_filter(b, k, &iter, bch_ptr_bad) {
|
||||
int ret = btree(bset_stats, k, b, op, stats);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
return MAP_CONTINUE;
|
||||
}
|
||||
|
||||
int bch_bset_print_stats(struct cache_set *c, char *buf)
|
||||
{
|
||||
struct btree_op op;
|
||||
struct bset_stats t;
|
||||
int ret;
|
||||
|
||||
bch_btree_op_init_stack(&op);
|
||||
memset(&t, 0, sizeof(struct bset_stats));
|
||||
bch_btree_op_init(&t.op, -1);
|
||||
|
||||
ret = btree_root(bset_stats, c, &op, &t);
|
||||
if (ret)
|
||||
ret = bch_btree_map_nodes(&t.op, c, &ZERO_KEY, btree_bset_stats);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
return snprintf(buf, PAGE_SIZE,
|
||||
|
@ -148,6 +148,9 @@
|
||||
|
||||
struct btree_iter {
|
||||
size_t size, used;
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
struct btree *b;
|
||||
#endif
|
||||
struct btree_iter_set {
|
||||
struct bkey *k, *end;
|
||||
} data[MAX_BSETS];
|
||||
@ -193,54 +196,26 @@ static __always_inline int64_t bkey_cmp(const struct bkey *l,
|
||||
: (int64_t) KEY_OFFSET(l) - (int64_t) KEY_OFFSET(r);
|
||||
}
|
||||
|
||||
static inline size_t bkey_u64s(const struct bkey *k)
|
||||
{
|
||||
BUG_ON(KEY_CSUM(k) > 1);
|
||||
return 2 + KEY_PTRS(k) + (KEY_CSUM(k) ? 1 : 0);
|
||||
}
|
||||
|
||||
static inline size_t bkey_bytes(const struct bkey *k)
|
||||
{
|
||||
return bkey_u64s(k) * sizeof(uint64_t);
|
||||
}
|
||||
|
||||
static inline void bkey_copy(struct bkey *dest, const struct bkey *src)
|
||||
{
|
||||
memcpy(dest, src, bkey_bytes(src));
|
||||
}
|
||||
|
||||
static inline void bkey_copy_key(struct bkey *dest, const struct bkey *src)
|
||||
{
|
||||
if (!src)
|
||||
src = &KEY(0, 0, 0);
|
||||
|
||||
SET_KEY_INODE(dest, KEY_INODE(src));
|
||||
SET_KEY_OFFSET(dest, KEY_OFFSET(src));
|
||||
}
|
||||
|
||||
static inline struct bkey *bkey_next(const struct bkey *k)
|
||||
{
|
||||
uint64_t *d = (void *) k;
|
||||
return (struct bkey *) (d + bkey_u64s(k));
|
||||
}
|
||||
|
||||
/* Keylists */
|
||||
|
||||
struct keylist {
|
||||
struct bkey *top;
|
||||
union {
|
||||
uint64_t *list;
|
||||
struct bkey *bottom;
|
||||
struct bkey *keys;
|
||||
uint64_t *keys_p;
|
||||
};
|
||||
union {
|
||||
struct bkey *top;
|
||||
uint64_t *top_p;
|
||||
};
|
||||
|
||||
/* Enough room for btree_split's keys without realloc */
|
||||
#define KEYLIST_INLINE 16
|
||||
uint64_t d[KEYLIST_INLINE];
|
||||
uint64_t inline_keys[KEYLIST_INLINE];
|
||||
};
|
||||
|
||||
static inline void bch_keylist_init(struct keylist *l)
|
||||
{
|
||||
l->top = (void *) (l->list = l->d);
|
||||
l->top_p = l->keys_p = l->inline_keys;
|
||||
}
|
||||
|
||||
static inline void bch_keylist_push(struct keylist *l)
|
||||
@ -256,17 +231,32 @@ static inline void bch_keylist_add(struct keylist *l, struct bkey *k)
|
||||
|
||||
static inline bool bch_keylist_empty(struct keylist *l)
|
||||
{
|
||||
return l->top == (void *) l->list;
|
||||
return l->top == l->keys;
|
||||
}
|
||||
|
||||
static inline void bch_keylist_reset(struct keylist *l)
|
||||
{
|
||||
l->top = l->keys;
|
||||
}
|
||||
|
||||
static inline void bch_keylist_free(struct keylist *l)
|
||||
{
|
||||
if (l->list != l->d)
|
||||
kfree(l->list);
|
||||
if (l->keys_p != l->inline_keys)
|
||||
kfree(l->keys_p);
|
||||
}
|
||||
|
||||
static inline size_t bch_keylist_nkeys(struct keylist *l)
|
||||
{
|
||||
return l->top_p - l->keys_p;
|
||||
}
|
||||
|
||||
static inline size_t bch_keylist_bytes(struct keylist *l)
|
||||
{
|
||||
return bch_keylist_nkeys(l) * sizeof(uint64_t);
|
||||
}
|
||||
|
||||
void bch_keylist_copy(struct keylist *, struct keylist *);
|
||||
struct bkey *bch_keylist_pop(struct keylist *);
|
||||
void bch_keylist_pop_front(struct keylist *);
|
||||
int bch_keylist_realloc(struct keylist *, int, struct cache_set *);
|
||||
|
||||
void bch_bkey_copy_single_ptr(struct bkey *, const struct bkey *,
|
||||
@ -287,7 +277,9 @@ static inline bool bch_cut_back(const struct bkey *where, struct bkey *k)
|
||||
}
|
||||
|
||||
const char *bch_ptr_status(struct cache_set *, const struct bkey *);
|
||||
bool __bch_ptr_invalid(struct cache_set *, int level, const struct bkey *);
|
||||
bool bch_btree_ptr_invalid(struct cache_set *, const struct bkey *);
|
||||
bool bch_extent_ptr_invalid(struct cache_set *, const struct bkey *);
|
||||
|
||||
bool bch_ptr_bad(struct btree *, const struct bkey *);
|
||||
|
||||
static inline uint8_t gen_after(uint8_t a, uint8_t b)
|
||||
@ -311,7 +303,6 @@ static inline bool ptr_available(struct cache_set *c, const struct bkey *k,
|
||||
|
||||
typedef bool (*ptr_filter_fn)(struct btree *, const struct bkey *);
|
||||
|
||||
struct bkey *bch_next_recurse_key(struct btree *, struct bkey *);
|
||||
struct bkey *bch_btree_iter_next(struct btree_iter *);
|
||||
struct bkey *bch_btree_iter_next_filter(struct btree_iter *,
|
||||
struct btree *, ptr_filter_fn);
|
||||
@ -361,12 +352,30 @@ void bch_bset_fix_lookup_table(struct btree *, struct bkey *);
|
||||
struct bkey *__bch_bset_search(struct btree *, struct bset_tree *,
|
||||
const struct bkey *);
|
||||
|
||||
/*
|
||||
* Returns the first key that is strictly greater than search
|
||||
*/
|
||||
static inline struct bkey *bch_bset_search(struct btree *b, struct bset_tree *t,
|
||||
const struct bkey *search)
|
||||
{
|
||||
return search ? __bch_bset_search(b, t, search) : t->data->start;
|
||||
}
|
||||
|
||||
#define PRECEDING_KEY(_k) \
|
||||
({ \
|
||||
struct bkey *_ret = NULL; \
|
||||
\
|
||||
if (KEY_INODE(_k) || KEY_OFFSET(_k)) { \
|
||||
_ret = &KEY(KEY_INODE(_k), KEY_OFFSET(_k), 0); \
|
||||
\
|
||||
if (!_ret->low) \
|
||||
_ret->high--; \
|
||||
_ret->low--; \
|
||||
} \
|
||||
\
|
||||
_ret; \
|
||||
})
|
||||
|
||||
bool bch_bkey_try_merge(struct btree *, struct bkey *, struct bkey *);
|
||||
void bch_btree_sort_lazy(struct btree *);
|
||||
void bch_btree_sort_into(struct btree *, struct btree *);
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -125,6 +125,7 @@ struct btree {
|
||||
unsigned long seq;
|
||||
struct rw_semaphore lock;
|
||||
struct cache_set *c;
|
||||
struct btree *parent;
|
||||
|
||||
unsigned long flags;
|
||||
uint16_t written; /* would be nice to kill */
|
||||
@ -200,12 +201,7 @@ static inline bool bkey_written(struct btree *b, struct bkey *k)
|
||||
|
||||
static inline void set_gc_sectors(struct cache_set *c)
|
||||
{
|
||||
atomic_set(&c->sectors_to_gc, c->sb.bucket_size * c->nbuckets / 8);
|
||||
}
|
||||
|
||||
static inline bool bch_ptr_invalid(struct btree *b, const struct bkey *k)
|
||||
{
|
||||
return __bch_ptr_invalid(b->c, b->level, k);
|
||||
atomic_set(&c->sectors_to_gc, c->sb.bucket_size * c->nbuckets / 16);
|
||||
}
|
||||
|
||||
static inline struct bkey *bch_btree_iter_init(struct btree *b,
|
||||
@ -215,6 +211,16 @@ static inline struct bkey *bch_btree_iter_init(struct btree *b,
|
||||
return __bch_btree_iter_init(b, iter, search, b->sets);
|
||||
}
|
||||
|
||||
static inline bool bch_ptr_invalid(struct btree *b, const struct bkey *k)
|
||||
{
|
||||
if (b->level)
|
||||
return bch_btree_ptr_invalid(b->c, k);
|
||||
else
|
||||
return bch_extent_ptr_invalid(b->c, k);
|
||||
}
|
||||
|
||||
void bkey_put(struct cache_set *c, struct bkey *k);
|
||||
|
||||
/* Looping macros */
|
||||
|
||||
#define for_each_cached_btree(b, c, iter) \
|
||||
@ -234,51 +240,17 @@ static inline struct bkey *bch_btree_iter_init(struct btree *b,
|
||||
/* Recursing down the btree */
|
||||
|
||||
struct btree_op {
|
||||
struct closure cl;
|
||||
struct cache_set *c;
|
||||
|
||||
/* Journal entry we have a refcount on */
|
||||
atomic_t *journal;
|
||||
|
||||
/* Bio to be inserted into the cache */
|
||||
struct bio *cache_bio;
|
||||
|
||||
unsigned inode;
|
||||
|
||||
uint16_t write_prio;
|
||||
|
||||
/* Btree level at which we start taking write locks */
|
||||
short lock;
|
||||
|
||||
/* Btree insertion type */
|
||||
enum {
|
||||
BTREE_INSERT,
|
||||
BTREE_REPLACE
|
||||
} type:8;
|
||||
|
||||
unsigned csum:1;
|
||||
unsigned skip:1;
|
||||
unsigned flush_journal:1;
|
||||
|
||||
unsigned insert_data_done:1;
|
||||
unsigned lookup_done:1;
|
||||
unsigned insert_collision:1;
|
||||
|
||||
/* Anything after this point won't get zeroed in do_bio_hook() */
|
||||
|
||||
/* Keys to be inserted */
|
||||
struct keylist keys;
|
||||
BKEY_PADDED(replace);
|
||||
};
|
||||
|
||||
enum {
|
||||
BTREE_INSERT_STATUS_INSERT,
|
||||
BTREE_INSERT_STATUS_BACK_MERGE,
|
||||
BTREE_INSERT_STATUS_OVERWROTE,
|
||||
BTREE_INSERT_STATUS_FRONT_MERGE,
|
||||
};
|
||||
|
||||
void bch_btree_op_init_stack(struct btree_op *);
|
||||
static inline void bch_btree_op_init(struct btree_op *op, int write_lock_level)
|
||||
{
|
||||
memset(op, 0, sizeof(struct btree_op));
|
||||
op->lock = write_lock_level;
|
||||
}
|
||||
|
||||
static inline void rw_lock(bool w, struct btree *b, int level)
|
||||
{
|
||||
@ -290,108 +262,71 @@ static inline void rw_lock(bool w, struct btree *b, int level)
|
||||
|
||||
static inline void rw_unlock(bool w, struct btree *b)
|
||||
{
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
unsigned i;
|
||||
|
||||
if (w && b->key.ptr[0])
|
||||
for (i = 0; i <= b->nsets; i++)
|
||||
bch_check_key_order(b, b->sets[i].data);
|
||||
#endif
|
||||
|
||||
if (w)
|
||||
b->seq++;
|
||||
(w ? up_write : up_read)(&b->lock);
|
||||
}
|
||||
|
||||
#define insert_lock(s, b) ((b)->level <= (s)->lock)
|
||||
|
||||
/*
|
||||
* These macros are for recursing down the btree - they handle the details of
|
||||
* locking and looking up nodes in the cache for you. They're best treated as
|
||||
* mere syntax when reading code that uses them.
|
||||
*
|
||||
* op->lock determines whether we take a read or a write lock at a given depth.
|
||||
* If you've got a read lock and find that you need a write lock (i.e. you're
|
||||
* going to have to split), set op->lock and return -EINTR; btree_root() will
|
||||
* call you again and you'll have the correct lock.
|
||||
*/
|
||||
|
||||
/**
|
||||
* btree - recurse down the btree on a specified key
|
||||
* @fn: function to call, which will be passed the child node
|
||||
* @key: key to recurse on
|
||||
* @b: parent btree node
|
||||
* @op: pointer to struct btree_op
|
||||
*/
|
||||
#define btree(fn, key, b, op, ...) \
|
||||
({ \
|
||||
int _r, l = (b)->level - 1; \
|
||||
bool _w = l <= (op)->lock; \
|
||||
struct btree *_b = bch_btree_node_get((b)->c, key, l, op); \
|
||||
if (!IS_ERR(_b)) { \
|
||||
_r = bch_btree_ ## fn(_b, op, ##__VA_ARGS__); \
|
||||
rw_unlock(_w, _b); \
|
||||
} else \
|
||||
_r = PTR_ERR(_b); \
|
||||
_r; \
|
||||
})
|
||||
|
||||
/**
|
||||
* btree_root - call a function on the root of the btree
|
||||
* @fn: function to call, which will be passed the child node
|
||||
* @c: cache set
|
||||
* @op: pointer to struct btree_op
|
||||
*/
|
||||
#define btree_root(fn, c, op, ...) \
|
||||
({ \
|
||||
int _r = -EINTR; \
|
||||
do { \
|
||||
struct btree *_b = (c)->root; \
|
||||
bool _w = insert_lock(op, _b); \
|
||||
rw_lock(_w, _b, _b->level); \
|
||||
if (_b == (c)->root && \
|
||||
_w == insert_lock(op, _b)) \
|
||||
_r = bch_btree_ ## fn(_b, op, ##__VA_ARGS__); \
|
||||
rw_unlock(_w, _b); \
|
||||
bch_cannibalize_unlock(c, &(op)->cl); \
|
||||
} while (_r == -EINTR); \
|
||||
\
|
||||
_r; \
|
||||
})
|
||||
|
||||
static inline bool should_split(struct btree *b)
|
||||
{
|
||||
struct bset *i = write_block(b);
|
||||
return b->written >= btree_blocks(b) ||
|
||||
(i->seq == b->sets[0].data->seq &&
|
||||
b->written + __set_blocks(i, i->keys + 15, b->c)
|
||||
> btree_blocks(b));
|
||||
}
|
||||
|
||||
void bch_btree_node_read(struct btree *);
|
||||
void bch_btree_node_write(struct btree *, struct closure *);
|
||||
|
||||
void bch_cannibalize_unlock(struct cache_set *, struct closure *);
|
||||
void bch_btree_set_root(struct btree *);
|
||||
struct btree *bch_btree_node_alloc(struct cache_set *, int, struct closure *);
|
||||
struct btree *bch_btree_node_get(struct cache_set *, struct bkey *,
|
||||
int, struct btree_op *);
|
||||
struct btree *bch_btree_node_alloc(struct cache_set *, int, bool);
|
||||
struct btree *bch_btree_node_get(struct cache_set *, struct bkey *, int, bool);
|
||||
|
||||
bool bch_btree_insert_check_key(struct btree *, struct btree_op *,
|
||||
struct bio *);
|
||||
int bch_btree_insert(struct btree_op *, struct cache_set *);
|
||||
int bch_btree_insert_check_key(struct btree *, struct btree_op *,
|
||||
struct bkey *);
|
||||
int bch_btree_insert(struct cache_set *, struct keylist *,
|
||||
atomic_t *, struct bkey *);
|
||||
|
||||
int bch_btree_search_recurse(struct btree *, struct btree_op *);
|
||||
|
||||
void bch_queue_gc(struct cache_set *);
|
||||
int bch_gc_thread_start(struct cache_set *);
|
||||
size_t bch_btree_gc_finish(struct cache_set *);
|
||||
void bch_moving_gc(struct closure *);
|
||||
int bch_btree_check(struct cache_set *, struct btree_op *);
|
||||
void bch_moving_gc(struct cache_set *);
|
||||
int bch_btree_check(struct cache_set *);
|
||||
uint8_t __bch_btree_mark_key(struct cache_set *, int, struct bkey *);
|
||||
|
||||
static inline void wake_up_gc(struct cache_set *c)
|
||||
{
|
||||
if (c->gc_thread)
|
||||
wake_up_process(c->gc_thread);
|
||||
}
|
||||
|
||||
#define MAP_DONE 0
|
||||
#define MAP_CONTINUE 1
|
||||
|
||||
#define MAP_ALL_NODES 0
|
||||
#define MAP_LEAF_NODES 1
|
||||
|
||||
#define MAP_END_KEY 1
|
||||
|
||||
typedef int (btree_map_nodes_fn)(struct btree_op *, struct btree *);
|
||||
int __bch_btree_map_nodes(struct btree_op *, struct cache_set *,
|
||||
struct bkey *, btree_map_nodes_fn *, int);
|
||||
|
||||
static inline int bch_btree_map_nodes(struct btree_op *op, struct cache_set *c,
|
||||
struct bkey *from, btree_map_nodes_fn *fn)
|
||||
{
|
||||
return __bch_btree_map_nodes(op, c, from, fn, MAP_ALL_NODES);
|
||||
}
|
||||
|
||||
static inline int bch_btree_map_leaf_nodes(struct btree_op *op,
|
||||
struct cache_set *c,
|
||||
struct bkey *from,
|
||||
btree_map_nodes_fn *fn)
|
||||
{
|
||||
return __bch_btree_map_nodes(op, c, from, fn, MAP_LEAF_NODES);
|
||||
}
|
||||
|
||||
typedef int (btree_map_keys_fn)(struct btree_op *, struct btree *,
|
||||
struct bkey *);
|
||||
int bch_btree_map_keys(struct btree_op *, struct cache_set *,
|
||||
struct bkey *, btree_map_keys_fn *, int);
|
||||
|
||||
typedef bool (keybuf_pred_fn)(struct keybuf *, struct bkey *);
|
||||
|
||||
void bch_keybuf_init(struct keybuf *);
|
||||
void bch_refill_keybuf(struct cache_set *, struct keybuf *, struct bkey *,
|
||||
keybuf_pred_fn *);
|
||||
void bch_refill_keybuf(struct cache_set *, struct keybuf *,
|
||||
struct bkey *, keybuf_pred_fn *);
|
||||
bool bch_keybuf_check_overlapping(struct keybuf *, struct bkey *,
|
||||
struct bkey *);
|
||||
void bch_keybuf_del(struct keybuf *, struct keybuf_key *);
|
||||
|
@ -11,17 +11,6 @@
|
||||
|
||||
#include "closure.h"
|
||||
|
||||
void closure_queue(struct closure *cl)
|
||||
{
|
||||
struct workqueue_struct *wq = cl->wq;
|
||||
if (wq) {
|
||||
INIT_WORK(&cl->work, cl->work.func);
|
||||
BUG_ON(!queue_work(wq, &cl->work));
|
||||
} else
|
||||
cl->fn(cl);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_queue);
|
||||
|
||||
#define CL_FIELD(type, field) \
|
||||
case TYPE_ ## type: \
|
||||
return &container_of(cl, struct type, cl)->field
|
||||
@ -30,17 +19,6 @@ static struct closure_waitlist *closure_waitlist(struct closure *cl)
|
||||
{
|
||||
switch (cl->type) {
|
||||
CL_FIELD(closure_with_waitlist, wait);
|
||||
CL_FIELD(closure_with_waitlist_and_timer, wait);
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static struct timer_list *closure_timer(struct closure *cl)
|
||||
{
|
||||
switch (cl->type) {
|
||||
CL_FIELD(closure_with_timer, timer);
|
||||
CL_FIELD(closure_with_waitlist_and_timer, timer);
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
@ -51,7 +29,7 @@ static inline void closure_put_after_sub(struct closure *cl, int flags)
|
||||
int r = flags & CLOSURE_REMAINING_MASK;
|
||||
|
||||
BUG_ON(flags & CLOSURE_GUARD_MASK);
|
||||
BUG_ON(!r && (flags & ~(CLOSURE_DESTRUCTOR|CLOSURE_BLOCKING)));
|
||||
BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR));
|
||||
|
||||
/* Must deliver precisely one wakeup */
|
||||
if (r == 1 && (flags & CLOSURE_SLEEPING))
|
||||
@ -59,7 +37,6 @@ static inline void closure_put_after_sub(struct closure *cl, int flags)
|
||||
|
||||
if (!r) {
|
||||
if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {
|
||||
/* CLOSURE_BLOCKING might be set - clear it */
|
||||
atomic_set(&cl->remaining,
|
||||
CLOSURE_REMAINING_INITIALIZER);
|
||||
closure_queue(cl);
|
||||
@ -90,13 +67,13 @@ void closure_sub(struct closure *cl, int v)
|
||||
{
|
||||
closure_put_after_sub(cl, atomic_sub_return(v, &cl->remaining));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_sub);
|
||||
EXPORT_SYMBOL(closure_sub);
|
||||
|
||||
void closure_put(struct closure *cl)
|
||||
{
|
||||
closure_put_after_sub(cl, atomic_dec_return(&cl->remaining));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_put);
|
||||
EXPORT_SYMBOL(closure_put);
|
||||
|
||||
static void set_waiting(struct closure *cl, unsigned long f)
|
||||
{
|
||||
@ -133,7 +110,7 @@ void __closure_wake_up(struct closure_waitlist *wait_list)
|
||||
closure_sub(cl, CLOSURE_WAITING + 1);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__closure_wake_up);
|
||||
EXPORT_SYMBOL(__closure_wake_up);
|
||||
|
||||
bool closure_wait(struct closure_waitlist *list, struct closure *cl)
|
||||
{
|
||||
@ -146,7 +123,7 @@ bool closure_wait(struct closure_waitlist *list, struct closure *cl)
|
||||
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_wait);
|
||||
EXPORT_SYMBOL(closure_wait);
|
||||
|
||||
/**
|
||||
* closure_sync() - sleep until a closure a closure has nothing left to wait on
|
||||
@ -169,7 +146,7 @@ void closure_sync(struct closure *cl)
|
||||
|
||||
__closure_end_sleep(cl);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_sync);
|
||||
EXPORT_SYMBOL(closure_sync);
|
||||
|
||||
/**
|
||||
* closure_trylock() - try to acquire the closure, without waiting
|
||||
@ -183,17 +160,17 @@ bool closure_trylock(struct closure *cl, struct closure *parent)
|
||||
CLOSURE_REMAINING_INITIALIZER) != -1)
|
||||
return false;
|
||||
|
||||
closure_set_ret_ip(cl);
|
||||
|
||||
smp_mb();
|
||||
|
||||
cl->parent = parent;
|
||||
if (parent)
|
||||
closure_get(parent);
|
||||
|
||||
closure_set_ret_ip(cl);
|
||||
closure_debug_create(cl);
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_trylock);
|
||||
EXPORT_SYMBOL(closure_trylock);
|
||||
|
||||
void __closure_lock(struct closure *cl, struct closure *parent,
|
||||
struct closure_waitlist *wait_list)
|
||||
@ -205,57 +182,11 @@ void __closure_lock(struct closure *cl, struct closure *parent,
|
||||
if (closure_trylock(cl, parent))
|
||||
return;
|
||||
|
||||
closure_wait_event_sync(wait_list, &wait,
|
||||
atomic_read(&cl->remaining) == -1);
|
||||
closure_wait_event(wait_list, &wait,
|
||||
atomic_read(&cl->remaining) == -1);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__closure_lock);
|
||||
|
||||
static void closure_delay_timer_fn(unsigned long data)
|
||||
{
|
||||
struct closure *cl = (struct closure *) data;
|
||||
closure_sub(cl, CLOSURE_TIMER + 1);
|
||||
}
|
||||
|
||||
void do_closure_timer_init(struct closure *cl)
|
||||
{
|
||||
struct timer_list *timer = closure_timer(cl);
|
||||
|
||||
init_timer(timer);
|
||||
timer->data = (unsigned long) cl;
|
||||
timer->function = closure_delay_timer_fn;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(do_closure_timer_init);
|
||||
|
||||
bool __closure_delay(struct closure *cl, unsigned long delay,
|
||||
struct timer_list *timer)
|
||||
{
|
||||
if (atomic_read(&cl->remaining) & CLOSURE_TIMER)
|
||||
return false;
|
||||
|
||||
BUG_ON(timer_pending(timer));
|
||||
|
||||
timer->expires = jiffies + delay;
|
||||
|
||||
atomic_add(CLOSURE_TIMER + 1, &cl->remaining);
|
||||
add_timer(timer);
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__closure_delay);
|
||||
|
||||
void __closure_flush(struct closure *cl, struct timer_list *timer)
|
||||
{
|
||||
if (del_timer(timer))
|
||||
closure_sub(cl, CLOSURE_TIMER + 1);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__closure_flush);
|
||||
|
||||
void __closure_flush_sync(struct closure *cl, struct timer_list *timer)
|
||||
{
|
||||
if (del_timer_sync(timer))
|
||||
closure_sub(cl, CLOSURE_TIMER + 1);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__closure_flush_sync);
|
||||
EXPORT_SYMBOL(__closure_lock);
|
||||
|
||||
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
|
||||
|
||||
@ -273,7 +204,7 @@ void closure_debug_create(struct closure *cl)
|
||||
list_add(&cl->all, &closure_list);
|
||||
spin_unlock_irqrestore(&closure_list_lock, flags);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_debug_create);
|
||||
EXPORT_SYMBOL(closure_debug_create);
|
||||
|
||||
void closure_debug_destroy(struct closure *cl)
|
||||
{
|
||||
@ -286,7 +217,7 @@ void closure_debug_destroy(struct closure *cl)
|
||||
list_del(&cl->all);
|
||||
spin_unlock_irqrestore(&closure_list_lock, flags);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(closure_debug_destroy);
|
||||
EXPORT_SYMBOL(closure_debug_destroy);
|
||||
|
||||
static struct dentry *debug;
|
||||
|
||||
@ -304,14 +235,12 @@ static int debug_seq_show(struct seq_file *f, void *data)
|
||||
cl, (void *) cl->ip, cl->fn, cl->parent,
|
||||
r & CLOSURE_REMAINING_MASK);
|
||||
|
||||
seq_printf(f, "%s%s%s%s%s%s\n",
|
||||
seq_printf(f, "%s%s%s%s\n",
|
||||
test_bit(WORK_STRUCT_PENDING,
|
||||
work_data_bits(&cl->work)) ? "Q" : "",
|
||||
r & CLOSURE_RUNNING ? "R" : "",
|
||||
r & CLOSURE_BLOCKING ? "B" : "",
|
||||
r & CLOSURE_STACK ? "S" : "",
|
||||
r & CLOSURE_SLEEPING ? "Sl" : "",
|
||||
r & CLOSURE_TIMER ? "T" : "");
|
||||
r & CLOSURE_SLEEPING ? "Sl" : "");
|
||||
|
||||
if (r & CLOSURE_WAITING)
|
||||
seq_printf(f, " W %pF\n",
|
||||
|
@ -155,21 +155,6 @@
|
||||
* delayed_work embeds a work item and a timer_list. The important thing is, use
|
||||
* it exactly like you would a regular closure and closure_put() will magically
|
||||
* handle everything for you.
|
||||
*
|
||||
* We've got closures that embed timers, too. They're called, appropriately
|
||||
* enough:
|
||||
* struct closure_with_timer;
|
||||
*
|
||||
* This gives you access to closure_delay(). It takes a refcount for a specified
|
||||
* number of jiffies - you could then call closure_sync() (for a slightly
|
||||
* convoluted version of msleep()) or continue_at() - which gives you the same
|
||||
* effect as using a delayed work item, except you can reuse the work_struct
|
||||
* already embedded in struct closure.
|
||||
*
|
||||
* Lastly, there's struct closure_with_waitlist_and_timer. It does what you
|
||||
* probably expect, if you happen to need the features of both. (You don't
|
||||
* really want to know how all this is implemented, but if I've done my job
|
||||
* right you shouldn't have to care).
|
||||
*/
|
||||
|
||||
struct closure;
|
||||
@ -182,16 +167,11 @@ struct closure_waitlist {
|
||||
enum closure_type {
|
||||
TYPE_closure = 0,
|
||||
TYPE_closure_with_waitlist = 1,
|
||||
TYPE_closure_with_timer = 2,
|
||||
TYPE_closure_with_waitlist_and_timer = 3,
|
||||
MAX_CLOSURE_TYPE = 3,
|
||||
MAX_CLOSURE_TYPE = 1,
|
||||
};
|
||||
|
||||
enum closure_state {
|
||||
/*
|
||||
* CLOSURE_BLOCKING: Causes closure_wait_event() to block, instead of
|
||||
* waiting asynchronously
|
||||
*
|
||||
* CLOSURE_WAITING: Set iff the closure is on a waitlist. Must be set by
|
||||
* the thread that owns the closure, and cleared by the thread that's
|
||||
* waking up the closure.
|
||||
@ -200,10 +180,6 @@ enum closure_state {
|
||||
* - indicates that cl->task is valid and closure_put() may wake it up.
|
||||
* Only set or cleared by the thread that owns the closure.
|
||||
*
|
||||
* CLOSURE_TIMER: Analagous to CLOSURE_WAITING, indicates that a closure
|
||||
* has an outstanding timer. Must be set by the thread that owns the
|
||||
* closure, and cleared by the timer function when the timer goes off.
|
||||
*
|
||||
* The rest are for debugging and don't affect behaviour:
|
||||
*
|
||||
* CLOSURE_RUNNING: Set when a closure is running (i.e. by
|
||||
@ -218,19 +194,17 @@ enum closure_state {
|
||||
* closure with this flag set
|
||||
*/
|
||||
|
||||
CLOSURE_BITS_START = (1 << 19),
|
||||
CLOSURE_DESTRUCTOR = (1 << 19),
|
||||
CLOSURE_BLOCKING = (1 << 21),
|
||||
CLOSURE_WAITING = (1 << 23),
|
||||
CLOSURE_SLEEPING = (1 << 25),
|
||||
CLOSURE_TIMER = (1 << 27),
|
||||
CLOSURE_BITS_START = (1 << 23),
|
||||
CLOSURE_DESTRUCTOR = (1 << 23),
|
||||
CLOSURE_WAITING = (1 << 25),
|
||||
CLOSURE_SLEEPING = (1 << 27),
|
||||
CLOSURE_RUNNING = (1 << 29),
|
||||
CLOSURE_STACK = (1 << 31),
|
||||
};
|
||||
|
||||
#define CLOSURE_GUARD_MASK \
|
||||
((CLOSURE_DESTRUCTOR|CLOSURE_BLOCKING|CLOSURE_WAITING| \
|
||||
CLOSURE_SLEEPING|CLOSURE_TIMER|CLOSURE_RUNNING|CLOSURE_STACK) << 1)
|
||||
((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_SLEEPING| \
|
||||
CLOSURE_RUNNING|CLOSURE_STACK) << 1)
|
||||
|
||||
#define CLOSURE_REMAINING_MASK (CLOSURE_BITS_START - 1)
|
||||
#define CLOSURE_REMAINING_INITIALIZER (1|CLOSURE_RUNNING)
|
||||
@ -268,17 +242,6 @@ struct closure_with_waitlist {
|
||||
struct closure_waitlist wait;
|
||||
};
|
||||
|
||||
struct closure_with_timer {
|
||||
struct closure cl;
|
||||
struct timer_list timer;
|
||||
};
|
||||
|
||||
struct closure_with_waitlist_and_timer {
|
||||
struct closure cl;
|
||||
struct closure_waitlist wait;
|
||||
struct timer_list timer;
|
||||
};
|
||||
|
||||
extern unsigned invalid_closure_type(void);
|
||||
|
||||
#define __CLOSURE_TYPE(cl, _t) \
|
||||
@ -289,14 +252,11 @@ extern unsigned invalid_closure_type(void);
|
||||
( \
|
||||
__CLOSURE_TYPE(cl, closure) \
|
||||
__CLOSURE_TYPE(cl, closure_with_waitlist) \
|
||||
__CLOSURE_TYPE(cl, closure_with_timer) \
|
||||
__CLOSURE_TYPE(cl, closure_with_waitlist_and_timer) \
|
||||
invalid_closure_type() \
|
||||
)
|
||||
|
||||
void closure_sub(struct closure *cl, int v);
|
||||
void closure_put(struct closure *cl);
|
||||
void closure_queue(struct closure *cl);
|
||||
void __closure_wake_up(struct closure_waitlist *list);
|
||||
bool closure_wait(struct closure_waitlist *list, struct closure *cl);
|
||||
void closure_sync(struct closure *cl);
|
||||
@ -305,12 +265,6 @@ bool closure_trylock(struct closure *cl, struct closure *parent);
|
||||
void __closure_lock(struct closure *cl, struct closure *parent,
|
||||
struct closure_waitlist *wait_list);
|
||||
|
||||
void do_closure_timer_init(struct closure *cl);
|
||||
bool __closure_delay(struct closure *cl, unsigned long delay,
|
||||
struct timer_list *timer);
|
||||
void __closure_flush(struct closure *cl, struct timer_list *timer);
|
||||
void __closure_flush_sync(struct closure *cl, struct timer_list *timer);
|
||||
|
||||
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
|
||||
|
||||
void closure_debug_init(void);
|
||||
@ -354,11 +308,6 @@ static inline void closure_set_stopped(struct closure *cl)
|
||||
atomic_sub(CLOSURE_RUNNING, &cl->remaining);
|
||||
}
|
||||
|
||||
static inline bool closure_is_stopped(struct closure *cl)
|
||||
{
|
||||
return !(atomic_read(&cl->remaining) & CLOSURE_RUNNING);
|
||||
}
|
||||
|
||||
static inline bool closure_is_unlocked(struct closure *cl)
|
||||
{
|
||||
return atomic_read(&cl->remaining) == -1;
|
||||
@ -367,14 +316,6 @@ static inline bool closure_is_unlocked(struct closure *cl)
|
||||
static inline void do_closure_init(struct closure *cl, struct closure *parent,
|
||||
bool running)
|
||||
{
|
||||
switch (cl->type) {
|
||||
case TYPE_closure_with_timer:
|
||||
case TYPE_closure_with_waitlist_and_timer:
|
||||
do_closure_timer_init(cl);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
cl->parent = parent;
|
||||
if (parent)
|
||||
closure_get(parent);
|
||||
@ -429,8 +370,7 @@ do { \
|
||||
static inline void closure_init_stack(struct closure *cl)
|
||||
{
|
||||
memset(cl, 0, sizeof(struct closure));
|
||||
atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER|
|
||||
CLOSURE_BLOCKING|CLOSURE_STACK);
|
||||
atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER|CLOSURE_STACK);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -461,24 +401,6 @@ do { \
|
||||
#define closure_lock(cl, parent) \
|
||||
__closure_lock(__to_internal_closure(cl), parent, &(cl)->wait)
|
||||
|
||||
/**
|
||||
* closure_delay() - delay some number of jiffies
|
||||
* @cl: the closure that will sleep
|
||||
* @delay: the delay in jiffies
|
||||
*
|
||||
* Takes a refcount on @cl which will be released after @delay jiffies; this may
|
||||
* be used to have a function run after a delay with continue_at(), or
|
||||
* closure_sync() may be used for a convoluted version of msleep().
|
||||
*/
|
||||
#define closure_delay(cl, delay) \
|
||||
__closure_delay(__to_internal_closure(cl), delay, &(cl)->timer)
|
||||
|
||||
#define closure_flush(cl) \
|
||||
__closure_flush(__to_internal_closure(cl), &(cl)->timer)
|
||||
|
||||
#define closure_flush_sync(cl) \
|
||||
__closure_flush_sync(__to_internal_closure(cl), &(cl)->timer)
|
||||
|
||||
static inline void __closure_end_sleep(struct closure *cl)
|
||||
{
|
||||
__set_current_state(TASK_RUNNING);
|
||||
@ -497,40 +419,6 @@ static inline void __closure_start_sleep(struct closure *cl)
|
||||
atomic_add(CLOSURE_SLEEPING, &cl->remaining);
|
||||
}
|
||||
|
||||
/**
|
||||
* closure_blocking() - returns true if the closure is in blocking mode.
|
||||
*
|
||||
* If a closure is in blocking mode, closure_wait_event() will sleep until the
|
||||
* condition is true instead of waiting asynchronously.
|
||||
*/
|
||||
static inline bool closure_blocking(struct closure *cl)
|
||||
{
|
||||
return atomic_read(&cl->remaining) & CLOSURE_BLOCKING;
|
||||
}
|
||||
|
||||
/**
|
||||
* set_closure_blocking() - put a closure in blocking mode.
|
||||
*
|
||||
* If a closure is in blocking mode, closure_wait_event() will sleep until the
|
||||
* condition is true instead of waiting asynchronously.
|
||||
*
|
||||
* Not thread safe - can only be called by the thread running the closure.
|
||||
*/
|
||||
static inline void set_closure_blocking(struct closure *cl)
|
||||
{
|
||||
if (!closure_blocking(cl))
|
||||
atomic_add(CLOSURE_BLOCKING, &cl->remaining);
|
||||
}
|
||||
|
||||
/*
|
||||
* Not thread safe - can only be called by the thread running the closure.
|
||||
*/
|
||||
static inline void clear_closure_blocking(struct closure *cl)
|
||||
{
|
||||
if (closure_blocking(cl))
|
||||
atomic_sub(CLOSURE_BLOCKING, &cl->remaining);
|
||||
}
|
||||
|
||||
/**
|
||||
* closure_wake_up() - wake up all closures on a wait list.
|
||||
*/
|
||||
@ -561,63 +449,36 @@ static inline void closure_wake_up(struct closure_waitlist *list)
|
||||
* refcount on our closure. If this was a stack allocated closure, that would be
|
||||
* bad.
|
||||
*/
|
||||
#define __closure_wait_event(list, cl, condition, _block) \
|
||||
#define closure_wait_event(list, cl, condition) \
|
||||
({ \
|
||||
bool block = _block; \
|
||||
typeof(condition) ret; \
|
||||
\
|
||||
while (1) { \
|
||||
ret = (condition); \
|
||||
if (ret) { \
|
||||
__closure_wake_up(list); \
|
||||
if (block) \
|
||||
closure_sync(cl); \
|
||||
\
|
||||
closure_sync(cl); \
|
||||
break; \
|
||||
} \
|
||||
\
|
||||
if (block) \
|
||||
__closure_start_sleep(cl); \
|
||||
\
|
||||
if (!closure_wait(list, cl)) { \
|
||||
if (!block) \
|
||||
break; \
|
||||
__closure_start_sleep(cl); \
|
||||
\
|
||||
if (!closure_wait(list, cl)) \
|
||||
schedule(); \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
ret; \
|
||||
})
|
||||
|
||||
/**
|
||||
* closure_wait_event() - wait on a condition, synchronously or asynchronously.
|
||||
* @list: the wait list to wait on
|
||||
* @cl: the closure that is doing the waiting
|
||||
* @condition: a C expression for the event to wait for
|
||||
*
|
||||
* If the closure is in blocking mode, sleeps until the @condition evaluates to
|
||||
* true - exactly like wait_event().
|
||||
*
|
||||
* If the closure is not in blocking mode, waits asynchronously; if the
|
||||
* condition is currently false the @cl is put onto @list and returns. @list
|
||||
* owns a refcount on @cl; closure_sync() or continue_at() may be used later to
|
||||
* wait for another thread to wake up @list, which drops the refcount on @cl.
|
||||
*
|
||||
* Returns the value of @condition; @cl will be on @list iff @condition was
|
||||
* false.
|
||||
*
|
||||
* closure_wake_up(@list) must be called after changing any variable that could
|
||||
* cause @condition to become true.
|
||||
*/
|
||||
#define closure_wait_event(list, cl, condition) \
|
||||
__closure_wait_event(list, cl, condition, closure_blocking(cl))
|
||||
|
||||
#define closure_wait_event_async(list, cl, condition) \
|
||||
__closure_wait_event(list, cl, condition, false)
|
||||
|
||||
#define closure_wait_event_sync(list, cl, condition) \
|
||||
__closure_wait_event(list, cl, condition, true)
|
||||
static inline void closure_queue(struct closure *cl)
|
||||
{
|
||||
struct workqueue_struct *wq = cl->wq;
|
||||
if (wq) {
|
||||
INIT_WORK(&cl->work, cl->work.func);
|
||||
BUG_ON(!queue_work(wq, &cl->work));
|
||||
} else
|
||||
cl->fn(cl);
|
||||
}
|
||||
|
||||
static inline void set_closure_fn(struct closure *cl, closure_fn *fn,
|
||||
struct workqueue_struct *wq)
|
||||
@ -642,7 +503,7 @@ do { \
|
||||
#define continue_at_nobarrier(_cl, _fn, _wq) \
|
||||
do { \
|
||||
set_closure_fn(_cl, _fn, _wq); \
|
||||
closure_queue(cl); \
|
||||
closure_queue(_cl); \
|
||||
return; \
|
||||
} while (0)
|
||||
|
||||
|
@ -8,7 +8,6 @@
|
||||
#include "bcache.h"
|
||||
#include "btree.h"
|
||||
#include "debug.h"
|
||||
#include "request.h"
|
||||
|
||||
#include <linux/console.h>
|
||||
#include <linux/debugfs.h>
|
||||
@ -77,29 +76,17 @@ int bch_bkey_to_text(char *buf, size_t size, const struct bkey *k)
|
||||
return out - buf;
|
||||
}
|
||||
|
||||
int bch_btree_to_text(char *buf, size_t size, const struct btree *b)
|
||||
{
|
||||
return scnprintf(buf, size, "%zu level %i/%i",
|
||||
PTR_BUCKET_NR(b->c, &b->key, 0),
|
||||
b->level, b->c->root ? b->c->root->level : -1);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_BCACHE_DEBUG) || defined(CONFIG_BCACHE_EDEBUG)
|
||||
|
||||
static bool skipped_backwards(struct btree *b, struct bkey *k)
|
||||
{
|
||||
return bkey_cmp(k, (!b->level)
|
||||
? &START_KEY(bkey_next(k))
|
||||
: bkey_next(k)) > 0;
|
||||
}
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
|
||||
static void dump_bset(struct btree *b, struct bset *i)
|
||||
{
|
||||
struct bkey *k;
|
||||
struct bkey *k, *next;
|
||||
unsigned j;
|
||||
char buf[80];
|
||||
|
||||
for (k = i->start; k < end(i); k = bkey_next(k)) {
|
||||
for (k = i->start; k < end(i); k = next) {
|
||||
next = bkey_next(k);
|
||||
|
||||
bch_bkey_to_text(buf, sizeof(buf), k);
|
||||
printk(KERN_ERR "block %zu key %zi/%u: %s", index(i, b),
|
||||
(uint64_t *) k - i->d, i->keys, buf);
|
||||
@ -115,15 +102,21 @@ static void dump_bset(struct btree *b, struct bset *i)
|
||||
|
||||
printk(" %s\n", bch_ptr_status(b->c, k));
|
||||
|
||||
if (bkey_next(k) < end(i) &&
|
||||
skipped_backwards(b, k))
|
||||
if (next < end(i) &&
|
||||
bkey_cmp(k, !b->level ? &START_KEY(next) : next) > 0)
|
||||
printk(KERN_ERR "Key skipped backwards\n");
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
static void bch_dump_bucket(struct btree *b)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
console_lock();
|
||||
for (i = 0; i <= b->nsets; i++)
|
||||
dump_bset(b, b->sets[i].data);
|
||||
console_unlock();
|
||||
}
|
||||
|
||||
void bch_btree_verify(struct btree *b, struct bset *new)
|
||||
{
|
||||
@ -176,66 +169,44 @@ void bch_btree_verify(struct btree *b, struct bset *new)
|
||||
mutex_unlock(&b->c->verify_lock);
|
||||
}
|
||||
|
||||
static void data_verify_endio(struct bio *bio, int error)
|
||||
{
|
||||
struct closure *cl = bio->bi_private;
|
||||
closure_put(cl);
|
||||
}
|
||||
|
||||
void bch_data_verify(struct search *s)
|
||||
void bch_data_verify(struct cached_dev *dc, struct bio *bio)
|
||||
{
|
||||
char name[BDEVNAME_SIZE];
|
||||
struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
|
||||
struct closure *cl = &s->cl;
|
||||
struct bio *check;
|
||||
struct bio_vec *bv;
|
||||
int i;
|
||||
|
||||
if (!s->unaligned_bvec)
|
||||
bio_for_each_segment(bv, s->orig_bio, i)
|
||||
bv->bv_offset = 0, bv->bv_len = PAGE_SIZE;
|
||||
|
||||
check = bio_clone(s->orig_bio, GFP_NOIO);
|
||||
check = bio_clone(bio, GFP_NOIO);
|
||||
if (!check)
|
||||
return;
|
||||
|
||||
if (bio_alloc_pages(check, GFP_NOIO))
|
||||
goto out_put;
|
||||
|
||||
check->bi_rw = READ_SYNC;
|
||||
check->bi_private = cl;
|
||||
check->bi_end_io = data_verify_endio;
|
||||
submit_bio_wait(READ_SYNC, check);
|
||||
|
||||
closure_bio_submit(check, cl, &dc->disk);
|
||||
closure_sync(cl);
|
||||
bio_for_each_segment(bv, bio, i) {
|
||||
void *p1 = kmap_atomic(bv->bv_page);
|
||||
void *p2 = page_address(check->bi_io_vec[i].bv_page);
|
||||
|
||||
bio_for_each_segment(bv, s->orig_bio, i) {
|
||||
void *p1 = kmap(bv->bv_page);
|
||||
void *p2 = kmap(check->bi_io_vec[i].bv_page);
|
||||
cache_set_err_on(memcmp(p1 + bv->bv_offset,
|
||||
p2 + bv->bv_offset,
|
||||
bv->bv_len),
|
||||
dc->disk.c,
|
||||
"verify failed at dev %s sector %llu",
|
||||
bdevname(dc->bdev, name),
|
||||
(uint64_t) bio->bi_sector);
|
||||
|
||||
if (memcmp(p1 + bv->bv_offset,
|
||||
p2 + bv->bv_offset,
|
||||
bv->bv_len))
|
||||
printk(KERN_ERR
|
||||
"bcache (%s): verify failed at sector %llu\n",
|
||||
bdevname(dc->bdev, name),
|
||||
(uint64_t) s->orig_bio->bi_sector);
|
||||
|
||||
kunmap(bv->bv_page);
|
||||
kunmap(check->bi_io_vec[i].bv_page);
|
||||
kunmap_atomic(p1);
|
||||
}
|
||||
|
||||
__bio_for_each_segment(bv, check, i, 0)
|
||||
bio_for_each_segment_all(bv, check, i)
|
||||
__free_page(bv->bv_page);
|
||||
out_put:
|
||||
bio_put(check);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
|
||||
unsigned bch_count_data(struct btree *b)
|
||||
int __bch_count_data(struct btree *b)
|
||||
{
|
||||
unsigned ret = 0;
|
||||
struct btree_iter iter;
|
||||
@ -247,72 +218,60 @@ unsigned bch_count_data(struct btree *b)
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void vdump_bucket_and_panic(struct btree *b, const char *fmt,
|
||||
va_list args)
|
||||
{
|
||||
unsigned i;
|
||||
char buf[80];
|
||||
|
||||
console_lock();
|
||||
|
||||
for (i = 0; i <= b->nsets; i++)
|
||||
dump_bset(b, b->sets[i].data);
|
||||
|
||||
vprintk(fmt, args);
|
||||
|
||||
console_unlock();
|
||||
|
||||
bch_btree_to_text(buf, sizeof(buf), b);
|
||||
panic("at %s\n", buf);
|
||||
}
|
||||
|
||||
void bch_check_key_order_msg(struct btree *b, struct bset *i,
|
||||
const char *fmt, ...)
|
||||
{
|
||||
struct bkey *k;
|
||||
|
||||
if (!i->keys)
|
||||
return;
|
||||
|
||||
for (k = i->start; bkey_next(k) < end(i); k = bkey_next(k))
|
||||
if (skipped_backwards(b, k)) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
|
||||
vdump_bucket_and_panic(b, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
}
|
||||
|
||||
void bch_check_keys(struct btree *b, const char *fmt, ...)
|
||||
void __bch_check_keys(struct btree *b, const char *fmt, ...)
|
||||
{
|
||||
va_list args;
|
||||
struct bkey *k, *p = NULL;
|
||||
struct btree_iter iter;
|
||||
|
||||
if (b->level)
|
||||
return;
|
||||
const char *err;
|
||||
|
||||
for_each_key(b, k, &iter) {
|
||||
if (p && bkey_cmp(&START_KEY(p), &START_KEY(k)) > 0) {
|
||||
printk(KERN_ERR "Keys out of order:\n");
|
||||
goto bug;
|
||||
}
|
||||
if (!b->level) {
|
||||
err = "Keys out of order";
|
||||
if (p && bkey_cmp(&START_KEY(p), &START_KEY(k)) > 0)
|
||||
goto bug;
|
||||
|
||||
if (bch_ptr_invalid(b, k))
|
||||
continue;
|
||||
if (bch_ptr_invalid(b, k))
|
||||
continue;
|
||||
|
||||
if (p && bkey_cmp(p, &START_KEY(k)) > 0) {
|
||||
printk(KERN_ERR "Overlapping keys:\n");
|
||||
goto bug;
|
||||
err = "Overlapping keys";
|
||||
if (p && bkey_cmp(p, &START_KEY(k)) > 0)
|
||||
goto bug;
|
||||
} else {
|
||||
if (bch_ptr_bad(b, k))
|
||||
continue;
|
||||
|
||||
err = "Duplicate keys";
|
||||
if (p && !bkey_cmp(p, k))
|
||||
goto bug;
|
||||
}
|
||||
p = k;
|
||||
}
|
||||
|
||||
err = "Key larger than btree node key";
|
||||
if (p && bkey_cmp(p, &b->key) > 0)
|
||||
goto bug;
|
||||
|
||||
return;
|
||||
bug:
|
||||
bch_dump_bucket(b);
|
||||
|
||||
va_start(args, fmt);
|
||||
vdump_bucket_and_panic(b, fmt, args);
|
||||
vprintk(fmt, args);
|
||||
va_end(args);
|
||||
|
||||
panic("bcache error: %s:\n", err);
|
||||
}
|
||||
|
||||
void bch_btree_iter_next_check(struct btree_iter *iter)
|
||||
{
|
||||
struct bkey *k = iter->data->k, *next = bkey_next(k);
|
||||
|
||||
if (next < iter->data->end &&
|
||||
bkey_cmp(k, iter->b->level ? next : &START_KEY(next)) > 0) {
|
||||
bch_dump_bucket(iter->b);
|
||||
panic("Key skipped backwards\n");
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -4,40 +4,44 @@
|
||||
/* Btree/bkey debug printing */
|
||||
|
||||
int bch_bkey_to_text(char *buf, size_t size, const struct bkey *k);
|
||||
int bch_btree_to_text(char *buf, size_t size, const struct btree *b);
|
||||
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
|
||||
unsigned bch_count_data(struct btree *);
|
||||
void bch_check_key_order_msg(struct btree *, struct bset *, const char *, ...);
|
||||
void bch_check_keys(struct btree *, const char *, ...);
|
||||
|
||||
#define bch_check_key_order(b, i) \
|
||||
bch_check_key_order_msg(b, i, "keys out of order")
|
||||
#define EBUG_ON(cond) BUG_ON(cond)
|
||||
|
||||
#else /* EDEBUG */
|
||||
|
||||
#define bch_count_data(b) 0
|
||||
#define bch_check_key_order(b, i) do {} while (0)
|
||||
#define bch_check_key_order_msg(b, i, ...) do {} while (0)
|
||||
#define bch_check_keys(b, ...) do {} while (0)
|
||||
#define EBUG_ON(cond) do {} while (0)
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
|
||||
void bch_btree_verify(struct btree *, struct bset *);
|
||||
void bch_data_verify(struct search *);
|
||||
void bch_data_verify(struct cached_dev *, struct bio *);
|
||||
int __bch_count_data(struct btree *);
|
||||
void __bch_check_keys(struct btree *, const char *, ...);
|
||||
void bch_btree_iter_next_check(struct btree_iter *);
|
||||
|
||||
#define EBUG_ON(cond) BUG_ON(cond)
|
||||
#define expensive_debug_checks(c) ((c)->expensive_debug_checks)
|
||||
#define key_merging_disabled(c) ((c)->key_merging_disabled)
|
||||
#define bypass_torture_test(d) ((d)->bypass_torture_test)
|
||||
|
||||
#else /* DEBUG */
|
||||
|
||||
static inline void bch_btree_verify(struct btree *b, struct bset *i) {}
|
||||
static inline void bch_data_verify(struct search *s) {};
|
||||
static inline void bch_data_verify(struct cached_dev *dc, struct bio *bio) {}
|
||||
static inline int __bch_count_data(struct btree *b) { return -1; }
|
||||
static inline void __bch_check_keys(struct btree *b, const char *fmt, ...) {}
|
||||
static inline void bch_btree_iter_next_check(struct btree_iter *iter) {}
|
||||
|
||||
#define EBUG_ON(cond) do { if (cond); } while (0)
|
||||
#define expensive_debug_checks(c) 0
|
||||
#define key_merging_disabled(c) 0
|
||||
#define bypass_torture_test(d) 0
|
||||
|
||||
#endif
|
||||
|
||||
#define bch_count_data(b) \
|
||||
(expensive_debug_checks((b)->c) ? __bch_count_data(b) : -1)
|
||||
|
||||
#define bch_check_keys(b, ...) \
|
||||
do { \
|
||||
if (expensive_debug_checks((b)->c)) \
|
||||
__bch_check_keys(b, __VA_ARGS__); \
|
||||
} while (0)
|
||||
|
||||
#ifdef CONFIG_DEBUG_FS
|
||||
void bch_debug_init_cache_set(struct cache_set *);
|
||||
#else
|
||||
|
@ -7,7 +7,6 @@
|
||||
#include "bcache.h"
|
||||
#include "btree.h"
|
||||
#include "debug.h"
|
||||
#include "request.h"
|
||||
|
||||
#include <trace/events/bcache.h>
|
||||
|
||||
@ -31,17 +30,20 @@ static void journal_read_endio(struct bio *bio, int error)
|
||||
}
|
||||
|
||||
static int journal_read_bucket(struct cache *ca, struct list_head *list,
|
||||
struct btree_op *op, unsigned bucket_index)
|
||||
unsigned bucket_index)
|
||||
{
|
||||
struct journal_device *ja = &ca->journal;
|
||||
struct bio *bio = &ja->bio;
|
||||
|
||||
struct journal_replay *i;
|
||||
struct jset *j, *data = ca->set->journal.w[0].data;
|
||||
struct closure cl;
|
||||
unsigned len, left, offset = 0;
|
||||
int ret = 0;
|
||||
sector_t bucket = bucket_to_sector(ca->set, ca->sb.d[bucket_index]);
|
||||
|
||||
closure_init_stack(&cl);
|
||||
|
||||
pr_debug("reading %llu", (uint64_t) bucket);
|
||||
|
||||
while (offset < ca->sb.bucket_size) {
|
||||
@ -55,11 +57,11 @@ reread: left = ca->sb.bucket_size - offset;
|
||||
bio->bi_size = len << 9;
|
||||
|
||||
bio->bi_end_io = journal_read_endio;
|
||||
bio->bi_private = &op->cl;
|
||||
bio->bi_private = &cl;
|
||||
bch_bio_map(bio, data);
|
||||
|
||||
closure_bio_submit(bio, &op->cl, ca);
|
||||
closure_sync(&op->cl);
|
||||
closure_bio_submit(bio, &cl, ca);
|
||||
closure_sync(&cl);
|
||||
|
||||
/* This function could be simpler now since we no longer write
|
||||
* journal entries that overlap bucket boundaries; this means
|
||||
@ -72,7 +74,7 @@ reread: left = ca->sb.bucket_size - offset;
|
||||
struct list_head *where;
|
||||
size_t blocks, bytes = set_bytes(j);
|
||||
|
||||
if (j->magic != jset_magic(ca->set))
|
||||
if (j->magic != jset_magic(&ca->sb))
|
||||
return ret;
|
||||
|
||||
if (bytes > left << 9)
|
||||
@ -129,12 +131,11 @@ next_set:
|
||||
return ret;
|
||||
}
|
||||
|
||||
int bch_journal_read(struct cache_set *c, struct list_head *list,
|
||||
struct btree_op *op)
|
||||
int bch_journal_read(struct cache_set *c, struct list_head *list)
|
||||
{
|
||||
#define read_bucket(b) \
|
||||
({ \
|
||||
int ret = journal_read_bucket(ca, list, op, b); \
|
||||
int ret = journal_read_bucket(ca, list, b); \
|
||||
__set_bit(b, bitmap); \
|
||||
if (ret < 0) \
|
||||
return ret; \
|
||||
@ -292,8 +293,7 @@ void bch_journal_mark(struct cache_set *c, struct list_head *list)
|
||||
}
|
||||
}
|
||||
|
||||
int bch_journal_replay(struct cache_set *s, struct list_head *list,
|
||||
struct btree_op *op)
|
||||
int bch_journal_replay(struct cache_set *s, struct list_head *list)
|
||||
{
|
||||
int ret = 0, keys = 0, entries = 0;
|
||||
struct bkey *k;
|
||||
@ -301,31 +301,30 @@ int bch_journal_replay(struct cache_set *s, struct list_head *list,
|
||||
list_entry(list->prev, struct journal_replay, list);
|
||||
|
||||
uint64_t start = i->j.last_seq, end = i->j.seq, n = start;
|
||||
struct keylist keylist;
|
||||
|
||||
bch_keylist_init(&keylist);
|
||||
|
||||
list_for_each_entry(i, list, list) {
|
||||
BUG_ON(i->pin && atomic_read(i->pin) != 1);
|
||||
|
||||
if (n != i->j.seq)
|
||||
pr_err(
|
||||
"journal entries %llu-%llu missing! (replaying %llu-%llu)\n",
|
||||
n, i->j.seq - 1, start, end);
|
||||
cache_set_err_on(n != i->j.seq, s,
|
||||
"bcache: journal entries %llu-%llu missing! (replaying %llu-%llu)",
|
||||
n, i->j.seq - 1, start, end);
|
||||
|
||||
for (k = i->j.start;
|
||||
k < end(&i->j);
|
||||
k = bkey_next(k)) {
|
||||
trace_bcache_journal_replay_key(k);
|
||||
|
||||
bkey_copy(op->keys.top, k);
|
||||
bch_keylist_push(&op->keys);
|
||||
bkey_copy(keylist.top, k);
|
||||
bch_keylist_push(&keylist);
|
||||
|
||||
op->journal = i->pin;
|
||||
atomic_inc(op->journal);
|
||||
|
||||
ret = bch_btree_insert(op, s);
|
||||
ret = bch_btree_insert(s, &keylist, i->pin, NULL);
|
||||
if (ret)
|
||||
goto err;
|
||||
|
||||
BUG_ON(!bch_keylist_empty(&op->keys));
|
||||
BUG_ON(!bch_keylist_empty(&keylist));
|
||||
keys++;
|
||||
|
||||
cond_resched();
|
||||
@ -339,14 +338,13 @@ int bch_journal_replay(struct cache_set *s, struct list_head *list,
|
||||
|
||||
pr_info("journal replay done, %i keys in %i entries, seq %llu",
|
||||
keys, entries, end);
|
||||
|
||||
err:
|
||||
while (!list_empty(list)) {
|
||||
i = list_first_entry(list, struct journal_replay, list);
|
||||
list_del(&i->list);
|
||||
kfree(i);
|
||||
}
|
||||
err:
|
||||
closure_sync(&op->cl);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -358,48 +356,35 @@ static void btree_flush_write(struct cache_set *c)
|
||||
* Try to find the btree node with that references the oldest journal
|
||||
* entry, best is our current candidate and is locked if non NULL:
|
||||
*/
|
||||
struct btree *b, *best = NULL;
|
||||
unsigned iter;
|
||||
struct btree *b, *best;
|
||||
unsigned i;
|
||||
retry:
|
||||
best = NULL;
|
||||
|
||||
for_each_cached_btree(b, c, iter) {
|
||||
if (!down_write_trylock(&b->lock))
|
||||
continue;
|
||||
|
||||
if (!btree_node_dirty(b) ||
|
||||
!btree_current_write(b)->journal) {
|
||||
rw_unlock(true, b);
|
||||
continue;
|
||||
for_each_cached_btree(b, c, i)
|
||||
if (btree_current_write(b)->journal) {
|
||||
if (!best)
|
||||
best = b;
|
||||
else if (journal_pin_cmp(c,
|
||||
btree_current_write(best)->journal,
|
||||
btree_current_write(b)->journal)) {
|
||||
best = b;
|
||||
}
|
||||
}
|
||||
|
||||
if (!best)
|
||||
best = b;
|
||||
else if (journal_pin_cmp(c,
|
||||
btree_current_write(best),
|
||||
btree_current_write(b))) {
|
||||
rw_unlock(true, best);
|
||||
best = b;
|
||||
} else
|
||||
b = best;
|
||||
if (b) {
|
||||
rw_lock(true, b, b->level);
|
||||
|
||||
if (!btree_current_write(b)->journal) {
|
||||
rw_unlock(true, b);
|
||||
/* We raced */
|
||||
goto retry;
|
||||
}
|
||||
|
||||
bch_btree_node_write(b, NULL);
|
||||
rw_unlock(true, b);
|
||||
}
|
||||
|
||||
if (best)
|
||||
goto out;
|
||||
|
||||
/* We can't find the best btree node, just pick the first */
|
||||
list_for_each_entry(b, &c->btree_cache, list)
|
||||
if (!b->level && btree_node_dirty(b)) {
|
||||
best = b;
|
||||
rw_lock(true, best, best->level);
|
||||
goto found;
|
||||
}
|
||||
|
||||
out:
|
||||
if (!best)
|
||||
return;
|
||||
found:
|
||||
if (btree_node_dirty(best))
|
||||
bch_btree_node_write(best, NULL);
|
||||
rw_unlock(true, best);
|
||||
}
|
||||
|
||||
#define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1)
|
||||
@ -495,7 +480,7 @@ static void journal_reclaim(struct cache_set *c)
|
||||
do_journal_discard(ca);
|
||||
|
||||
if (c->journal.blocks_free)
|
||||
return;
|
||||
goto out;
|
||||
|
||||
/*
|
||||
* Allocate:
|
||||
@ -521,7 +506,7 @@ static void journal_reclaim(struct cache_set *c)
|
||||
|
||||
if (n)
|
||||
c->journal.blocks_free = c->sb.bucket_size >> c->block_bits;
|
||||
|
||||
out:
|
||||
if (!journal_full(&c->journal))
|
||||
__closure_wake_up(&c->journal.wait);
|
||||
}
|
||||
@ -554,32 +539,26 @@ static void journal_write_endio(struct bio *bio, int error)
|
||||
struct journal_write *w = bio->bi_private;
|
||||
|
||||
cache_set_err_on(error, w->c, "journal io error");
|
||||
closure_put(&w->c->journal.io.cl);
|
||||
closure_put(&w->c->journal.io);
|
||||
}
|
||||
|
||||
static void journal_write(struct closure *);
|
||||
|
||||
static void journal_write_done(struct closure *cl)
|
||||
{
|
||||
struct journal *j = container_of(cl, struct journal, io.cl);
|
||||
struct cache_set *c = container_of(j, struct cache_set, journal);
|
||||
|
||||
struct journal *j = container_of(cl, struct journal, io);
|
||||
struct journal_write *w = (j->cur == j->w)
|
||||
? &j->w[1]
|
||||
: &j->w[0];
|
||||
|
||||
__closure_wake_up(&w->wait);
|
||||
|
||||
if (c->journal_delay_ms)
|
||||
closure_delay(&j->io, msecs_to_jiffies(c->journal_delay_ms));
|
||||
|
||||
continue_at(cl, journal_write, system_wq);
|
||||
continue_at_nobarrier(cl, journal_write, system_wq);
|
||||
}
|
||||
|
||||
static void journal_write_unlocked(struct closure *cl)
|
||||
__releases(c->journal.lock)
|
||||
{
|
||||
struct cache_set *c = container_of(cl, struct cache_set, journal.io.cl);
|
||||
struct cache_set *c = container_of(cl, struct cache_set, journal.io);
|
||||
struct cache *ca;
|
||||
struct journal_write *w = c->journal.cur;
|
||||
struct bkey *k = &c->journal.key;
|
||||
@ -617,7 +596,7 @@ static void journal_write_unlocked(struct closure *cl)
|
||||
for_each_cache(ca, c, i)
|
||||
w->data->prio_bucket[ca->sb.nr_this_dev] = ca->prio_buckets[0];
|
||||
|
||||
w->data->magic = jset_magic(c);
|
||||
w->data->magic = jset_magic(&c->sb);
|
||||
w->data->version = BCACHE_JSET_VERSION;
|
||||
w->data->last_seq = last_seq(&c->journal);
|
||||
w->data->csum = csum_set(w->data);
|
||||
@ -660,121 +639,134 @@ static void journal_write_unlocked(struct closure *cl)
|
||||
|
||||
static void journal_write(struct closure *cl)
|
||||
{
|
||||
struct cache_set *c = container_of(cl, struct cache_set, journal.io.cl);
|
||||
struct cache_set *c = container_of(cl, struct cache_set, journal.io);
|
||||
|
||||
spin_lock(&c->journal.lock);
|
||||
journal_write_unlocked(cl);
|
||||
}
|
||||
|
||||
static void __journal_try_write(struct cache_set *c, bool noflush)
|
||||
static void journal_try_write(struct cache_set *c)
|
||||
__releases(c->journal.lock)
|
||||
{
|
||||
struct closure *cl = &c->journal.io.cl;
|
||||
struct closure *cl = &c->journal.io;
|
||||
struct journal_write *w = c->journal.cur;
|
||||
|
||||
if (!closure_trylock(cl, &c->cl))
|
||||
spin_unlock(&c->journal.lock);
|
||||
else if (noflush && journal_full(&c->journal)) {
|
||||
spin_unlock(&c->journal.lock);
|
||||
continue_at(cl, journal_write, system_wq);
|
||||
} else
|
||||
w->need_write = true;
|
||||
|
||||
if (closure_trylock(cl, &c->cl))
|
||||
journal_write_unlocked(cl);
|
||||
else
|
||||
spin_unlock(&c->journal.lock);
|
||||
}
|
||||
|
||||
#define journal_try_write(c) __journal_try_write(c, false)
|
||||
|
||||
void bch_journal_meta(struct cache_set *c, struct closure *cl)
|
||||
static struct journal_write *journal_wait_for_write(struct cache_set *c,
|
||||
unsigned nkeys)
|
||||
{
|
||||
struct journal_write *w;
|
||||
size_t sectors;
|
||||
struct closure cl;
|
||||
|
||||
if (CACHE_SYNC(&c->sb)) {
|
||||
closure_init_stack(&cl);
|
||||
|
||||
spin_lock(&c->journal.lock);
|
||||
|
||||
while (1) {
|
||||
struct journal_write *w = c->journal.cur;
|
||||
|
||||
sectors = __set_blocks(w->data, w->data->keys + nkeys,
|
||||
c) * c->sb.block_size;
|
||||
|
||||
if (sectors <= min_t(size_t,
|
||||
c->journal.blocks_free * c->sb.block_size,
|
||||
PAGE_SECTORS << JSET_BITS))
|
||||
return w;
|
||||
|
||||
/* XXX: tracepoint */
|
||||
if (!journal_full(&c->journal)) {
|
||||
trace_bcache_journal_entry_full(c);
|
||||
|
||||
/*
|
||||
* XXX: If we were inserting so many keys that they
|
||||
* won't fit in an _empty_ journal write, we'll
|
||||
* deadlock. For now, handle this in
|
||||
* bch_keylist_realloc() - but something to think about.
|
||||
*/
|
||||
BUG_ON(!w->data->keys);
|
||||
|
||||
closure_wait(&w->wait, &cl);
|
||||
journal_try_write(c); /* unlocks */
|
||||
} else {
|
||||
trace_bcache_journal_full(c);
|
||||
|
||||
closure_wait(&c->journal.wait, &cl);
|
||||
journal_reclaim(c);
|
||||
spin_unlock(&c->journal.lock);
|
||||
|
||||
btree_flush_write(c);
|
||||
}
|
||||
|
||||
closure_sync(&cl);
|
||||
spin_lock(&c->journal.lock);
|
||||
|
||||
w = c->journal.cur;
|
||||
w->need_write = true;
|
||||
|
||||
if (cl)
|
||||
BUG_ON(!closure_wait(&w->wait, cl));
|
||||
|
||||
closure_flush(&c->journal.io);
|
||||
__journal_try_write(c, true);
|
||||
}
|
||||
}
|
||||
|
||||
static void journal_write_work(struct work_struct *work)
|
||||
{
|
||||
struct cache_set *c = container_of(to_delayed_work(work),
|
||||
struct cache_set,
|
||||
journal.work);
|
||||
spin_lock(&c->journal.lock);
|
||||
journal_try_write(c);
|
||||
}
|
||||
|
||||
/*
|
||||
* Entry point to the journalling code - bio_insert() and btree_invalidate()
|
||||
* pass bch_journal() a list of keys to be journalled, and then
|
||||
* bch_journal() hands those same keys off to btree_insert_async()
|
||||
*/
|
||||
|
||||
void bch_journal(struct closure *cl)
|
||||
atomic_t *bch_journal(struct cache_set *c,
|
||||
struct keylist *keys,
|
||||
struct closure *parent)
|
||||
{
|
||||
struct btree_op *op = container_of(cl, struct btree_op, cl);
|
||||
struct cache_set *c = op->c;
|
||||
struct journal_write *w;
|
||||
size_t b, n = ((uint64_t *) op->keys.top) - op->keys.list;
|
||||
atomic_t *ret;
|
||||
|
||||
if (op->type != BTREE_INSERT ||
|
||||
!CACHE_SYNC(&c->sb))
|
||||
goto out;
|
||||
if (!CACHE_SYNC(&c->sb))
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* If we're looping because we errored, might already be waiting on
|
||||
* another journal write:
|
||||
*/
|
||||
while (atomic_read(&cl->parent->remaining) & CLOSURE_WAITING)
|
||||
closure_sync(cl->parent);
|
||||
w = journal_wait_for_write(c, bch_keylist_nkeys(keys));
|
||||
|
||||
spin_lock(&c->journal.lock);
|
||||
memcpy(end(w->data), keys->keys, bch_keylist_bytes(keys));
|
||||
w->data->keys += bch_keylist_nkeys(keys);
|
||||
|
||||
if (journal_full(&c->journal)) {
|
||||
trace_bcache_journal_full(c);
|
||||
|
||||
closure_wait(&c->journal.wait, cl);
|
||||
|
||||
journal_reclaim(c);
|
||||
spin_unlock(&c->journal.lock);
|
||||
|
||||
btree_flush_write(c);
|
||||
continue_at(cl, bch_journal, bcache_wq);
|
||||
}
|
||||
|
||||
w = c->journal.cur;
|
||||
w->need_write = true;
|
||||
b = __set_blocks(w->data, w->data->keys + n, c);
|
||||
|
||||
if (b * c->sb.block_size > PAGE_SECTORS << JSET_BITS ||
|
||||
b > c->journal.blocks_free) {
|
||||
trace_bcache_journal_entry_full(c);
|
||||
|
||||
/*
|
||||
* XXX: If we were inserting so many keys that they won't fit in
|
||||
* an _empty_ journal write, we'll deadlock. For now, handle
|
||||
* this in bch_keylist_realloc() - but something to think about.
|
||||
*/
|
||||
BUG_ON(!w->data->keys);
|
||||
|
||||
BUG_ON(!closure_wait(&w->wait, cl));
|
||||
|
||||
closure_flush(&c->journal.io);
|
||||
ret = &fifo_back(&c->journal.pin);
|
||||
atomic_inc(ret);
|
||||
|
||||
if (parent) {
|
||||
closure_wait(&w->wait, parent);
|
||||
journal_try_write(c);
|
||||
continue_at(cl, bch_journal, bcache_wq);
|
||||
} else if (!w->need_write) {
|
||||
schedule_delayed_work(&c->journal.work,
|
||||
msecs_to_jiffies(c->journal_delay_ms));
|
||||
spin_unlock(&c->journal.lock);
|
||||
} else {
|
||||
spin_unlock(&c->journal.lock);
|
||||
}
|
||||
|
||||
memcpy(end(w->data), op->keys.list, n * sizeof(uint64_t));
|
||||
w->data->keys += n;
|
||||
|
||||
op->journal = &fifo_back(&c->journal.pin);
|
||||
atomic_inc(op->journal);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (op->flush_journal) {
|
||||
closure_flush(&c->journal.io);
|
||||
closure_wait(&w->wait, cl->parent);
|
||||
}
|
||||
void bch_journal_meta(struct cache_set *c, struct closure *cl)
|
||||
{
|
||||
struct keylist keys;
|
||||
atomic_t *ref;
|
||||
|
||||
journal_try_write(c);
|
||||
out:
|
||||
bch_btree_insert_async(cl);
|
||||
bch_keylist_init(&keys);
|
||||
|
||||
ref = bch_journal(c, &keys, cl);
|
||||
if (ref)
|
||||
atomic_dec_bug(ref);
|
||||
}
|
||||
|
||||
void bch_journal_free(struct cache_set *c)
|
||||
@ -790,6 +782,7 @@ int bch_journal_alloc(struct cache_set *c)
|
||||
|
||||
closure_init_unlocked(&j->io);
|
||||
spin_lock_init(&j->lock);
|
||||
INIT_DELAYED_WORK(&j->work, journal_write_work);
|
||||
|
||||
c->journal_delay_ms = 100;
|
||||
|
||||
|
@ -75,43 +75,6 @@
|
||||
* nodes that are pinning the oldest journal entries first.
|
||||
*/
|
||||
|
||||
#define BCACHE_JSET_VERSION_UUIDv1 1
|
||||
/* Always latest UUID format */
|
||||
#define BCACHE_JSET_VERSION_UUID 1
|
||||
#define BCACHE_JSET_VERSION 1
|
||||
|
||||
/*
|
||||
* On disk format for a journal entry:
|
||||
* seq is monotonically increasing; every journal entry has its own unique
|
||||
* sequence number.
|
||||
*
|
||||
* last_seq is the oldest journal entry that still has keys the btree hasn't
|
||||
* flushed to disk yet.
|
||||
*
|
||||
* version is for on disk format changes.
|
||||
*/
|
||||
struct jset {
|
||||
uint64_t csum;
|
||||
uint64_t magic;
|
||||
uint64_t seq;
|
||||
uint32_t version;
|
||||
uint32_t keys;
|
||||
|
||||
uint64_t last_seq;
|
||||
|
||||
BKEY_PADDED(uuid_bucket);
|
||||
BKEY_PADDED(btree_root);
|
||||
uint16_t btree_level;
|
||||
uint16_t pad[3];
|
||||
|
||||
uint64_t prio_bucket[MAX_CACHES_PER_SET];
|
||||
|
||||
union {
|
||||
struct bkey start[0];
|
||||
uint64_t d[0];
|
||||
};
|
||||
};
|
||||
|
||||
/*
|
||||
* Only used for holding the journal entries we read in btree_journal_read()
|
||||
* during cache_registration
|
||||
@ -140,7 +103,8 @@ struct journal {
|
||||
spinlock_t lock;
|
||||
/* used when waiting because the journal was full */
|
||||
struct closure_waitlist wait;
|
||||
struct closure_with_timer io;
|
||||
struct closure io;
|
||||
struct delayed_work work;
|
||||
|
||||
/* Number of blocks free in the bucket(s) we're currently writing to */
|
||||
unsigned blocks_free;
|
||||
@ -188,8 +152,7 @@ struct journal_device {
|
||||
};
|
||||
|
||||
#define journal_pin_cmp(c, l, r) \
|
||||
(fifo_idx(&(c)->journal.pin, (l)->journal) > \
|
||||
fifo_idx(&(c)->journal.pin, (r)->journal))
|
||||
(fifo_idx(&(c)->journal.pin, (l)) > fifo_idx(&(c)->journal.pin, (r)))
|
||||
|
||||
#define JOURNAL_PIN 20000
|
||||
|
||||
@ -199,15 +162,14 @@ struct journal_device {
|
||||
struct closure;
|
||||
struct cache_set;
|
||||
struct btree_op;
|
||||
struct keylist;
|
||||
|
||||
void bch_journal(struct closure *);
|
||||
atomic_t *bch_journal(struct cache_set *, struct keylist *, struct closure *);
|
||||
void bch_journal_next(struct journal *);
|
||||
void bch_journal_mark(struct cache_set *, struct list_head *);
|
||||
void bch_journal_meta(struct cache_set *, struct closure *);
|
||||
int bch_journal_read(struct cache_set *, struct list_head *,
|
||||
struct btree_op *);
|
||||
int bch_journal_replay(struct cache_set *, struct list_head *,
|
||||
struct btree_op *);
|
||||
int bch_journal_read(struct cache_set *, struct list_head *);
|
||||
int bch_journal_replay(struct cache_set *, struct list_head *);
|
||||
|
||||
void bch_journal_free(struct cache_set *);
|
||||
int bch_journal_alloc(struct cache_set *);
|
||||
|
@ -12,8 +12,9 @@
|
||||
#include <trace/events/bcache.h>
|
||||
|
||||
struct moving_io {
|
||||
struct closure cl;
|
||||
struct keybuf_key *w;
|
||||
struct search s;
|
||||
struct data_insert_op op;
|
||||
struct bbio bio;
|
||||
};
|
||||
|
||||
@ -38,13 +39,13 @@ static bool moving_pred(struct keybuf *buf, struct bkey *k)
|
||||
|
||||
static void moving_io_destructor(struct closure *cl)
|
||||
{
|
||||
struct moving_io *io = container_of(cl, struct moving_io, s.cl);
|
||||
struct moving_io *io = container_of(cl, struct moving_io, cl);
|
||||
kfree(io);
|
||||
}
|
||||
|
||||
static void write_moving_finish(struct closure *cl)
|
||||
{
|
||||
struct moving_io *io = container_of(cl, struct moving_io, s.cl);
|
||||
struct moving_io *io = container_of(cl, struct moving_io, cl);
|
||||
struct bio *bio = &io->bio.bio;
|
||||
struct bio_vec *bv;
|
||||
int i;
|
||||
@ -52,13 +53,12 @@ static void write_moving_finish(struct closure *cl)
|
||||
bio_for_each_segment_all(bv, bio, i)
|
||||
__free_page(bv->bv_page);
|
||||
|
||||
if (io->s.op.insert_collision)
|
||||
if (io->op.replace_collision)
|
||||
trace_bcache_gc_copy_collision(&io->w->key);
|
||||
|
||||
bch_keybuf_del(&io->s.op.c->moving_gc_keys, io->w);
|
||||
bch_keybuf_del(&io->op.c->moving_gc_keys, io->w);
|
||||
|
||||
atomic_dec_bug(&io->s.op.c->in_flight);
|
||||
closure_wake_up(&io->s.op.c->moving_gc_wait);
|
||||
up(&io->op.c->moving_in_flight);
|
||||
|
||||
closure_return_with_destructor(cl, moving_io_destructor);
|
||||
}
|
||||
@ -66,12 +66,12 @@ static void write_moving_finish(struct closure *cl)
|
||||
static void read_moving_endio(struct bio *bio, int error)
|
||||
{
|
||||
struct moving_io *io = container_of(bio->bi_private,
|
||||
struct moving_io, s.cl);
|
||||
struct moving_io, cl);
|
||||
|
||||
if (error)
|
||||
io->s.error = error;
|
||||
io->op.error = error;
|
||||
|
||||
bch_bbio_endio(io->s.op.c, bio, error, "reading data to move");
|
||||
bch_bbio_endio(io->op.c, bio, error, "reading data to move");
|
||||
}
|
||||
|
||||
static void moving_init(struct moving_io *io)
|
||||
@ -85,54 +85,53 @@ static void moving_init(struct moving_io *io)
|
||||
bio->bi_size = KEY_SIZE(&io->w->key) << 9;
|
||||
bio->bi_max_vecs = DIV_ROUND_UP(KEY_SIZE(&io->w->key),
|
||||
PAGE_SECTORS);
|
||||
bio->bi_private = &io->s.cl;
|
||||
bio->bi_private = &io->cl;
|
||||
bio->bi_io_vec = bio->bi_inline_vecs;
|
||||
bch_bio_map(bio, NULL);
|
||||
}
|
||||
|
||||
static void write_moving(struct closure *cl)
|
||||
{
|
||||
struct search *s = container_of(cl, struct search, cl);
|
||||
struct moving_io *io = container_of(s, struct moving_io, s);
|
||||
struct moving_io *io = container_of(cl, struct moving_io, cl);
|
||||
struct data_insert_op *op = &io->op;
|
||||
|
||||
if (!s->error) {
|
||||
if (!op->error) {
|
||||
moving_init(io);
|
||||
|
||||
io->bio.bio.bi_sector = KEY_START(&io->w->key);
|
||||
s->op.lock = -1;
|
||||
s->op.write_prio = 1;
|
||||
s->op.cache_bio = &io->bio.bio;
|
||||
io->bio.bio.bi_sector = KEY_START(&io->w->key);
|
||||
op->write_prio = 1;
|
||||
op->bio = &io->bio.bio;
|
||||
|
||||
s->writeback = KEY_DIRTY(&io->w->key);
|
||||
s->op.csum = KEY_CSUM(&io->w->key);
|
||||
op->writeback = KEY_DIRTY(&io->w->key);
|
||||
op->csum = KEY_CSUM(&io->w->key);
|
||||
|
||||
s->op.type = BTREE_REPLACE;
|
||||
bkey_copy(&s->op.replace, &io->w->key);
|
||||
bkey_copy(&op->replace_key, &io->w->key);
|
||||
op->replace = true;
|
||||
|
||||
closure_init(&s->op.cl, cl);
|
||||
bch_insert_data(&s->op.cl);
|
||||
closure_call(&op->cl, bch_data_insert, NULL, cl);
|
||||
}
|
||||
|
||||
continue_at(cl, write_moving_finish, NULL);
|
||||
continue_at(cl, write_moving_finish, system_wq);
|
||||
}
|
||||
|
||||
static void read_moving_submit(struct closure *cl)
|
||||
{
|
||||
struct search *s = container_of(cl, struct search, cl);
|
||||
struct moving_io *io = container_of(s, struct moving_io, s);
|
||||
struct moving_io *io = container_of(cl, struct moving_io, cl);
|
||||
struct bio *bio = &io->bio.bio;
|
||||
|
||||
bch_submit_bbio(bio, s->op.c, &io->w->key, 0);
|
||||
bch_submit_bbio(bio, io->op.c, &io->w->key, 0);
|
||||
|
||||
continue_at(cl, write_moving, bch_gc_wq);
|
||||
continue_at(cl, write_moving, system_wq);
|
||||
}
|
||||
|
||||
static void read_moving(struct closure *cl)
|
||||
static void read_moving(struct cache_set *c)
|
||||
{
|
||||
struct cache_set *c = container_of(cl, struct cache_set, moving_gc);
|
||||
struct keybuf_key *w;
|
||||
struct moving_io *io;
|
||||
struct bio *bio;
|
||||
struct closure cl;
|
||||
|
||||
closure_init_stack(&cl);
|
||||
|
||||
/* XXX: if we error, background writeback could stall indefinitely */
|
||||
|
||||
@ -150,8 +149,8 @@ static void read_moving(struct closure *cl)
|
||||
|
||||
w->private = io;
|
||||
io->w = w;
|
||||
io->s.op.inode = KEY_INODE(&w->key);
|
||||
io->s.op.c = c;
|
||||
io->op.inode = KEY_INODE(&w->key);
|
||||
io->op.c = c;
|
||||
|
||||
moving_init(io);
|
||||
bio = &io->bio.bio;
|
||||
@ -164,13 +163,8 @@ static void read_moving(struct closure *cl)
|
||||
|
||||
trace_bcache_gc_copy(&w->key);
|
||||
|
||||
closure_call(&io->s.cl, read_moving_submit, NULL, &c->gc.cl);
|
||||
|
||||
if (atomic_inc_return(&c->in_flight) >= 64) {
|
||||
closure_wait_event(&c->moving_gc_wait, cl,
|
||||
atomic_read(&c->in_flight) < 64);
|
||||
continue_at(cl, read_moving, bch_gc_wq);
|
||||
}
|
||||
down(&c->moving_in_flight);
|
||||
closure_call(&io->cl, read_moving_submit, NULL, &cl);
|
||||
}
|
||||
|
||||
if (0) {
|
||||
@ -180,7 +174,7 @@ err: if (!IS_ERR_OR_NULL(w->private))
|
||||
bch_keybuf_del(&c->moving_gc_keys, w);
|
||||
}
|
||||
|
||||
closure_return(cl);
|
||||
closure_sync(&cl);
|
||||
}
|
||||
|
||||
static bool bucket_cmp(struct bucket *l, struct bucket *r)
|
||||
@ -193,15 +187,14 @@ static unsigned bucket_heap_top(struct cache *ca)
|
||||
return GC_SECTORS_USED(heap_peek(&ca->heap));
|
||||
}
|
||||
|
||||
void bch_moving_gc(struct closure *cl)
|
||||
void bch_moving_gc(struct cache_set *c)
|
||||
{
|
||||
struct cache_set *c = container_of(cl, struct cache_set, gc.cl);
|
||||
struct cache *ca;
|
||||
struct bucket *b;
|
||||
unsigned i;
|
||||
|
||||
if (!c->copy_gc_enabled)
|
||||
closure_return(cl);
|
||||
return;
|
||||
|
||||
mutex_lock(&c->bucket_lock);
|
||||
|
||||
@ -242,13 +235,11 @@ void bch_moving_gc(struct closure *cl)
|
||||
|
||||
c->moving_gc_keys.last_scanned = ZERO_KEY;
|
||||
|
||||
closure_init(&c->moving_gc, cl);
|
||||
read_moving(&c->moving_gc);
|
||||
|
||||
closure_return(cl);
|
||||
read_moving(c);
|
||||
}
|
||||
|
||||
void bch_moving_init_cache_set(struct cache_set *c)
|
||||
{
|
||||
bch_keybuf_init(&c->moving_gc_keys);
|
||||
sema_init(&c->moving_in_flight, 64);
|
||||
}
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -3,40 +3,33 @@
|
||||
|
||||
#include <linux/cgroup.h>
|
||||
|
||||
struct search {
|
||||
/* Stack frame for bio_complete */
|
||||
struct data_insert_op {
|
||||
struct closure cl;
|
||||
struct cache_set *c;
|
||||
struct bio *bio;
|
||||
|
||||
struct bcache_device *d;
|
||||
struct task_struct *task;
|
||||
|
||||
struct bbio bio;
|
||||
struct bio *orig_bio;
|
||||
struct bio *cache_miss;
|
||||
unsigned cache_bio_sectors;
|
||||
|
||||
unsigned recoverable:1;
|
||||
unsigned unaligned_bvec:1;
|
||||
|
||||
unsigned write:1;
|
||||
unsigned writeback:1;
|
||||
|
||||
/* IO error returned to s->bio */
|
||||
unsigned inode;
|
||||
uint16_t write_point;
|
||||
uint16_t write_prio;
|
||||
short error;
|
||||
unsigned long start_time;
|
||||
|
||||
/* Anything past op->keys won't get zeroed in do_bio_hook */
|
||||
struct btree_op op;
|
||||
unsigned bypass:1;
|
||||
unsigned writeback:1;
|
||||
unsigned flush_journal:1;
|
||||
unsigned csum:1;
|
||||
|
||||
unsigned replace:1;
|
||||
unsigned replace_collision:1;
|
||||
|
||||
unsigned insert_data_done:1;
|
||||
|
||||
/* Anything past this point won't get zeroed in search_alloc() */
|
||||
struct keylist insert_keys;
|
||||
BKEY_PADDED(replace_key);
|
||||
};
|
||||
|
||||
void bch_cache_read_endio(struct bio *, int);
|
||||
unsigned bch_get_congested(struct cache_set *);
|
||||
void bch_insert_data(struct closure *cl);
|
||||
void bch_btree_insert_async(struct closure *);
|
||||
void bch_cache_read_endio(struct bio *, int);
|
||||
|
||||
void bch_open_buckets_free(struct cache_set *);
|
||||
int bch_open_buckets_alloc(struct cache_set *);
|
||||
void bch_data_insert(struct closure *cl);
|
||||
|
||||
void bch_cached_dev_request_init(struct cached_dev *dc);
|
||||
void bch_flash_dev_request_init(struct bcache_device *d);
|
||||
|
@ -7,7 +7,6 @@
|
||||
#include "bcache.h"
|
||||
#include "stats.h"
|
||||
#include "btree.h"
|
||||
#include "request.h"
|
||||
#include "sysfs.h"
|
||||
|
||||
/*
|
||||
@ -196,35 +195,36 @@ static void mark_cache_stats(struct cache_stat_collector *stats,
|
||||
atomic_inc(&stats->cache_bypass_misses);
|
||||
}
|
||||
|
||||
void bch_mark_cache_accounting(struct search *s, bool hit, bool bypass)
|
||||
void bch_mark_cache_accounting(struct cache_set *c, struct bcache_device *d,
|
||||
bool hit, bool bypass)
|
||||
{
|
||||
struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
|
||||
struct cached_dev *dc = container_of(d, struct cached_dev, disk);
|
||||
mark_cache_stats(&dc->accounting.collector, hit, bypass);
|
||||
mark_cache_stats(&s->op.c->accounting.collector, hit, bypass);
|
||||
mark_cache_stats(&c->accounting.collector, hit, bypass);
|
||||
#ifdef CONFIG_CGROUP_BCACHE
|
||||
mark_cache_stats(&(bch_bio_to_cgroup(s->orig_bio)->stats), hit, bypass);
|
||||
#endif
|
||||
}
|
||||
|
||||
void bch_mark_cache_readahead(struct search *s)
|
||||
void bch_mark_cache_readahead(struct cache_set *c, struct bcache_device *d)
|
||||
{
|
||||
struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
|
||||
struct cached_dev *dc = container_of(d, struct cached_dev, disk);
|
||||
atomic_inc(&dc->accounting.collector.cache_readaheads);
|
||||
atomic_inc(&s->op.c->accounting.collector.cache_readaheads);
|
||||
atomic_inc(&c->accounting.collector.cache_readaheads);
|
||||
}
|
||||
|
||||
void bch_mark_cache_miss_collision(struct search *s)
|
||||
void bch_mark_cache_miss_collision(struct cache_set *c, struct bcache_device *d)
|
||||
{
|
||||
struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
|
||||
struct cached_dev *dc = container_of(d, struct cached_dev, disk);
|
||||
atomic_inc(&dc->accounting.collector.cache_miss_collisions);
|
||||
atomic_inc(&s->op.c->accounting.collector.cache_miss_collisions);
|
||||
atomic_inc(&c->accounting.collector.cache_miss_collisions);
|
||||
}
|
||||
|
||||
void bch_mark_sectors_bypassed(struct search *s, int sectors)
|
||||
void bch_mark_sectors_bypassed(struct cache_set *c, struct cached_dev *dc,
|
||||
int sectors)
|
||||
{
|
||||
struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
|
||||
atomic_add(sectors, &dc->accounting.collector.sectors_bypassed);
|
||||
atomic_add(sectors, &s->op.c->accounting.collector.sectors_bypassed);
|
||||
atomic_add(sectors, &c->accounting.collector.sectors_bypassed);
|
||||
}
|
||||
|
||||
void bch_cache_accounting_init(struct cache_accounting *acc,
|
||||
|
@ -38,7 +38,9 @@ struct cache_accounting {
|
||||
struct cache_stats day;
|
||||
};
|
||||
|
||||
struct search;
|
||||
struct cache_set;
|
||||
struct cached_dev;
|
||||
struct bcache_device;
|
||||
|
||||
void bch_cache_accounting_init(struct cache_accounting *acc,
|
||||
struct closure *parent);
|
||||
@ -50,9 +52,10 @@ void bch_cache_accounting_clear(struct cache_accounting *acc);
|
||||
|
||||
void bch_cache_accounting_destroy(struct cache_accounting *acc);
|
||||
|
||||
void bch_mark_cache_accounting(struct search *s, bool hit, bool bypass);
|
||||
void bch_mark_cache_readahead(struct search *s);
|
||||
void bch_mark_cache_miss_collision(struct search *s);
|
||||
void bch_mark_sectors_bypassed(struct search *s, int sectors);
|
||||
void bch_mark_cache_accounting(struct cache_set *, struct bcache_device *,
|
||||
bool, bool);
|
||||
void bch_mark_cache_readahead(struct cache_set *, struct bcache_device *);
|
||||
void bch_mark_cache_miss_collision(struct cache_set *, struct bcache_device *);
|
||||
void bch_mark_sectors_bypassed(struct cache_set *, struct cached_dev *, int);
|
||||
|
||||
#endif /* _BCACHE_STATS_H_ */
|
||||
|
@ -16,6 +16,7 @@
|
||||
#include <linux/buffer_head.h>
|
||||
#include <linux/debugfs.h>
|
||||
#include <linux/genhd.h>
|
||||
#include <linux/idr.h>
|
||||
#include <linux/kthread.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/random.h>
|
||||
@ -45,21 +46,13 @@ const char * const bch_cache_modes[] = {
|
||||
NULL
|
||||
};
|
||||
|
||||
struct uuid_entry_v0 {
|
||||
uint8_t uuid[16];
|
||||
uint8_t label[32];
|
||||
uint32_t first_reg;
|
||||
uint32_t last_reg;
|
||||
uint32_t invalidated;
|
||||
uint32_t pad;
|
||||
};
|
||||
|
||||
static struct kobject *bcache_kobj;
|
||||
struct mutex bch_register_lock;
|
||||
LIST_HEAD(bch_cache_sets);
|
||||
static LIST_HEAD(uncached_devices);
|
||||
|
||||
static int bcache_major, bcache_minor;
|
||||
static int bcache_major;
|
||||
static DEFINE_IDA(bcache_minor);
|
||||
static wait_queue_head_t unregister_wait;
|
||||
struct workqueue_struct *bcache_wq;
|
||||
|
||||
@ -382,7 +375,7 @@ static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
|
||||
{
|
||||
struct bkey *k = &j->uuid_bucket;
|
||||
|
||||
if (__bch_ptr_invalid(c, 1, k))
|
||||
if (bch_btree_ptr_invalid(c, k))
|
||||
return "bad uuid pointer";
|
||||
|
||||
bkey_copy(&c->uuid_bucket, k);
|
||||
@ -427,7 +420,7 @@ static int __uuid_write(struct cache_set *c)
|
||||
|
||||
lockdep_assert_held(&bch_register_lock);
|
||||
|
||||
if (bch_bucket_alloc_set(c, WATERMARK_METADATA, &k.key, 1, &cl))
|
||||
if (bch_bucket_alloc_set(c, WATERMARK_METADATA, &k.key, 1, true))
|
||||
return 1;
|
||||
|
||||
SET_KEY_SIZE(&k.key, c->sb.bucket_size);
|
||||
@ -435,7 +428,7 @@ static int __uuid_write(struct cache_set *c)
|
||||
closure_sync(&cl);
|
||||
|
||||
bkey_copy(&c->uuid_bucket, &k.key);
|
||||
__bkey_put(c, &k.key);
|
||||
bkey_put(c, &k.key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -562,10 +555,10 @@ void bch_prio_write(struct cache *ca)
|
||||
}
|
||||
|
||||
p->next_bucket = ca->prio_buckets[i + 1];
|
||||
p->magic = pset_magic(ca);
|
||||
p->magic = pset_magic(&ca->sb);
|
||||
p->csum = bch_crc64(&p->magic, bucket_bytes(ca) - 8);
|
||||
|
||||
bucket = bch_bucket_alloc(ca, WATERMARK_PRIO, &cl);
|
||||
bucket = bch_bucket_alloc(ca, WATERMARK_PRIO, true);
|
||||
BUG_ON(bucket == -1);
|
||||
|
||||
mutex_unlock(&ca->set->bucket_lock);
|
||||
@ -613,7 +606,7 @@ static void prio_read(struct cache *ca, uint64_t bucket)
|
||||
if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8))
|
||||
pr_warn("bad csum reading priorities");
|
||||
|
||||
if (p->magic != pset_magic(ca))
|
||||
if (p->magic != pset_magic(&ca->sb))
|
||||
pr_warn("bad magic reading priorities");
|
||||
|
||||
bucket = p->next_bucket;
|
||||
@ -630,7 +623,7 @@ static void prio_read(struct cache *ca, uint64_t bucket)
|
||||
static int open_dev(struct block_device *b, fmode_t mode)
|
||||
{
|
||||
struct bcache_device *d = b->bd_disk->private_data;
|
||||
if (atomic_read(&d->closing))
|
||||
if (test_bit(BCACHE_DEV_CLOSING, &d->flags))
|
||||
return -ENXIO;
|
||||
|
||||
closure_get(&d->cl);
|
||||
@ -659,20 +652,24 @@ static const struct block_device_operations bcache_ops = {
|
||||
|
||||
void bcache_device_stop(struct bcache_device *d)
|
||||
{
|
||||
if (!atomic_xchg(&d->closing, 1))
|
||||
if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags))
|
||||
closure_queue(&d->cl);
|
||||
}
|
||||
|
||||
static void bcache_device_unlink(struct bcache_device *d)
|
||||
{
|
||||
unsigned i;
|
||||
struct cache *ca;
|
||||
lockdep_assert_held(&bch_register_lock);
|
||||
|
||||
sysfs_remove_link(&d->c->kobj, d->name);
|
||||
sysfs_remove_link(&d->kobj, "cache");
|
||||
if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) {
|
||||
unsigned i;
|
||||
struct cache *ca;
|
||||
|
||||
for_each_cache(ca, d->c, i)
|
||||
bd_unlink_disk_holder(ca->bdev, d->disk);
|
||||
sysfs_remove_link(&d->c->kobj, d->name);
|
||||
sysfs_remove_link(&d->kobj, "cache");
|
||||
|
||||
for_each_cache(ca, d->c, i)
|
||||
bd_unlink_disk_holder(ca->bdev, d->disk);
|
||||
}
|
||||
}
|
||||
|
||||
static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
|
||||
@ -696,19 +693,16 @@ static void bcache_device_detach(struct bcache_device *d)
|
||||
{
|
||||
lockdep_assert_held(&bch_register_lock);
|
||||
|
||||
if (atomic_read(&d->detaching)) {
|
||||
if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) {
|
||||
struct uuid_entry *u = d->c->uuids + d->id;
|
||||
|
||||
SET_UUID_FLASH_ONLY(u, 0);
|
||||
memcpy(u->uuid, invalid_uuid, 16);
|
||||
u->invalidated = cpu_to_le32(get_seconds());
|
||||
bch_uuid_write(d->c);
|
||||
|
||||
atomic_set(&d->detaching, 0);
|
||||
}
|
||||
|
||||
if (!d->flush_done)
|
||||
bcache_device_unlink(d);
|
||||
bcache_device_unlink(d);
|
||||
|
||||
d->c->devices[d->id] = NULL;
|
||||
closure_put(&d->c->caching);
|
||||
@ -739,14 +733,20 @@ static void bcache_device_free(struct bcache_device *d)
|
||||
del_gendisk(d->disk);
|
||||
if (d->disk && d->disk->queue)
|
||||
blk_cleanup_queue(d->disk->queue);
|
||||
if (d->disk)
|
||||
if (d->disk) {
|
||||
ida_simple_remove(&bcache_minor, d->disk->first_minor);
|
||||
put_disk(d->disk);
|
||||
}
|
||||
|
||||
bio_split_pool_free(&d->bio_split_hook);
|
||||
if (d->unaligned_bvec)
|
||||
mempool_destroy(d->unaligned_bvec);
|
||||
if (d->bio_split)
|
||||
bioset_free(d->bio_split);
|
||||
if (is_vmalloc_addr(d->full_dirty_stripes))
|
||||
vfree(d->full_dirty_stripes);
|
||||
else
|
||||
kfree(d->full_dirty_stripes);
|
||||
if (is_vmalloc_addr(d->stripe_sectors_dirty))
|
||||
vfree(d->stripe_sectors_dirty);
|
||||
else
|
||||
@ -760,15 +760,19 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
|
||||
{
|
||||
struct request_queue *q;
|
||||
size_t n;
|
||||
int minor;
|
||||
|
||||
if (!d->stripe_size_bits)
|
||||
d->stripe_size_bits = 31;
|
||||
if (!d->stripe_size)
|
||||
d->stripe_size = 1 << 31;
|
||||
|
||||
d->nr_stripes = round_up(sectors, 1 << d->stripe_size_bits) >>
|
||||
d->stripe_size_bits;
|
||||
d->nr_stripes = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
|
||||
|
||||
if (!d->nr_stripes || d->nr_stripes > SIZE_MAX / sizeof(atomic_t))
|
||||
if (!d->nr_stripes ||
|
||||
d->nr_stripes > INT_MAX ||
|
||||
d->nr_stripes > SIZE_MAX / sizeof(atomic_t)) {
|
||||
pr_err("nr_stripes too large");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
n = d->nr_stripes * sizeof(atomic_t);
|
||||
d->stripe_sectors_dirty = n < PAGE_SIZE << 6
|
||||
@ -777,22 +781,38 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
|
||||
if (!d->stripe_sectors_dirty)
|
||||
return -ENOMEM;
|
||||
|
||||
n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);
|
||||
d->full_dirty_stripes = n < PAGE_SIZE << 6
|
||||
? kzalloc(n, GFP_KERNEL)
|
||||
: vzalloc(n);
|
||||
if (!d->full_dirty_stripes)
|
||||
return -ENOMEM;
|
||||
|
||||
minor = ida_simple_get(&bcache_minor, 0, MINORMASK + 1, GFP_KERNEL);
|
||||
if (minor < 0)
|
||||
return minor;
|
||||
|
||||
if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
|
||||
!(d->unaligned_bvec = mempool_create_kmalloc_pool(1,
|
||||
sizeof(struct bio_vec) * BIO_MAX_PAGES)) ||
|
||||
bio_split_pool_init(&d->bio_split_hook) ||
|
||||
!(d->disk = alloc_disk(1)) ||
|
||||
!(q = blk_alloc_queue(GFP_KERNEL)))
|
||||
!(d->disk = alloc_disk(1))) {
|
||||
ida_simple_remove(&bcache_minor, minor);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
set_capacity(d->disk, sectors);
|
||||
snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", bcache_minor);
|
||||
snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", minor);
|
||||
|
||||
d->disk->major = bcache_major;
|
||||
d->disk->first_minor = bcache_minor++;
|
||||
d->disk->first_minor = minor;
|
||||
d->disk->fops = &bcache_ops;
|
||||
d->disk->private_data = d;
|
||||
|
||||
q = blk_alloc_queue(GFP_KERNEL);
|
||||
if (!q)
|
||||
return -ENOMEM;
|
||||
|
||||
blk_queue_make_request(q, NULL);
|
||||
d->disk->queue = q;
|
||||
q->queuedata = d;
|
||||
@ -874,7 +894,7 @@ static void cached_dev_detach_finish(struct work_struct *w)
|
||||
struct closure cl;
|
||||
closure_init_stack(&cl);
|
||||
|
||||
BUG_ON(!atomic_read(&dc->disk.detaching));
|
||||
BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
|
||||
BUG_ON(atomic_read(&dc->count));
|
||||
|
||||
mutex_lock(&bch_register_lock);
|
||||
@ -888,6 +908,8 @@ static void cached_dev_detach_finish(struct work_struct *w)
|
||||
bcache_device_detach(&dc->disk);
|
||||
list_move(&dc->list, &uncached_devices);
|
||||
|
||||
clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);
|
||||
|
||||
mutex_unlock(&bch_register_lock);
|
||||
|
||||
pr_info("Caching disabled for %s", bdevname(dc->bdev, buf));
|
||||
@ -900,10 +922,10 @@ void bch_cached_dev_detach(struct cached_dev *dc)
|
||||
{
|
||||
lockdep_assert_held(&bch_register_lock);
|
||||
|
||||
if (atomic_read(&dc->disk.closing))
|
||||
if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
|
||||
return;
|
||||
|
||||
if (atomic_xchg(&dc->disk.detaching, 1))
|
||||
if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
|
||||
return;
|
||||
|
||||
/*
|
||||
@ -1030,6 +1052,7 @@ static void cached_dev_free(struct closure *cl)
|
||||
struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
|
||||
|
||||
cancel_delayed_work_sync(&dc->writeback_rate_update);
|
||||
kthread_stop(dc->writeback_thread);
|
||||
|
||||
mutex_lock(&bch_register_lock);
|
||||
|
||||
@ -1058,11 +1081,7 @@ static void cached_dev_flush(struct closure *cl)
|
||||
struct bcache_device *d = &dc->disk;
|
||||
|
||||
mutex_lock(&bch_register_lock);
|
||||
d->flush_done = 1;
|
||||
|
||||
if (d->c)
|
||||
bcache_device_unlink(d);
|
||||
|
||||
bcache_device_unlink(d);
|
||||
mutex_unlock(&bch_register_lock);
|
||||
|
||||
bch_cache_accounting_destroy(&dc->accounting);
|
||||
@ -1088,7 +1107,6 @@ static int cached_dev_init(struct cached_dev *dc, unsigned block_size)
|
||||
spin_lock_init(&dc->io_lock);
|
||||
bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);
|
||||
|
||||
dc->sequential_merge = true;
|
||||
dc->sequential_cutoff = 4 << 20;
|
||||
|
||||
for (io = dc->io; io < dc->io + RECENT_IO; io++) {
|
||||
@ -1260,7 +1278,8 @@ bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
|
||||
{
|
||||
va_list args;
|
||||
|
||||
if (test_bit(CACHE_SET_STOPPING, &c->flags))
|
||||
if (c->on_error != ON_ERROR_PANIC &&
|
||||
test_bit(CACHE_SET_STOPPING, &c->flags))
|
||||
return false;
|
||||
|
||||
/* XXX: we can be called from atomic context
|
||||
@ -1275,6 +1294,9 @@ bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
|
||||
|
||||
printk(", disabling caching\n");
|
||||
|
||||
if (c->on_error == ON_ERROR_PANIC)
|
||||
panic("panic forced after error\n");
|
||||
|
||||
bch_cache_set_unregister(c);
|
||||
return true;
|
||||
}
|
||||
@ -1339,6 +1361,9 @@ static void cache_set_flush(struct closure *cl)
|
||||
kobject_put(&c->internal);
|
||||
kobject_del(&c->kobj);
|
||||
|
||||
if (c->gc_thread)
|
||||
kthread_stop(c->gc_thread);
|
||||
|
||||
if (!IS_ERR_OR_NULL(c->root))
|
||||
list_add(&c->root->list, &c->btree_cache);
|
||||
|
||||
@ -1433,12 +1458,19 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
|
||||
|
||||
c->sort_crit_factor = int_sqrt(c->btree_pages);
|
||||
|
||||
mutex_init(&c->bucket_lock);
|
||||
mutex_init(&c->sort_lock);
|
||||
spin_lock_init(&c->sort_time_lock);
|
||||
closure_init_unlocked(&c->sb_write);
|
||||
mutex_init(&c->bucket_lock);
|
||||
init_waitqueue_head(&c->try_wait);
|
||||
init_waitqueue_head(&c->bucket_wait);
|
||||
closure_init_unlocked(&c->uuid_write);
|
||||
spin_lock_init(&c->btree_read_time_lock);
|
||||
mutex_init(&c->sort_lock);
|
||||
|
||||
spin_lock_init(&c->sort_time.lock);
|
||||
spin_lock_init(&c->btree_gc_time.lock);
|
||||
spin_lock_init(&c->btree_split_time.lock);
|
||||
spin_lock_init(&c->btree_read_time.lock);
|
||||
spin_lock_init(&c->try_harder_time.lock);
|
||||
|
||||
bch_moving_init_cache_set(c);
|
||||
|
||||
INIT_LIST_HEAD(&c->list);
|
||||
@ -1483,11 +1515,10 @@ static void run_cache_set(struct cache_set *c)
|
||||
const char *err = "cannot allocate memory";
|
||||
struct cached_dev *dc, *t;
|
||||
struct cache *ca;
|
||||
struct closure cl;
|
||||
unsigned i;
|
||||
|
||||
struct btree_op op;
|
||||
bch_btree_op_init_stack(&op);
|
||||
op.lock = SHRT_MAX;
|
||||
closure_init_stack(&cl);
|
||||
|
||||
for_each_cache(ca, c, i)
|
||||
c->nbuckets += ca->sb.nbuckets;
|
||||
@ -1498,7 +1529,7 @@ static void run_cache_set(struct cache_set *c)
|
||||
struct jset *j;
|
||||
|
||||
err = "cannot allocate memory for journal";
|
||||
if (bch_journal_read(c, &journal, &op))
|
||||
if (bch_journal_read(c, &journal))
|
||||
goto err;
|
||||
|
||||
pr_debug("btree_journal_read() done");
|
||||
@ -1522,23 +1553,23 @@ static void run_cache_set(struct cache_set *c)
|
||||
k = &j->btree_root;
|
||||
|
||||
err = "bad btree root";
|
||||
if (__bch_ptr_invalid(c, j->btree_level + 1, k))
|
||||
if (bch_btree_ptr_invalid(c, k))
|
||||
goto err;
|
||||
|
||||
err = "error reading btree root";
|
||||
c->root = bch_btree_node_get(c, k, j->btree_level, &op);
|
||||
c->root = bch_btree_node_get(c, k, j->btree_level, true);
|
||||
if (IS_ERR_OR_NULL(c->root))
|
||||
goto err;
|
||||
|
||||
list_del_init(&c->root->list);
|
||||
rw_unlock(true, c->root);
|
||||
|
||||
err = uuid_read(c, j, &op.cl);
|
||||
err = uuid_read(c, j, &cl);
|
||||
if (err)
|
||||
goto err;
|
||||
|
||||
err = "error in recovery";
|
||||
if (bch_btree_check(c, &op))
|
||||
if (bch_btree_check(c))
|
||||
goto err;
|
||||
|
||||
bch_journal_mark(c, &journal);
|
||||
@ -1570,11 +1601,9 @@ static void run_cache_set(struct cache_set *c)
|
||||
if (j->version < BCACHE_JSET_VERSION_UUID)
|
||||
__uuid_write(c);
|
||||
|
||||
bch_journal_replay(c, &journal, &op);
|
||||
bch_journal_replay(c, &journal);
|
||||
} else {
|
||||
pr_notice("invalidating existing data");
|
||||
/* Don't want invalidate_buckets() to queue a gc yet */
|
||||
closure_lock(&c->gc, NULL);
|
||||
|
||||
for_each_cache(ca, c, i) {
|
||||
unsigned j;
|
||||
@ -1600,15 +1629,15 @@ static void run_cache_set(struct cache_set *c)
|
||||
|
||||
err = "cannot allocate new UUID bucket";
|
||||
if (__uuid_write(c))
|
||||
goto err_unlock_gc;
|
||||
goto err;
|
||||
|
||||
err = "cannot allocate new btree root";
|
||||
c->root = bch_btree_node_alloc(c, 0, &op.cl);
|
||||
c->root = bch_btree_node_alloc(c, 0, true);
|
||||
if (IS_ERR_OR_NULL(c->root))
|
||||
goto err_unlock_gc;
|
||||
goto err;
|
||||
|
||||
bkey_copy_key(&c->root->key, &MAX_KEY);
|
||||
bch_btree_node_write(c->root, &op.cl);
|
||||
bch_btree_node_write(c->root, &cl);
|
||||
|
||||
bch_btree_set_root(c->root);
|
||||
rw_unlock(true, c->root);
|
||||
@ -1621,14 +1650,14 @@ static void run_cache_set(struct cache_set *c)
|
||||
SET_CACHE_SYNC(&c->sb, true);
|
||||
|
||||
bch_journal_next(&c->journal);
|
||||
bch_journal_meta(c, &op.cl);
|
||||
|
||||
/* Unlock */
|
||||
closure_set_stopped(&c->gc.cl);
|
||||
closure_put(&c->gc.cl);
|
||||
bch_journal_meta(c, &cl);
|
||||
}
|
||||
|
||||
closure_sync(&op.cl);
|
||||
err = "error starting gc thread";
|
||||
if (bch_gc_thread_start(c))
|
||||
goto err;
|
||||
|
||||
closure_sync(&cl);
|
||||
c->sb.last_mount = get_seconds();
|
||||
bcache_write_super(c);
|
||||
|
||||
@ -1638,13 +1667,10 @@ static void run_cache_set(struct cache_set *c)
|
||||
flash_devs_run(c);
|
||||
|
||||
return;
|
||||
err_unlock_gc:
|
||||
closure_set_stopped(&c->gc.cl);
|
||||
closure_put(&c->gc.cl);
|
||||
err:
|
||||
closure_sync(&op.cl);
|
||||
closure_sync(&cl);
|
||||
/* XXX: test this, it's broken */
|
||||
bch_cache_set_error(c, err);
|
||||
bch_cache_set_error(c, "%s", err);
|
||||
}
|
||||
|
||||
static bool can_attach_cache(struct cache *ca, struct cache_set *c)
|
||||
@ -1725,8 +1751,6 @@ void bch_cache_release(struct kobject *kobj)
|
||||
if (ca->set)
|
||||
ca->set->cache[ca->sb.nr_this_dev] = NULL;
|
||||
|
||||
bch_cache_allocator_exit(ca);
|
||||
|
||||
bio_split_pool_free(&ca->bio_split_hook);
|
||||
|
||||
free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));
|
||||
@ -1758,8 +1782,6 @@ static int cache_alloc(struct cache_sb *sb, struct cache *ca)
|
||||
__module_get(THIS_MODULE);
|
||||
kobject_init(&ca->kobj, &bch_cache_ktype);
|
||||
|
||||
INIT_LIST_HEAD(&ca->discards);
|
||||
|
||||
bio_init(&ca->journal.bio);
|
||||
ca->journal.bio.bi_max_vecs = 8;
|
||||
ca->journal.bio.bi_io_vec = ca->journal.bio.bi_inline_vecs;
|
||||
@ -2006,7 +2028,6 @@ static struct notifier_block reboot = {
|
||||
static void bcache_exit(void)
|
||||
{
|
||||
bch_debug_exit();
|
||||
bch_writeback_exit();
|
||||
bch_request_exit();
|
||||
bch_btree_exit();
|
||||
if (bcache_kobj)
|
||||
@ -2039,7 +2060,6 @@ static int __init bcache_init(void)
|
||||
sysfs_create_files(bcache_kobj, files) ||
|
||||
bch_btree_init() ||
|
||||
bch_request_init() ||
|
||||
bch_writeback_init() ||
|
||||
bch_debug_init(bcache_kobj))
|
||||
goto err;
|
||||
|
||||
|
@ -21,6 +21,12 @@ static const char * const cache_replacement_policies[] = {
|
||||
NULL
|
||||
};
|
||||
|
||||
static const char * const error_actions[] = {
|
||||
"unregister",
|
||||
"panic",
|
||||
NULL
|
||||
};
|
||||
|
||||
write_attribute(attach);
|
||||
write_attribute(detach);
|
||||
write_attribute(unregister);
|
||||
@ -66,7 +72,6 @@ rw_attribute(congested_read_threshold_us);
|
||||
rw_attribute(congested_write_threshold_us);
|
||||
|
||||
rw_attribute(sequential_cutoff);
|
||||
rw_attribute(sequential_merge);
|
||||
rw_attribute(data_csum);
|
||||
rw_attribute(cache_mode);
|
||||
rw_attribute(writeback_metadata);
|
||||
@ -90,11 +95,14 @@ rw_attribute(discard);
|
||||
rw_attribute(running);
|
||||
rw_attribute(label);
|
||||
rw_attribute(readahead);
|
||||
rw_attribute(errors);
|
||||
rw_attribute(io_error_limit);
|
||||
rw_attribute(io_error_halflife);
|
||||
rw_attribute(verify);
|
||||
rw_attribute(bypass_torture_test);
|
||||
rw_attribute(key_merging_disabled);
|
||||
rw_attribute(gc_always_rewrite);
|
||||
rw_attribute(expensive_debug_checks);
|
||||
rw_attribute(freelist_percent);
|
||||
rw_attribute(cache_replacement_policy);
|
||||
rw_attribute(btree_shrinker_disabled);
|
||||
@ -116,6 +124,7 @@ SHOW(__bch_cached_dev)
|
||||
|
||||
sysfs_printf(data_csum, "%i", dc->disk.data_csum);
|
||||
var_printf(verify, "%i");
|
||||
var_printf(bypass_torture_test, "%i");
|
||||
var_printf(writeback_metadata, "%i");
|
||||
var_printf(writeback_running, "%i");
|
||||
var_print(writeback_delay);
|
||||
@ -150,10 +159,9 @@ SHOW(__bch_cached_dev)
|
||||
sysfs_hprint(dirty_data,
|
||||
bcache_dev_sectors_dirty(&dc->disk) << 9);
|
||||
|
||||
sysfs_hprint(stripe_size, (1 << dc->disk.stripe_size_bits) << 9);
|
||||
sysfs_hprint(stripe_size, dc->disk.stripe_size << 9);
|
||||
var_printf(partial_stripes_expensive, "%u");
|
||||
|
||||
var_printf(sequential_merge, "%i");
|
||||
var_hprint(sequential_cutoff);
|
||||
var_hprint(readahead);
|
||||
|
||||
@ -185,6 +193,7 @@ STORE(__cached_dev)
|
||||
|
||||
sysfs_strtoul(data_csum, dc->disk.data_csum);
|
||||
d_strtoul(verify);
|
||||
d_strtoul(bypass_torture_test);
|
||||
d_strtoul(writeback_metadata);
|
||||
d_strtoul(writeback_running);
|
||||
d_strtoul(writeback_delay);
|
||||
@ -199,7 +208,6 @@ STORE(__cached_dev)
|
||||
dc->writeback_rate_p_term_inverse, 1, INT_MAX);
|
||||
d_strtoul(writeback_rate_d_smooth);
|
||||
|
||||
d_strtoul(sequential_merge);
|
||||
d_strtoi_h(sequential_cutoff);
|
||||
d_strtoi_h(readahead);
|
||||
|
||||
@ -311,7 +319,6 @@ static struct attribute *bch_cached_dev_files[] = {
|
||||
&sysfs_stripe_size,
|
||||
&sysfs_partial_stripes_expensive,
|
||||
&sysfs_sequential_cutoff,
|
||||
&sysfs_sequential_merge,
|
||||
&sysfs_clear_stats,
|
||||
&sysfs_running,
|
||||
&sysfs_state,
|
||||
@ -319,6 +326,7 @@ static struct attribute *bch_cached_dev_files[] = {
|
||||
&sysfs_readahead,
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
&sysfs_verify,
|
||||
&sysfs_bypass_torture_test,
|
||||
#endif
|
||||
NULL
|
||||
};
|
||||
@ -366,7 +374,7 @@ STORE(__bch_flash_dev)
|
||||
}
|
||||
|
||||
if (attr == &sysfs_unregister) {
|
||||
atomic_set(&d->detaching, 1);
|
||||
set_bit(BCACHE_DEV_DETACHING, &d->flags);
|
||||
bcache_device_stop(d);
|
||||
}
|
||||
|
||||
@ -481,7 +489,6 @@ lock_root:
|
||||
|
||||
sysfs_print(btree_used_percent, btree_used(c));
|
||||
sysfs_print(btree_nodes, c->gc_stats.nodes);
|
||||
sysfs_hprint(dirty_data, c->gc_stats.dirty);
|
||||
sysfs_hprint(average_key_size, average_key_size(c));
|
||||
|
||||
sysfs_print(cache_read_races,
|
||||
@ -492,6 +499,10 @@ lock_root:
|
||||
sysfs_print(writeback_keys_failed,
|
||||
atomic_long_read(&c->writeback_keys_failed));
|
||||
|
||||
if (attr == &sysfs_errors)
|
||||
return bch_snprint_string_list(buf, PAGE_SIZE, error_actions,
|
||||
c->on_error);
|
||||
|
||||
/* See count_io_errors for why 88 */
|
||||
sysfs_print(io_error_halflife, c->error_decay * 88);
|
||||
sysfs_print(io_error_limit, c->error_limit >> IO_ERROR_SHIFT);
|
||||
@ -506,6 +517,8 @@ lock_root:
|
||||
sysfs_print(active_journal_entries, fifo_used(&c->journal.pin));
|
||||
sysfs_printf(verify, "%i", c->verify);
|
||||
sysfs_printf(key_merging_disabled, "%i", c->key_merging_disabled);
|
||||
sysfs_printf(expensive_debug_checks,
|
||||
"%i", c->expensive_debug_checks);
|
||||
sysfs_printf(gc_always_rewrite, "%i", c->gc_always_rewrite);
|
||||
sysfs_printf(btree_shrinker_disabled, "%i", c->shrinker_disabled);
|
||||
sysfs_printf(copy_gc_enabled, "%i", c->copy_gc_enabled);
|
||||
@ -555,7 +568,7 @@ STORE(__bch_cache_set)
|
||||
}
|
||||
|
||||
if (attr == &sysfs_trigger_gc)
|
||||
bch_queue_gc(c);
|
||||
wake_up_gc(c);
|
||||
|
||||
if (attr == &sysfs_prune_cache) {
|
||||
struct shrink_control sc;
|
||||
@ -569,6 +582,15 @@ STORE(__bch_cache_set)
|
||||
sysfs_strtoul(congested_write_threshold_us,
|
||||
c->congested_write_threshold_us);
|
||||
|
||||
if (attr == &sysfs_errors) {
|
||||
ssize_t v = bch_read_string_list(buf, error_actions);
|
||||
|
||||
if (v < 0)
|
||||
return v;
|
||||
|
||||
c->on_error = v;
|
||||
}
|
||||
|
||||
if (attr == &sysfs_io_error_limit)
|
||||
c->error_limit = strtoul_or_return(buf) << IO_ERROR_SHIFT;
|
||||
|
||||
@ -579,6 +601,7 @@ STORE(__bch_cache_set)
|
||||
sysfs_strtoul(journal_delay_ms, c->journal_delay_ms);
|
||||
sysfs_strtoul(verify, c->verify);
|
||||
sysfs_strtoul(key_merging_disabled, c->key_merging_disabled);
|
||||
sysfs_strtoul(expensive_debug_checks, c->expensive_debug_checks);
|
||||
sysfs_strtoul(gc_always_rewrite, c->gc_always_rewrite);
|
||||
sysfs_strtoul(btree_shrinker_disabled, c->shrinker_disabled);
|
||||
sysfs_strtoul(copy_gc_enabled, c->copy_gc_enabled);
|
||||
@ -618,8 +641,8 @@ static struct attribute *bch_cache_set_files[] = {
|
||||
&sysfs_cache_available_percent,
|
||||
|
||||
&sysfs_average_key_size,
|
||||
&sysfs_dirty_data,
|
||||
|
||||
&sysfs_errors,
|
||||
&sysfs_io_error_limit,
|
||||
&sysfs_io_error_halflife,
|
||||
&sysfs_congested,
|
||||
@ -653,6 +676,7 @@ static struct attribute *bch_cache_set_internal_files[] = {
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
&sysfs_verify,
|
||||
&sysfs_key_merging_disabled,
|
||||
&sysfs_expensive_debug_checks,
|
||||
#endif
|
||||
&sysfs_gc_always_rewrite,
|
||||
&sysfs_btree_shrinker_disabled,
|
||||
|
@ -1,6 +1,5 @@
|
||||
#include "bcache.h"
|
||||
#include "btree.h"
|
||||
#include "request.h"
|
||||
|
||||
#include <linux/blktrace_api.h>
|
||||
#include <linux/module.h>
|
||||
|
@ -168,10 +168,14 @@ int bch_parse_uuid(const char *s, char *uuid)
|
||||
|
||||
void bch_time_stats_update(struct time_stats *stats, uint64_t start_time)
|
||||
{
|
||||
uint64_t now = local_clock();
|
||||
uint64_t duration = time_after64(now, start_time)
|
||||
uint64_t now, duration, last;
|
||||
|
||||
spin_lock(&stats->lock);
|
||||
|
||||
now = local_clock();
|
||||
duration = time_after64(now, start_time)
|
||||
? now - start_time : 0;
|
||||
uint64_t last = time_after64(now, stats->last)
|
||||
last = time_after64(now, stats->last)
|
||||
? now - stats->last : 0;
|
||||
|
||||
stats->max_duration = max(stats->max_duration, duration);
|
||||
@ -188,6 +192,8 @@ void bch_time_stats_update(struct time_stats *stats, uint64_t start_time)
|
||||
}
|
||||
|
||||
stats->last = now ?: 1;
|
||||
|
||||
spin_unlock(&stats->lock);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -15,28 +15,18 @@
|
||||
|
||||
struct closure;
|
||||
|
||||
#ifdef CONFIG_BCACHE_EDEBUG
|
||||
#ifdef CONFIG_BCACHE_DEBUG
|
||||
|
||||
#define atomic_dec_bug(v) BUG_ON(atomic_dec_return(v) < 0)
|
||||
#define atomic_inc_bug(v, i) BUG_ON(atomic_inc_return(v) <= i)
|
||||
|
||||
#else /* EDEBUG */
|
||||
#else /* DEBUG */
|
||||
|
||||
#define atomic_dec_bug(v) atomic_dec(v)
|
||||
#define atomic_inc_bug(v, i) atomic_inc(v)
|
||||
|
||||
#endif
|
||||
|
||||
#define BITMASK(name, type, field, offset, size) \
|
||||
static inline uint64_t name(const type *k) \
|
||||
{ return (k->field >> offset) & ~(((uint64_t) ~0) << size); } \
|
||||
\
|
||||
static inline void SET_##name(type *k, uint64_t v) \
|
||||
{ \
|
||||
k->field &= ~(~((uint64_t) ~0 << size) << offset); \
|
||||
k->field |= v << offset; \
|
||||
}
|
||||
|
||||
#define DECLARE_HEAP(type, name) \
|
||||
struct { \
|
||||
size_t size, used; \
|
||||
@ -388,6 +378,7 @@ ssize_t bch_snprint_string_list(char *buf, size_t size, const char * const list[
|
||||
ssize_t bch_read_string_list(const char *buf, const char * const list[]);
|
||||
|
||||
struct time_stats {
|
||||
spinlock_t lock;
|
||||
/*
|
||||
* all fields are in nanoseconds, averages are ewmas stored left shifted
|
||||
* by 8
|
||||
|
@ -11,18 +11,11 @@
|
||||
#include "debug.h"
|
||||
#include "writeback.h"
|
||||
|
||||
#include <linux/delay.h>
|
||||
#include <linux/freezer.h>
|
||||
#include <linux/kthread.h>
|
||||
#include <trace/events/bcache.h>
|
||||
|
||||
static struct workqueue_struct *dirty_wq;
|
||||
|
||||
static void read_dirty(struct closure *);
|
||||
|
||||
struct dirty_io {
|
||||
struct closure cl;
|
||||
struct cached_dev *dc;
|
||||
struct bio bio;
|
||||
};
|
||||
|
||||
/* Rate limiting */
|
||||
|
||||
static void __update_writeback_rate(struct cached_dev *dc)
|
||||
@ -72,9 +65,6 @@ out:
|
||||
dc->writeback_rate_derivative = derivative;
|
||||
dc->writeback_rate_change = change;
|
||||
dc->writeback_rate_target = target;
|
||||
|
||||
schedule_delayed_work(&dc->writeback_rate_update,
|
||||
dc->writeback_rate_update_seconds * HZ);
|
||||
}
|
||||
|
||||
static void update_writeback_rate(struct work_struct *work)
|
||||
@ -90,13 +80,16 @@ static void update_writeback_rate(struct work_struct *work)
|
||||
__update_writeback_rate(dc);
|
||||
|
||||
up_read(&dc->writeback_lock);
|
||||
|
||||
schedule_delayed_work(&dc->writeback_rate_update,
|
||||
dc->writeback_rate_update_seconds * HZ);
|
||||
}
|
||||
|
||||
static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
|
||||
{
|
||||
uint64_t ret;
|
||||
|
||||
if (atomic_read(&dc->disk.detaching) ||
|
||||
if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
|
||||
!dc->writeback_percent)
|
||||
return 0;
|
||||
|
||||
@ -105,37 +98,11 @@ static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
|
||||
return min_t(uint64_t, ret, HZ);
|
||||
}
|
||||
|
||||
/* Background writeback */
|
||||
|
||||
static bool dirty_pred(struct keybuf *buf, struct bkey *k)
|
||||
{
|
||||
return KEY_DIRTY(k);
|
||||
}
|
||||
|
||||
static bool dirty_full_stripe_pred(struct keybuf *buf, struct bkey *k)
|
||||
{
|
||||
uint64_t stripe;
|
||||
unsigned nr_sectors = KEY_SIZE(k);
|
||||
struct cached_dev *dc = container_of(buf, struct cached_dev,
|
||||
writeback_keys);
|
||||
unsigned stripe_size = 1 << dc->disk.stripe_size_bits;
|
||||
|
||||
if (!KEY_DIRTY(k))
|
||||
return false;
|
||||
|
||||
stripe = KEY_START(k) >> dc->disk.stripe_size_bits;
|
||||
while (1) {
|
||||
if (atomic_read(dc->disk.stripe_sectors_dirty + stripe) !=
|
||||
stripe_size)
|
||||
return false;
|
||||
|
||||
if (nr_sectors <= stripe_size)
|
||||
return true;
|
||||
|
||||
nr_sectors -= stripe_size;
|
||||
stripe++;
|
||||
}
|
||||
}
|
||||
struct dirty_io {
|
||||
struct closure cl;
|
||||
struct cached_dev *dc;
|
||||
struct bio bio;
|
||||
};
|
||||
|
||||
static void dirty_init(struct keybuf_key *w)
|
||||
{
|
||||
@ -153,131 +120,6 @@ static void dirty_init(struct keybuf_key *w)
|
||||
bch_bio_map(bio, NULL);
|
||||
}
|
||||
|
||||
static void refill_dirty(struct closure *cl)
|
||||
{
|
||||
struct cached_dev *dc = container_of(cl, struct cached_dev,
|
||||
writeback.cl);
|
||||
struct keybuf *buf = &dc->writeback_keys;
|
||||
bool searched_from_start = false;
|
||||
struct bkey end = MAX_KEY;
|
||||
SET_KEY_INODE(&end, dc->disk.id);
|
||||
|
||||
if (!atomic_read(&dc->disk.detaching) &&
|
||||
!dc->writeback_running)
|
||||
closure_return(cl);
|
||||
|
||||
down_write(&dc->writeback_lock);
|
||||
|
||||
if (!atomic_read(&dc->has_dirty)) {
|
||||
SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
|
||||
bch_write_bdev_super(dc, NULL);
|
||||
|
||||
up_write(&dc->writeback_lock);
|
||||
closure_return(cl);
|
||||
}
|
||||
|
||||
if (bkey_cmp(&buf->last_scanned, &end) >= 0) {
|
||||
buf->last_scanned = KEY(dc->disk.id, 0, 0);
|
||||
searched_from_start = true;
|
||||
}
|
||||
|
||||
if (dc->partial_stripes_expensive) {
|
||||
uint64_t i;
|
||||
|
||||
for (i = 0; i < dc->disk.nr_stripes; i++)
|
||||
if (atomic_read(dc->disk.stripe_sectors_dirty + i) ==
|
||||
1 << dc->disk.stripe_size_bits)
|
||||
goto full_stripes;
|
||||
|
||||
goto normal_refill;
|
||||
full_stripes:
|
||||
bch_refill_keybuf(dc->disk.c, buf, &end,
|
||||
dirty_full_stripe_pred);
|
||||
} else {
|
||||
normal_refill:
|
||||
bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
|
||||
}
|
||||
|
||||
if (bkey_cmp(&buf->last_scanned, &end) >= 0 && searched_from_start) {
|
||||
/* Searched the entire btree - delay awhile */
|
||||
|
||||
if (RB_EMPTY_ROOT(&buf->keys)) {
|
||||
atomic_set(&dc->has_dirty, 0);
|
||||
cached_dev_put(dc);
|
||||
}
|
||||
|
||||
if (!atomic_read(&dc->disk.detaching))
|
||||
closure_delay(&dc->writeback, dc->writeback_delay * HZ);
|
||||
}
|
||||
|
||||
up_write(&dc->writeback_lock);
|
||||
|
||||
bch_ratelimit_reset(&dc->writeback_rate);
|
||||
|
||||
/* Punt to workqueue only so we don't recurse and blow the stack */
|
||||
continue_at(cl, read_dirty, dirty_wq);
|
||||
}
|
||||
|
||||
void bch_writeback_queue(struct cached_dev *dc)
|
||||
{
|
||||
if (closure_trylock(&dc->writeback.cl, &dc->disk.cl)) {
|
||||
if (!atomic_read(&dc->disk.detaching))
|
||||
closure_delay(&dc->writeback, dc->writeback_delay * HZ);
|
||||
|
||||
continue_at(&dc->writeback.cl, refill_dirty, dirty_wq);
|
||||
}
|
||||
}
|
||||
|
||||
void bch_writeback_add(struct cached_dev *dc)
|
||||
{
|
||||
if (!atomic_read(&dc->has_dirty) &&
|
||||
!atomic_xchg(&dc->has_dirty, 1)) {
|
||||
atomic_inc(&dc->count);
|
||||
|
||||
if (BDEV_STATE(&dc->sb) != BDEV_STATE_DIRTY) {
|
||||
SET_BDEV_STATE(&dc->sb, BDEV_STATE_DIRTY);
|
||||
/* XXX: should do this synchronously */
|
||||
bch_write_bdev_super(dc, NULL);
|
||||
}
|
||||
|
||||
bch_writeback_queue(dc);
|
||||
|
||||
if (dc->writeback_percent)
|
||||
schedule_delayed_work(&dc->writeback_rate_update,
|
||||
dc->writeback_rate_update_seconds * HZ);
|
||||
}
|
||||
}
|
||||
|
||||
void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
|
||||
uint64_t offset, int nr_sectors)
|
||||
{
|
||||
struct bcache_device *d = c->devices[inode];
|
||||
unsigned stripe_size, stripe_offset;
|
||||
uint64_t stripe;
|
||||
|
||||
if (!d)
|
||||
return;
|
||||
|
||||
stripe_size = 1 << d->stripe_size_bits;
|
||||
stripe = offset >> d->stripe_size_bits;
|
||||
stripe_offset = offset & (stripe_size - 1);
|
||||
|
||||
while (nr_sectors) {
|
||||
int s = min_t(unsigned, abs(nr_sectors),
|
||||
stripe_size - stripe_offset);
|
||||
|
||||
if (nr_sectors < 0)
|
||||
s = -s;
|
||||
|
||||
atomic_add(s, d->stripe_sectors_dirty + stripe);
|
||||
nr_sectors -= s;
|
||||
stripe_offset = 0;
|
||||
stripe++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Background writeback - IO loop */
|
||||
|
||||
static void dirty_io_destructor(struct closure *cl)
|
||||
{
|
||||
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
|
||||
@ -297,26 +139,25 @@ static void write_dirty_finish(struct closure *cl)
|
||||
|
||||
/* This is kind of a dumb way of signalling errors. */
|
||||
if (KEY_DIRTY(&w->key)) {
|
||||
int ret;
|
||||
unsigned i;
|
||||
struct btree_op op;
|
||||
bch_btree_op_init_stack(&op);
|
||||
struct keylist keys;
|
||||
|
||||
op.type = BTREE_REPLACE;
|
||||
bkey_copy(&op.replace, &w->key);
|
||||
bch_keylist_init(&keys);
|
||||
|
||||
SET_KEY_DIRTY(&w->key, false);
|
||||
bch_keylist_add(&op.keys, &w->key);
|
||||
bkey_copy(keys.top, &w->key);
|
||||
SET_KEY_DIRTY(keys.top, false);
|
||||
bch_keylist_push(&keys);
|
||||
|
||||
for (i = 0; i < KEY_PTRS(&w->key); i++)
|
||||
atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);
|
||||
|
||||
bch_btree_insert(&op, dc->disk.c);
|
||||
closure_sync(&op.cl);
|
||||
ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);
|
||||
|
||||
if (op.insert_collision)
|
||||
if (ret)
|
||||
trace_bcache_writeback_collision(&w->key);
|
||||
|
||||
atomic_long_inc(op.insert_collision
|
||||
atomic_long_inc(ret
|
||||
? &dc->disk.c->writeback_keys_failed
|
||||
: &dc->disk.c->writeback_keys_done);
|
||||
}
|
||||
@ -374,30 +215,33 @@ static void read_dirty_submit(struct closure *cl)
|
||||
continue_at(cl, write_dirty, system_wq);
|
||||
}
|
||||
|
||||
static void read_dirty(struct closure *cl)
|
||||
static void read_dirty(struct cached_dev *dc)
|
||||
{
|
||||
struct cached_dev *dc = container_of(cl, struct cached_dev,
|
||||
writeback.cl);
|
||||
unsigned delay = writeback_delay(dc, 0);
|
||||
unsigned delay = 0;
|
||||
struct keybuf_key *w;
|
||||
struct dirty_io *io;
|
||||
struct closure cl;
|
||||
|
||||
closure_init_stack(&cl);
|
||||
|
||||
/*
|
||||
* XXX: if we error, background writeback just spins. Should use some
|
||||
* mempools.
|
||||
*/
|
||||
|
||||
while (1) {
|
||||
while (!kthread_should_stop()) {
|
||||
try_to_freeze();
|
||||
|
||||
w = bch_keybuf_next(&dc->writeback_keys);
|
||||
if (!w)
|
||||
break;
|
||||
|
||||
BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));
|
||||
|
||||
if (delay > 0 &&
|
||||
(KEY_START(&w->key) != dc->last_read ||
|
||||
jiffies_to_msecs(delay) > 50))
|
||||
delay = schedule_timeout_uninterruptible(delay);
|
||||
if (KEY_START(&w->key) != dc->last_read ||
|
||||
jiffies_to_msecs(delay) > 50)
|
||||
while (!kthread_should_stop() && delay)
|
||||
delay = schedule_timeout_interruptible(delay);
|
||||
|
||||
dc->last_read = KEY_OFFSET(&w->key);
|
||||
|
||||
@ -423,7 +267,7 @@ static void read_dirty(struct closure *cl)
|
||||
trace_bcache_writeback(&w->key);
|
||||
|
||||
down(&dc->in_flight);
|
||||
closure_call(&io->cl, read_dirty_submit, NULL, cl);
|
||||
closure_call(&io->cl, read_dirty_submit, NULL, &cl);
|
||||
|
||||
delay = writeback_delay(dc, KEY_SIZE(&w->key));
|
||||
}
|
||||
@ -439,52 +283,205 @@ err:
|
||||
* Wait for outstanding writeback IOs to finish (and keybuf slots to be
|
||||
* freed) before refilling again
|
||||
*/
|
||||
continue_at(cl, refill_dirty, dirty_wq);
|
||||
closure_sync(&cl);
|
||||
}
|
||||
|
||||
/* Init */
|
||||
/* Scan for dirty data */
|
||||
|
||||
static int bch_btree_sectors_dirty_init(struct btree *b, struct btree_op *op,
|
||||
struct cached_dev *dc)
|
||||
void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
|
||||
uint64_t offset, int nr_sectors)
|
||||
{
|
||||
struct bkey *k;
|
||||
struct btree_iter iter;
|
||||
struct bcache_device *d = c->devices[inode];
|
||||
unsigned stripe_offset, stripe, sectors_dirty;
|
||||
|
||||
bch_btree_iter_init(b, &iter, &KEY(dc->disk.id, 0, 0));
|
||||
while ((k = bch_btree_iter_next_filter(&iter, b, bch_ptr_bad)))
|
||||
if (!b->level) {
|
||||
if (KEY_INODE(k) > dc->disk.id)
|
||||
break;
|
||||
if (!d)
|
||||
return;
|
||||
|
||||
if (KEY_DIRTY(k))
|
||||
bcache_dev_sectors_dirty_add(b->c, dc->disk.id,
|
||||
KEY_START(k),
|
||||
KEY_SIZE(k));
|
||||
} else {
|
||||
btree(sectors_dirty_init, k, b, op, dc);
|
||||
if (KEY_INODE(k) > dc->disk.id)
|
||||
break;
|
||||
stripe = offset_to_stripe(d, offset);
|
||||
stripe_offset = offset & (d->stripe_size - 1);
|
||||
|
||||
cond_resched();
|
||||
while (nr_sectors) {
|
||||
int s = min_t(unsigned, abs(nr_sectors),
|
||||
d->stripe_size - stripe_offset);
|
||||
|
||||
if (nr_sectors < 0)
|
||||
s = -s;
|
||||
|
||||
if (stripe >= d->nr_stripes)
|
||||
return;
|
||||
|
||||
sectors_dirty = atomic_add_return(s,
|
||||
d->stripe_sectors_dirty + stripe);
|
||||
if (sectors_dirty == d->stripe_size)
|
||||
set_bit(stripe, d->full_dirty_stripes);
|
||||
else
|
||||
clear_bit(stripe, d->full_dirty_stripes);
|
||||
|
||||
nr_sectors -= s;
|
||||
stripe_offset = 0;
|
||||
stripe++;
|
||||
}
|
||||
}
|
||||
|
||||
static bool dirty_pred(struct keybuf *buf, struct bkey *k)
|
||||
{
|
||||
return KEY_DIRTY(k);
|
||||
}
|
||||
|
||||
static void refill_full_stripes(struct cached_dev *dc)
|
||||
{
|
||||
struct keybuf *buf = &dc->writeback_keys;
|
||||
unsigned start_stripe, stripe, next_stripe;
|
||||
bool wrapped = false;
|
||||
|
||||
stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));
|
||||
|
||||
if (stripe >= dc->disk.nr_stripes)
|
||||
stripe = 0;
|
||||
|
||||
start_stripe = stripe;
|
||||
|
||||
while (1) {
|
||||
stripe = find_next_bit(dc->disk.full_dirty_stripes,
|
||||
dc->disk.nr_stripes, stripe);
|
||||
|
||||
if (stripe == dc->disk.nr_stripes)
|
||||
goto next;
|
||||
|
||||
next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
|
||||
dc->disk.nr_stripes, stripe);
|
||||
|
||||
buf->last_scanned = KEY(dc->disk.id,
|
||||
stripe * dc->disk.stripe_size, 0);
|
||||
|
||||
bch_refill_keybuf(dc->disk.c, buf,
|
||||
&KEY(dc->disk.id,
|
||||
next_stripe * dc->disk.stripe_size, 0),
|
||||
dirty_pred);
|
||||
|
||||
if (array_freelist_empty(&buf->freelist))
|
||||
return;
|
||||
|
||||
stripe = next_stripe;
|
||||
next:
|
||||
if (wrapped && stripe > start_stripe)
|
||||
return;
|
||||
|
||||
if (stripe == dc->disk.nr_stripes) {
|
||||
stripe = 0;
|
||||
wrapped = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool refill_dirty(struct cached_dev *dc)
|
||||
{
|
||||
struct keybuf *buf = &dc->writeback_keys;
|
||||
struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
|
||||
bool searched_from_start = false;
|
||||
|
||||
if (dc->partial_stripes_expensive) {
|
||||
refill_full_stripes(dc);
|
||||
if (array_freelist_empty(&buf->freelist))
|
||||
return false;
|
||||
}
|
||||
|
||||
if (bkey_cmp(&buf->last_scanned, &end) >= 0) {
|
||||
buf->last_scanned = KEY(dc->disk.id, 0, 0);
|
||||
searched_from_start = true;
|
||||
}
|
||||
|
||||
bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
|
||||
|
||||
return bkey_cmp(&buf->last_scanned, &end) >= 0 && searched_from_start;
|
||||
}
|
||||
|
||||
static int bch_writeback_thread(void *arg)
|
||||
{
|
||||
struct cached_dev *dc = arg;
|
||||
bool searched_full_index;
|
||||
|
||||
while (!kthread_should_stop()) {
|
||||
down_write(&dc->writeback_lock);
|
||||
if (!atomic_read(&dc->has_dirty) ||
|
||||
(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
|
||||
!dc->writeback_running)) {
|
||||
up_write(&dc->writeback_lock);
|
||||
set_current_state(TASK_INTERRUPTIBLE);
|
||||
|
||||
if (kthread_should_stop())
|
||||
return 0;
|
||||
|
||||
try_to_freeze();
|
||||
schedule();
|
||||
continue;
|
||||
}
|
||||
|
||||
searched_full_index = refill_dirty(dc);
|
||||
|
||||
if (searched_full_index &&
|
||||
RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
|
||||
atomic_set(&dc->has_dirty, 0);
|
||||
cached_dev_put(dc);
|
||||
SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
|
||||
bch_write_bdev_super(dc, NULL);
|
||||
}
|
||||
|
||||
up_write(&dc->writeback_lock);
|
||||
|
||||
bch_ratelimit_reset(&dc->writeback_rate);
|
||||
read_dirty(dc);
|
||||
|
||||
if (searched_full_index) {
|
||||
unsigned delay = dc->writeback_delay * HZ;
|
||||
|
||||
while (delay &&
|
||||
!kthread_should_stop() &&
|
||||
!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
|
||||
delay = schedule_timeout_interruptible(delay);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void bch_sectors_dirty_init(struct cached_dev *dc)
|
||||
{
|
||||
struct btree_op op;
|
||||
/* Init */
|
||||
|
||||
bch_btree_op_init_stack(&op);
|
||||
btree_root(sectors_dirty_init, dc->disk.c, &op, dc);
|
||||
struct sectors_dirty_init {
|
||||
struct btree_op op;
|
||||
unsigned inode;
|
||||
};
|
||||
|
||||
static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
|
||||
struct bkey *k)
|
||||
{
|
||||
struct sectors_dirty_init *op = container_of(_op,
|
||||
struct sectors_dirty_init, op);
|
||||
if (KEY_INODE(k) > op->inode)
|
||||
return MAP_DONE;
|
||||
|
||||
if (KEY_DIRTY(k))
|
||||
bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
|
||||
KEY_START(k), KEY_SIZE(k));
|
||||
|
||||
return MAP_CONTINUE;
|
||||
}
|
||||
|
||||
void bch_cached_dev_writeback_init(struct cached_dev *dc)
|
||||
void bch_sectors_dirty_init(struct cached_dev *dc)
|
||||
{
|
||||
struct sectors_dirty_init op;
|
||||
|
||||
bch_btree_op_init(&op.op, -1);
|
||||
op.inode = dc->disk.id;
|
||||
|
||||
bch_btree_map_keys(&op.op, dc->disk.c, &KEY(op.inode, 0, 0),
|
||||
sectors_dirty_init_fn, 0);
|
||||
}
|
||||
|
||||
int bch_cached_dev_writeback_init(struct cached_dev *dc)
|
||||
{
|
||||
sema_init(&dc->in_flight, 64);
|
||||
closure_init_unlocked(&dc->writeback);
|
||||
init_rwsem(&dc->writeback_lock);
|
||||
|
||||
bch_keybuf_init(&dc->writeback_keys);
|
||||
|
||||
dc->writeback_metadata = true;
|
||||
@ -498,22 +495,16 @@ void bch_cached_dev_writeback_init(struct cached_dev *dc)
|
||||
dc->writeback_rate_p_term_inverse = 64;
|
||||
dc->writeback_rate_d_smooth = 8;
|
||||
|
||||
dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
|
||||
"bcache_writeback");
|
||||
if (IS_ERR(dc->writeback_thread))
|
||||
return PTR_ERR(dc->writeback_thread);
|
||||
|
||||
set_task_state(dc->writeback_thread, TASK_INTERRUPTIBLE);
|
||||
|
||||
INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
|
||||
schedule_delayed_work(&dc->writeback_rate_update,
|
||||
dc->writeback_rate_update_seconds * HZ);
|
||||
}
|
||||
|
||||
void bch_writeback_exit(void)
|
||||
{
|
||||
if (dirty_wq)
|
||||
destroy_workqueue(dirty_wq);
|
||||
}
|
||||
|
||||
int __init bch_writeback_init(void)
|
||||
{
|
||||
dirty_wq = create_workqueue("bcache_writeback");
|
||||
if (!dirty_wq)
|
||||
return -ENOMEM;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -14,20 +14,27 @@ static inline uint64_t bcache_dev_sectors_dirty(struct bcache_device *d)
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline bool bcache_dev_stripe_dirty(struct bcache_device *d,
|
||||
static inline unsigned offset_to_stripe(struct bcache_device *d,
|
||||
uint64_t offset)
|
||||
{
|
||||
do_div(offset, d->stripe_size);
|
||||
return offset;
|
||||
}
|
||||
|
||||
static inline bool bcache_dev_stripe_dirty(struct cached_dev *dc,
|
||||
uint64_t offset,
|
||||
unsigned nr_sectors)
|
||||
{
|
||||
uint64_t stripe = offset >> d->stripe_size_bits;
|
||||
unsigned stripe = offset_to_stripe(&dc->disk, offset);
|
||||
|
||||
while (1) {
|
||||
if (atomic_read(d->stripe_sectors_dirty + stripe))
|
||||
if (atomic_read(dc->disk.stripe_sectors_dirty + stripe))
|
||||
return true;
|
||||
|
||||
if (nr_sectors <= 1 << d->stripe_size_bits)
|
||||
if (nr_sectors <= dc->disk.stripe_size)
|
||||
return false;
|
||||
|
||||
nr_sectors -= 1 << d->stripe_size_bits;
|
||||
nr_sectors -= dc->disk.stripe_size;
|
||||
stripe++;
|
||||
}
|
||||
}
|
||||
@ -38,12 +45,12 @@ static inline bool should_writeback(struct cached_dev *dc, struct bio *bio,
|
||||
unsigned in_use = dc->disk.c->gc_stats.in_use;
|
||||
|
||||
if (cache_mode != CACHE_MODE_WRITEBACK ||
|
||||
atomic_read(&dc->disk.detaching) ||
|
||||
test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
|
||||
in_use > CUTOFF_WRITEBACK_SYNC)
|
||||
return false;
|
||||
|
||||
if (dc->partial_stripes_expensive &&
|
||||
bcache_dev_stripe_dirty(&dc->disk, bio->bi_sector,
|
||||
bcache_dev_stripe_dirty(dc, bio->bi_sector,
|
||||
bio_sectors(bio)))
|
||||
return true;
|
||||
|
||||
@ -54,11 +61,30 @@ static inline bool should_writeback(struct cached_dev *dc, struct bio *bio,
|
||||
in_use <= CUTOFF_WRITEBACK;
|
||||
}
|
||||
|
||||
static inline void bch_writeback_queue(struct cached_dev *dc)
|
||||
{
|
||||
wake_up_process(dc->writeback_thread);
|
||||
}
|
||||
|
||||
static inline void bch_writeback_add(struct cached_dev *dc)
|
||||
{
|
||||
if (!atomic_read(&dc->has_dirty) &&
|
||||
!atomic_xchg(&dc->has_dirty, 1)) {
|
||||
atomic_inc(&dc->count);
|
||||
|
||||
if (BDEV_STATE(&dc->sb) != BDEV_STATE_DIRTY) {
|
||||
SET_BDEV_STATE(&dc->sb, BDEV_STATE_DIRTY);
|
||||
/* XXX: should do this synchronously */
|
||||
bch_write_bdev_super(dc, NULL);
|
||||
}
|
||||
|
||||
bch_writeback_queue(dc);
|
||||
}
|
||||
}
|
||||
|
||||
void bcache_dev_sectors_dirty_add(struct cache_set *, unsigned, uint64_t, int);
|
||||
void bch_writeback_queue(struct cached_dev *);
|
||||
void bch_writeback_add(struct cached_dev *);
|
||||
|
||||
void bch_sectors_dirty_init(struct cached_dev *dc);
|
||||
void bch_cached_dev_writeback_init(struct cached_dev *);
|
||||
int bch_cached_dev_writeback_init(struct cached_dev *);
|
||||
|
||||
#endif
|
||||
|
@ -2978,12 +2978,12 @@ static int dasd_alloc_queue(struct dasd_block *block)
|
||||
|
||||
elevator_exit(block->request_queue->elevator);
|
||||
block->request_queue->elevator = NULL;
|
||||
mutex_lock(&block->request_queue->sysfs_lock);
|
||||
rc = elevator_init(block->request_queue, "deadline");
|
||||
if (rc) {
|
||||
if (rc)
|
||||
blk_cleanup_queue(block->request_queue);
|
||||
return rc;
|
||||
}
|
||||
return 0;
|
||||
mutex_unlock(&block->request_queue->sysfs_lock);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -6,11 +6,9 @@
|
||||
|
||||
#include <linux/tracepoint.h>
|
||||
|
||||
struct search;
|
||||
|
||||
DECLARE_EVENT_CLASS(bcache_request,
|
||||
TP_PROTO(struct search *s, struct bio *bio),
|
||||
TP_ARGS(s, bio),
|
||||
TP_PROTO(struct bcache_device *d, struct bio *bio),
|
||||
TP_ARGS(d, bio),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(dev_t, dev )
|
||||
@ -24,8 +22,8 @@ DECLARE_EVENT_CLASS(bcache_request,
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->dev = bio->bi_bdev->bd_dev;
|
||||
__entry->orig_major = s->d->disk->major;
|
||||
__entry->orig_minor = s->d->disk->first_minor;
|
||||
__entry->orig_major = d->disk->major;
|
||||
__entry->orig_minor = d->disk->first_minor;
|
||||
__entry->sector = bio->bi_sector;
|
||||
__entry->orig_sector = bio->bi_sector - 16;
|
||||
__entry->nr_sector = bio->bi_size >> 9;
|
||||
@ -79,13 +77,13 @@ DECLARE_EVENT_CLASS(btree_node,
|
||||
/* request.c */
|
||||
|
||||
DEFINE_EVENT(bcache_request, bcache_request_start,
|
||||
TP_PROTO(struct search *s, struct bio *bio),
|
||||
TP_ARGS(s, bio)
|
||||
TP_PROTO(struct bcache_device *d, struct bio *bio),
|
||||
TP_ARGS(d, bio)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(bcache_request, bcache_request_end,
|
||||
TP_PROTO(struct search *s, struct bio *bio),
|
||||
TP_ARGS(s, bio)
|
||||
TP_PROTO(struct bcache_device *d, struct bio *bio),
|
||||
TP_ARGS(d, bio)
|
||||
);
|
||||
|
||||
DECLARE_EVENT_CLASS(bcache_bio,
|
||||
@ -370,6 +368,35 @@ DEFINE_EVENT(btree_node, bcache_btree_set_root,
|
||||
TP_ARGS(b)
|
||||
);
|
||||
|
||||
TRACE_EVENT(bcache_keyscan,
|
||||
TP_PROTO(unsigned nr_found,
|
||||
unsigned start_inode, uint64_t start_offset,
|
||||
unsigned end_inode, uint64_t end_offset),
|
||||
TP_ARGS(nr_found,
|
||||
start_inode, start_offset,
|
||||
end_inode, end_offset),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(__u32, nr_found )
|
||||
__field(__u32, start_inode )
|
||||
__field(__u64, start_offset )
|
||||
__field(__u32, end_inode )
|
||||
__field(__u64, end_offset )
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->nr_found = nr_found;
|
||||
__entry->start_inode = start_inode;
|
||||
__entry->start_offset = start_offset;
|
||||
__entry->end_inode = end_inode;
|
||||
__entry->end_offset = end_offset;
|
||||
),
|
||||
|
||||
TP_printk("found %u keys from %u:%llu to %u:%llu", __entry->nr_found,
|
||||
__entry->start_inode, __entry->start_offset,
|
||||
__entry->end_inode, __entry->end_offset)
|
||||
);
|
||||
|
||||
/* Allocator */
|
||||
|
||||
TRACE_EVENT(bcache_alloc_invalidate,
|
||||
|
373
include/uapi/linux/bcache.h
Normal file
373
include/uapi/linux/bcache.h
Normal file
@ -0,0 +1,373 @@
|
||||
#ifndef _LINUX_BCACHE_H
|
||||
#define _LINUX_BCACHE_H
|
||||
|
||||
/*
|
||||
* Bcache on disk data structures
|
||||
*/
|
||||
|
||||
#include <asm/types.h>
|
||||
|
||||
#define BITMASK(name, type, field, offset, size) \
|
||||
static inline __u64 name(const type *k) \
|
||||
{ return (k->field >> offset) & ~(~0ULL << size); } \
|
||||
\
|
||||
static inline void SET_##name(type *k, __u64 v) \
|
||||
{ \
|
||||
k->field &= ~(~(~0ULL << size) << offset); \
|
||||
k->field |= (v & ~(~0ULL << size)) << offset; \
|
||||
}
|
||||
|
||||
/* Btree keys - all units are in sectors */
|
||||
|
||||
struct bkey {
|
||||
__u64 high;
|
||||
__u64 low;
|
||||
__u64 ptr[];
|
||||
};
|
||||
|
||||
#define KEY_FIELD(name, field, offset, size) \
|
||||
BITMASK(name, struct bkey, field, offset, size)
|
||||
|
||||
#define PTR_FIELD(name, offset, size) \
|
||||
static inline __u64 name(const struct bkey *k, unsigned i) \
|
||||
{ return (k->ptr[i] >> offset) & ~(~0ULL << size); } \
|
||||
\
|
||||
static inline void SET_##name(struct bkey *k, unsigned i, __u64 v) \
|
||||
{ \
|
||||
k->ptr[i] &= ~(~(~0ULL << size) << offset); \
|
||||
k->ptr[i] |= (v & ~(~0ULL << size)) << offset; \
|
||||
}
|
||||
|
||||
#define KEY_SIZE_BITS 16
|
||||
|
||||
KEY_FIELD(KEY_PTRS, high, 60, 3)
|
||||
KEY_FIELD(HEADER_SIZE, high, 58, 2)
|
||||
KEY_FIELD(KEY_CSUM, high, 56, 2)
|
||||
KEY_FIELD(KEY_PINNED, high, 55, 1)
|
||||
KEY_FIELD(KEY_DIRTY, high, 36, 1)
|
||||
|
||||
KEY_FIELD(KEY_SIZE, high, 20, KEY_SIZE_BITS)
|
||||
KEY_FIELD(KEY_INODE, high, 0, 20)
|
||||
|
||||
/* Next time I change the on disk format, KEY_OFFSET() won't be 64 bits */
|
||||
|
||||
static inline __u64 KEY_OFFSET(const struct bkey *k)
|
||||
{
|
||||
return k->low;
|
||||
}
|
||||
|
||||
static inline void SET_KEY_OFFSET(struct bkey *k, __u64 v)
|
||||
{
|
||||
k->low = v;
|
||||
}
|
||||
|
||||
/*
|
||||
* The high bit being set is a relic from when we used it to do binary
|
||||
* searches - it told you where a key started. It's not used anymore,
|
||||
* and can probably be safely dropped.
|
||||
*/
|
||||
#define KEY(inode, offset, size) \
|
||||
((struct bkey) { \
|
||||
.high = (1ULL << 63) | ((__u64) (size) << 20) | (inode), \
|
||||
.low = (offset) \
|
||||
})
|
||||
|
||||
#define ZERO_KEY KEY(0, 0, 0)
|
||||
|
||||
#define MAX_KEY_INODE (~(~0 << 20))
|
||||
#define MAX_KEY_OFFSET (~0ULL >> 1)
|
||||
#define MAX_KEY KEY(MAX_KEY_INODE, MAX_KEY_OFFSET, 0)
|
||||
|
||||
#define KEY_START(k) (KEY_OFFSET(k) - KEY_SIZE(k))
|
||||
#define START_KEY(k) KEY(KEY_INODE(k), KEY_START(k), 0)
|
||||
|
||||
#define PTR_DEV_BITS 12
|
||||
|
||||
PTR_FIELD(PTR_DEV, 51, PTR_DEV_BITS)
|
||||
PTR_FIELD(PTR_OFFSET, 8, 43)
|
||||
PTR_FIELD(PTR_GEN, 0, 8)
|
||||
|
||||
#define PTR_CHECK_DEV ((1 << PTR_DEV_BITS) - 1)
|
||||
|
||||
#define PTR(gen, offset, dev) \
|
||||
((((__u64) dev) << 51) | ((__u64) offset) << 8 | gen)
|
||||
|
||||
/* Bkey utility code */
|
||||
|
||||
static inline unsigned long bkey_u64s(const struct bkey *k)
|
||||
{
|
||||
return (sizeof(struct bkey) / sizeof(__u64)) + KEY_PTRS(k);
|
||||
}
|
||||
|
||||
static inline unsigned long bkey_bytes(const struct bkey *k)
|
||||
{
|
||||
return bkey_u64s(k) * sizeof(__u64);
|
||||
}
|
||||
|
||||
#define bkey_copy(_dest, _src) memcpy(_dest, _src, bkey_bytes(_src))
|
||||
|
||||
static inline void bkey_copy_key(struct bkey *dest, const struct bkey *src)
|
||||
{
|
||||
SET_KEY_INODE(dest, KEY_INODE(src));
|
||||
SET_KEY_OFFSET(dest, KEY_OFFSET(src));
|
||||
}
|
||||
|
||||
static inline struct bkey *bkey_next(const struct bkey *k)
|
||||
{
|
||||
__u64 *d = (void *) k;
|
||||
return (struct bkey *) (d + bkey_u64s(k));
|
||||
}
|
||||
|
||||
static inline struct bkey *bkey_last(const struct bkey *k, unsigned nr_keys)
|
||||
{
|
||||
__u64 *d = (void *) k;
|
||||
return (struct bkey *) (d + nr_keys);
|
||||
}
|
||||
/* Enough for a key with 6 pointers */
|
||||
#define BKEY_PAD 8
|
||||
|
||||
#define BKEY_PADDED(key) \
|
||||
union { struct bkey key; __u64 key ## _pad[BKEY_PAD]; }
|
||||
|
||||
/* Superblock */
|
||||
|
||||
/* Version 0: Cache device
|
||||
* Version 1: Backing device
|
||||
* Version 2: Seed pointer into btree node checksum
|
||||
* Version 3: Cache device with new UUID format
|
||||
* Version 4: Backing device with data offset
|
||||
*/
|
||||
#define BCACHE_SB_VERSION_CDEV 0
|
||||
#define BCACHE_SB_VERSION_BDEV 1
|
||||
#define BCACHE_SB_VERSION_CDEV_WITH_UUID 3
|
||||
#define BCACHE_SB_VERSION_BDEV_WITH_OFFSET 4
|
||||
#define BCACHE_SB_MAX_VERSION 4
|
||||
|
||||
#define SB_SECTOR 8
|
||||
#define SB_SIZE 4096
|
||||
#define SB_LABEL_SIZE 32
|
||||
#define SB_JOURNAL_BUCKETS 256U
|
||||
/* SB_JOURNAL_BUCKETS must be divisible by BITS_PER_LONG */
|
||||
#define MAX_CACHES_PER_SET 8
|
||||
|
||||
#define BDEV_DATA_START_DEFAULT 16 /* sectors */
|
||||
|
||||
struct cache_sb {
|
||||
__u64 csum;
|
||||
__u64 offset; /* sector where this sb was written */
|
||||
__u64 version;
|
||||
|
||||
__u8 magic[16];
|
||||
|
||||
__u8 uuid[16];
|
||||
union {
|
||||
__u8 set_uuid[16];
|
||||
__u64 set_magic;
|
||||
};
|
||||
__u8 label[SB_LABEL_SIZE];
|
||||
|
||||
__u64 flags;
|
||||
__u64 seq;
|
||||
__u64 pad[8];
|
||||
|
||||
union {
|
||||
struct {
|
||||
/* Cache devices */
|
||||
__u64 nbuckets; /* device size */
|
||||
|
||||
__u16 block_size; /* sectors */
|
||||
__u16 bucket_size; /* sectors */
|
||||
|
||||
__u16 nr_in_set;
|
||||
__u16 nr_this_dev;
|
||||
};
|
||||
struct {
|
||||
/* Backing devices */
|
||||
__u64 data_offset;
|
||||
|
||||
/*
|
||||
* block_size from the cache device section is still used by
|
||||
* backing devices, so don't add anything here until we fix
|
||||
* things to not need it for backing devices anymore
|
||||
*/
|
||||
};
|
||||
};
|
||||
|
||||
__u32 last_mount; /* time_t */
|
||||
|
||||
__u16 first_bucket;
|
||||
union {
|
||||
__u16 njournal_buckets;
|
||||
__u16 keys;
|
||||
};
|
||||
__u64 d[SB_JOURNAL_BUCKETS]; /* journal buckets */
|
||||
};
|
||||
|
||||
static inline _Bool SB_IS_BDEV(const struct cache_sb *sb)
|
||||
{
|
||||
return sb->version == BCACHE_SB_VERSION_BDEV
|
||||
|| sb->version == BCACHE_SB_VERSION_BDEV_WITH_OFFSET;
|
||||
}
|
||||
|
||||
BITMASK(CACHE_SYNC, struct cache_sb, flags, 0, 1);
|
||||
BITMASK(CACHE_DISCARD, struct cache_sb, flags, 1, 1);
|
||||
BITMASK(CACHE_REPLACEMENT, struct cache_sb, flags, 2, 3);
|
||||
#define CACHE_REPLACEMENT_LRU 0U
|
||||
#define CACHE_REPLACEMENT_FIFO 1U
|
||||
#define CACHE_REPLACEMENT_RANDOM 2U
|
||||
|
||||
BITMASK(BDEV_CACHE_MODE, struct cache_sb, flags, 0, 4);
|
||||
#define CACHE_MODE_WRITETHROUGH 0U
|
||||
#define CACHE_MODE_WRITEBACK 1U
|
||||
#define CACHE_MODE_WRITEAROUND 2U
|
||||
#define CACHE_MODE_NONE 3U
|
||||
BITMASK(BDEV_STATE, struct cache_sb, flags, 61, 2);
|
||||
#define BDEV_STATE_NONE 0U
|
||||
#define BDEV_STATE_CLEAN 1U
|
||||
#define BDEV_STATE_DIRTY 2U
|
||||
#define BDEV_STATE_STALE 3U
|
||||
|
||||
/*
|
||||
* Magic numbers
|
||||
*
|
||||
* The various other data structures have their own magic numbers, which are
|
||||
* xored with the first part of the cache set's UUID
|
||||
*/
|
||||
|
||||
#define JSET_MAGIC 0x245235c1a3625032ULL
|
||||
#define PSET_MAGIC 0x6750e15f87337f91ULL
|
||||
#define BSET_MAGIC 0x90135c78b99e07f5ULL
|
||||
|
||||
static inline __u64 jset_magic(struct cache_sb *sb)
|
||||
{
|
||||
return sb->set_magic ^ JSET_MAGIC;
|
||||
}
|
||||
|
||||
static inline __u64 pset_magic(struct cache_sb *sb)
|
||||
{
|
||||
return sb->set_magic ^ PSET_MAGIC;
|
||||
}
|
||||
|
||||
static inline __u64 bset_magic(struct cache_sb *sb)
|
||||
{
|
||||
return sb->set_magic ^ BSET_MAGIC;
|
||||
}
|
||||
|
||||
/*
|
||||
* Journal
|
||||
*
|
||||
* On disk format for a journal entry:
|
||||
* seq is monotonically increasing; every journal entry has its own unique
|
||||
* sequence number.
|
||||
*
|
||||
* last_seq is the oldest journal entry that still has keys the btree hasn't
|
||||
* flushed to disk yet.
|
||||
*
|
||||
* version is for on disk format changes.
|
||||
*/
|
||||
|
||||
#define BCACHE_JSET_VERSION_UUIDv1 1
|
||||
#define BCACHE_JSET_VERSION_UUID 1 /* Always latest UUID format */
|
||||
#define BCACHE_JSET_VERSION 1
|
||||
|
||||
struct jset {
|
||||
__u64 csum;
|
||||
__u64 magic;
|
||||
__u64 seq;
|
||||
__u32 version;
|
||||
__u32 keys;
|
||||
|
||||
__u64 last_seq;
|
||||
|
||||
BKEY_PADDED(uuid_bucket);
|
||||
BKEY_PADDED(btree_root);
|
||||
__u16 btree_level;
|
||||
__u16 pad[3];
|
||||
|
||||
__u64 prio_bucket[MAX_CACHES_PER_SET];
|
||||
|
||||
union {
|
||||
struct bkey start[0];
|
||||
__u64 d[0];
|
||||
};
|
||||
};
|
||||
|
||||
/* Bucket prios/gens */
|
||||
|
||||
struct prio_set {
|
||||
__u64 csum;
|
||||
__u64 magic;
|
||||
__u64 seq;
|
||||
__u32 version;
|
||||
__u32 pad;
|
||||
|
||||
__u64 next_bucket;
|
||||
|
||||
struct bucket_disk {
|
||||
__u16 prio;
|
||||
__u8 gen;
|
||||
} __attribute((packed)) data[];
|
||||
};
|
||||
|
||||
/* UUIDS - per backing device/flash only volume metadata */
|
||||
|
||||
struct uuid_entry {
|
||||
union {
|
||||
struct {
|
||||
__u8 uuid[16];
|
||||
__u8 label[32];
|
||||
__u32 first_reg;
|
||||
__u32 last_reg;
|
||||
__u32 invalidated;
|
||||
|
||||
__u32 flags;
|
||||
/* Size of flash only volumes */
|
||||
__u64 sectors;
|
||||
};
|
||||
|
||||
__u8 pad[128];
|
||||
};
|
||||
};
|
||||
|
||||
BITMASK(UUID_FLASH_ONLY, struct uuid_entry, flags, 0, 1);
|
||||
|
||||
/* Btree nodes */
|
||||
|
||||
/* Version 1: Seed pointer into btree node checksum
|
||||
*/
|
||||
#define BCACHE_BSET_CSUM 1
|
||||
#define BCACHE_BSET_VERSION 1
|
||||
|
||||
/*
|
||||
* Btree nodes
|
||||
*
|
||||
* On disk a btree node is a list/log of these; within each set the keys are
|
||||
* sorted
|
||||
*/
|
||||
struct bset {
|
||||
__u64 csum;
|
||||
__u64 magic;
|
||||
__u64 seq;
|
||||
__u32 version;
|
||||
__u32 keys;
|
||||
|
||||
union {
|
||||
struct bkey start[0];
|
||||
__u64 d[0];
|
||||
};
|
||||
};
|
||||
|
||||
/* OBSOLETE */
|
||||
|
||||
/* UUIDS - per backing device/flash only volume metadata */
|
||||
|
||||
struct uuid_entry_v0 {
|
||||
__u8 uuid[16];
|
||||
__u8 label[32];
|
||||
__u32 first_reg;
|
||||
__u32 last_reg;
|
||||
__u32 invalidated;
|
||||
__u32 pad;
|
||||
};
|
||||
|
||||
#endif /* _LINUX_BCACHE_H */
|
Loading…
Reference in New Issue
Block a user