iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IB/mthca: prevent event queue overrun

Browse Source 
I am seeing EQ overruns in SDP stress tests: if the CQ completion
handler arms a CQ, this could generate more EQEs, so that EQ will
never get empty and consumer index will never get updated.

This is similiar to what we have with command interface:
		/*
		 * cmd_event() may add more commands.
		 * The card will think the queue has overflowed if
		 * we don't tell it we've been processing events.
		 */
However, for completion events, we *don't* want to update the consumer
index on each event. So, perform EQ doorbell coalescing: allocate EQs
with some spare EQEs, and update once we run out of them.

The value 0x80 was selected to avoid any performance impact.

Signed-off-by: Michael S. Tsirkin <mst@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
This commit is contained in:
Michael S. Tsirkin 2006-01-09 14:04:40 -08:00 committed by Roland Dreier
parent 6627fa662e
commit 92898522e3
1 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
drivers/infiniband/hw/mthca/mthca_eq.c
View File

@ -45,6 +45,7 @@
enum { enum {
MTHCA_NUM_ASYNC_EQE = 0x80, MTHCA_NUM_ASYNC_EQE = 0x80,
MTHCA_NUM_CMD_EQE = 0x80, MTHCA_NUM_CMD_EQE = 0x80,
MTHCA_NUM_SPARE_EQE = 0x80,
MTHCA_EQ_ENTRY_SIZE = 0x20 MTHCA_EQ_ENTRY_SIZE = 0x20
}; };
@ -278,10 +279,9 @@ static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)
struct mthca_eqe *eqe; struct mthca_eqe *eqe;
int disarm_cqn; int disarm_cqn;
int eqes_found = 0; int eqes_found = 0;
while ((eqe = next_eqe_sw(eq))) {
int set_ci = 0; int set_ci = 0;
while ((eqe = next_eqe_sw(eq))) {
/* /*
* Make sure we read EQ entry contents after we've * Make sure we read EQ entry contents after we've
* checked the ownership bit. * checked the ownership bit.
@ -345,12 +345,6 @@ static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)
be16_to_cpu(eqe->event.cmd.token), be16_to_cpu(eqe->event.cmd.token),
eqe->event.cmd.status, eqe->event.cmd.status,
be64_to_cpu(eqe->event.cmd.out_param)); be64_to_cpu(eqe->event.cmd.out_param));
/*
* cmd_event() may add more commands.
* The card will think the queue has overflowed if
* we don't tell it we've been processing events.
*/
set_ci = 1;
break; break;
case MTHCA_EVENT_TYPE_PORT_CHANGE: case MTHCA_EVENT_TYPE_PORT_CHANGE:
@ -385,8 +379,16 @@ static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)
set_eqe_hw(eqe); set_eqe_hw(eqe);
++eq->cons_index; ++eq->cons_index;
eqes_found = 1; eqes_found = 1;
++set_ci;
if (unlikely(set_ci)) { /*
* The HCA will think the queue has overflowed if we
* don't tell it we've been processing events. We
* create our EQs with MTHCA_NUM_SPARE_EQE extra
* entries, so we must update our consumer index at
* least that often.
*/
if (unlikely(set_ci >= MTHCA_NUM_SPARE_EQE)) {
/* /*
* Conditional on hca_type is OK here because * Conditional on hca_type is OK here because
* this is a rare case, not the fast path. * this is a rare case, not the fast path.
@ -862,19 +864,19 @@ int __devinit mthca_init_eq_table(struct mthca_dev *dev)
intr = (dev->mthca_flags & MTHCA_FLAG_MSI) ? intr = (dev->mthca_flags & MTHCA_FLAG_MSI) ?
128 : dev->eq_table.inta_pin; 128 : dev->eq_table.inta_pin;
err = mthca_create_eq(dev, dev->limits.num_cqs, err = mthca_create_eq(dev, dev->limits.num_cqs + MTHCA_NUM_SPARE_EQE,
(dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr, (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr,
&dev->eq_table.eq[MTHCA_EQ_COMP]); &dev->eq_table.eq[MTHCA_EQ_COMP]);
if (err) if (err)
goto err_out_unmap; goto err_out_unmap;
err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE, err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE + MTHCA_NUM_SPARE_EQE,
(dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr, (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr,
&dev->eq_table.eq[MTHCA_EQ_ASYNC]); &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
if (err) if (err)
goto err_out_comp; goto err_out_comp;
err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE, err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE + MTHCA_NUM_SPARE_EQE,
(dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr, (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr,
&dev->eq_table.eq[MTHCA_EQ_CMD]); &dev->eq_table.eq[MTHCA_EQ_CMD]);
if (err) if (err)