From 6b00a76a1db6e8898b5f09e0f09ed129ce870ce3 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:44 -0600 Subject: [PATCH 1/7] net: ipa: don't thaw channel if error starting If an error occurs starting a channel, don't "thaw" it. We should assume the channel remains in a non-started state. Update the comment in gsi_channel_stop(); calls to this function are no longer retried. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 204fd8ba7728..1df7775a9bee 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -885,7 +885,9 @@ int gsi_channel_start(struct gsi *gsi, u32 channel_id) mutex_unlock(&gsi->mutex); - gsi_channel_thaw(channel); + /* Thaw the channel if successful */ + if (!ret) + gsi_channel_thaw(channel); return ret; } @@ -910,7 +912,7 @@ int gsi_channel_stop(struct gsi *gsi, u32 channel_id) mutex_unlock(&gsi->mutex); - /* Thaw the channel if we need to retry (or on error) */ + /* Re-thaw the channel if an error occurred while stopping */ if (ret) gsi_channel_thaw(channel); From 697e834e143afa733913fe2c9bc06b5e4d139c66 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:45 -0600 Subject: [PATCH 2/7] net: ipa: introduce gsi_channel_stop_retry() Create a new helper function that encapsulates issuing a set of channel stop commands, retrying if appropriate, with a short delay between attempts. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 1df7775a9bee..1d89ea6b9fe7 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -892,15 +892,12 @@ int gsi_channel_start(struct gsi *gsi, u32 channel_id) return ret; } -/* Stop a started channel */ -int gsi_channel_stop(struct gsi *gsi, u32 channel_id) +static int gsi_channel_stop_retry(struct gsi_channel *channel) { - struct gsi_channel *channel = &gsi->channel[channel_id]; u32 retries = GSI_CHANNEL_STOP_RETRIES; + struct gsi *gsi = channel->gsi; int ret; - gsi_channel_freeze(channel); - mutex_lock(&gsi->mutex); do { @@ -912,6 +909,19 @@ int gsi_channel_stop(struct gsi *gsi, u32 channel_id) mutex_unlock(&gsi->mutex); + return ret; +} + +/* Stop a started channel */ +int gsi_channel_stop(struct gsi *gsi, u32 channel_id) +{ + struct gsi_channel *channel = &gsi->channel[channel_id]; + int ret; + + gsi_channel_freeze(channel); + + ret = gsi_channel_stop_retry(channel); + /* Re-thaw the channel if an error occurred while stopping */ if (ret) gsi_channel_thaw(channel); From 893b838e73391c97c9388ef972899213bbb3049d Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:46 -0600 Subject: [PATCH 3/7] net: ipa: introduce __gsi_channel_start() Create a new function that does most of the work of starting a channel. What's different is that it takes a flag indicating whether the channel should really be started or not. Create another new function __gsi_channel_stop() that behaves similarly. IPA v3.5.1 implements suspend using a special SUSPEND endpoint setting. If the endpoint is suspended when an I/O completes on the underlying GSI channel, a SUSPEND interrupt is generated. Newer versions of IPA do not implement the SUSPEND endpoint mode. Instead, endpoint suspend is implemented by simply stopping the underlying GSI channel. In this case, a completing I/O on a *stopped* channel causes the SUSPEND interrupt condition. These new functions put all activity related to starting or stopping a channel (including "thawing/freezing" the channel) in one place, whether or not the channel is actually started or stopped. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 1d89ea6b9fe7..32c27fa39873 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -873,15 +873,14 @@ static void gsi_channel_deprogram(struct gsi_channel *channel) /* Nothing to do */ } -/* Start an allocated GSI channel */ -int gsi_channel_start(struct gsi *gsi, u32 channel_id) +static int __gsi_channel_start(struct gsi_channel *channel, bool start) { - struct gsi_channel *channel = &gsi->channel[channel_id]; + struct gsi *gsi = channel->gsi; int ret; mutex_lock(&gsi->mutex); - ret = gsi_channel_start_command(channel); + ret = start ? gsi_channel_start_command(channel) : 0; mutex_unlock(&gsi->mutex); @@ -892,6 +891,14 @@ int gsi_channel_start(struct gsi *gsi, u32 channel_id) return ret; } +/* Start an allocated GSI channel */ +int gsi_channel_start(struct gsi *gsi, u32 channel_id) +{ + struct gsi_channel *channel = &gsi->channel[channel_id]; + + return __gsi_channel_start(channel, true); +} + static int gsi_channel_stop_retry(struct gsi_channel *channel) { u32 retries = GSI_CHANNEL_STOP_RETRIES; @@ -912,15 +919,13 @@ static int gsi_channel_stop_retry(struct gsi_channel *channel) return ret; } -/* Stop a started channel */ -int gsi_channel_stop(struct gsi *gsi, u32 channel_id) +static int __gsi_channel_stop(struct gsi_channel *channel, bool stop) { - struct gsi_channel *channel = &gsi->channel[channel_id]; int ret; gsi_channel_freeze(channel); - ret = gsi_channel_stop_retry(channel); + ret = stop ? gsi_channel_stop_retry(channel) : 0; /* Re-thaw the channel if an error occurred while stopping */ if (ret) @@ -929,6 +934,14 @@ int gsi_channel_stop(struct gsi *gsi, u32 channel_id) return ret; } +/* Stop a started channel */ +int gsi_channel_stop(struct gsi *gsi, u32 channel_id) +{ + struct gsi_channel *channel = &gsi->channel[channel_id]; + + return __gsi_channel_stop(channel, true); +} + /* Reset and reconfigure a channel, (possibly) enabling the doorbell engine */ void gsi_channel_reset(struct gsi *gsi, u32 channel_id, bool doorbell) { @@ -952,12 +965,7 @@ int gsi_channel_suspend(struct gsi *gsi, u32 channel_id, bool stop) { struct gsi_channel *channel = &gsi->channel[channel_id]; - if (stop) - return gsi_channel_stop(gsi, channel_id); - - gsi_channel_freeze(channel); - - return 0; + return __gsi_channel_stop(channel, stop); } /* Resume a suspended channel (starting will be requested if STOPPED) */ @@ -965,12 +973,7 @@ int gsi_channel_resume(struct gsi *gsi, u32 channel_id, bool start) { struct gsi_channel *channel = &gsi->channel[channel_id]; - if (start) - return gsi_channel_start(gsi, channel_id); - - gsi_channel_thaw(channel); - - return 0; + return __gsi_channel_start(channel, start); } /** From bd1ea1e46448992a4a3dfb6e6e2c410ca069a41c Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:47 -0600 Subject: [PATCH 4/7] net: ipa: kill gsi_channel_freeze() and gsi_channel_thaw() Open-code gsi_channel_freeze() and gsi_channel_thaw() in all callers and get rid of these two functions. This is part of reworking the sequence of things done during channel suspend/resume and start/stop. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 32c27fa39873..049a63e21bca 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -764,24 +764,6 @@ static void gsi_channel_trans_quiesce(struct gsi_channel *channel) } } -/* Stop channel activity. Transactions may not be allocated until thawed. */ -static void gsi_channel_freeze(struct gsi_channel *channel) -{ - gsi_channel_trans_quiesce(channel); - - napi_disable(&channel->napi); - - gsi_irq_ieob_disable_one(channel->gsi, channel->evt_ring_id); -} - -/* Allow transactions to be used on the channel again. */ -static void gsi_channel_thaw(struct gsi_channel *channel) -{ - gsi_irq_ieob_enable_one(channel->gsi, channel->evt_ring_id); - - napi_enable(&channel->napi); -} - /* Program a channel for use */ static void gsi_channel_program(struct gsi_channel *channel, bool doorbell) { @@ -884,9 +866,10 @@ static int __gsi_channel_start(struct gsi_channel *channel, bool start) mutex_unlock(&gsi->mutex); - /* Thaw the channel if successful */ - if (!ret) - gsi_channel_thaw(channel); + if (!ret) { + gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); + napi_enable(&channel->napi); + } return ret; } @@ -921,15 +904,19 @@ static int gsi_channel_stop_retry(struct gsi_channel *channel) static int __gsi_channel_stop(struct gsi_channel *channel, bool stop) { + struct gsi *gsi = channel->gsi; int ret; - gsi_channel_freeze(channel); + gsi_channel_trans_quiesce(channel); + napi_disable(&channel->napi); + gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); ret = stop ? gsi_channel_stop_retry(channel) : 0; - /* Re-thaw the channel if an error occurred while stopping */ - if (ret) - gsi_channel_thaw(channel); + if (ret) { + gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); + napi_enable(&channel->napi); + } return ret; } From 4fef691c9b6ab5ed92b874bc6ddfb14a34c650ab Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:48 -0600 Subject: [PATCH 5/7] net: ipa: disable interrupt and NAPI after channel stop Disable both the I/O completion interrupt and NAPI polling on a channel *after* we successfully stop it rather than before. This ensures a completion occurring just before the channel is stopped gets processed. Enable NAPI polling and the interrupt *before* starting a channel rather than after, to be symmetric. A stopped channel won't generate any completion interrupts anyway. Enable NAPI before the interrupt and disable it afterward. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 049a63e21bca..9b3d5b639dac 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -860,15 +860,18 @@ static int __gsi_channel_start(struct gsi_channel *channel, bool start) struct gsi *gsi = channel->gsi; int ret; + napi_enable(&channel->napi); + gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); + mutex_lock(&gsi->mutex); ret = start ? gsi_channel_start_command(channel) : 0; mutex_unlock(&gsi->mutex); - if (!ret) { - gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); - napi_enable(&channel->napi); + if (ret) { + gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); + napi_disable(&channel->napi); } return ret; @@ -908,14 +911,11 @@ static int __gsi_channel_stop(struct gsi_channel *channel, bool stop) int ret; gsi_channel_trans_quiesce(channel); - napi_disable(&channel->napi); - gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); ret = stop ? gsi_channel_stop_retry(channel) : 0; - - if (ret) { - gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); - napi_enable(&channel->napi); + if (!ret) { + gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); + napi_disable(&channel->napi); } return ret; From a65c0288b355a8bb71f43cf9b4c33ada51e0ec26 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:49 -0600 Subject: [PATCH 6/7] net: ipa: don't disable interrupt on suspend No completion interrupts will occur while an endpoint is suspended, nor when a channel has been stopped for suspend. So there's no need to disable the interrupt during suspend and re-enable it when resuming. Without any interrupts occurring, there is no need to disable/re-enable NAPI for channel suspend/resume either. We'll only enable NAPI and the interrupt when we first start the channel, and disable it again only when it's "really" stopped. To accomplish this, move the enable/disable calls out of __gsi_channel_start() and __gsi_channel_stop(), and into gsi_channel_start() and gsi_channel_stop() instead. Add a call to napi_synchronize() to gsi_channel_suspend(), to ensure NAPI polling is done before moving on. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 44 ++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 9b3d5b639dac..dcc27b56102b 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -860,20 +860,15 @@ static int __gsi_channel_start(struct gsi_channel *channel, bool start) struct gsi *gsi = channel->gsi; int ret; - napi_enable(&channel->napi); - gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); + if (!start) + return 0; mutex_lock(&gsi->mutex); - ret = start ? gsi_channel_start_command(channel) : 0; + ret = gsi_channel_start_command(channel); mutex_unlock(&gsi->mutex); - if (ret) { - gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); - napi_disable(&channel->napi); - } - return ret; } @@ -881,8 +876,19 @@ static int __gsi_channel_start(struct gsi_channel *channel, bool start) int gsi_channel_start(struct gsi *gsi, u32 channel_id) { struct gsi_channel *channel = &gsi->channel[channel_id]; + int ret; - return __gsi_channel_start(channel, true); + /* Enable NAPI and the completion interrupt */ + napi_enable(&channel->napi); + gsi_irq_ieob_enable_one(gsi, channel->evt_ring_id); + + ret = __gsi_channel_start(channel, true); + if (ret) { + gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); + napi_disable(&channel->napi); + } + + return ret; } static int gsi_channel_stop_retry(struct gsi_channel *channel) @@ -907,16 +913,15 @@ static int gsi_channel_stop_retry(struct gsi_channel *channel) static int __gsi_channel_stop(struct gsi_channel *channel, bool stop) { - struct gsi *gsi = channel->gsi; int ret; + /* Wait for any underway transactions to complete before stopping. */ gsi_channel_trans_quiesce(channel); ret = stop ? gsi_channel_stop_retry(channel) : 0; - if (!ret) { - gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); - napi_disable(&channel->napi); - } + /* Finally, ensure NAPI polling has finished. */ + if (!ret) + napi_synchronize(&channel->napi); return ret; } @@ -925,8 +930,17 @@ static int __gsi_channel_stop(struct gsi_channel *channel, bool stop) int gsi_channel_stop(struct gsi *gsi, u32 channel_id) { struct gsi_channel *channel = &gsi->channel[channel_id]; + int ret; - return __gsi_channel_stop(channel, true); + /* Only disable the completion interrupt if stop is successful */ + ret = __gsi_channel_stop(channel, true); + if (ret) + return ret; + + gsi_irq_ieob_disable_one(gsi, channel->evt_ring_id); + napi_disable(&channel->napi); + + return 0; } /* Reset and reconfigure a channel, (possibly) enabling the doorbell engine */ From e63169208b25f1aaf3b6dc47a1df986d260efc3f Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Mon, 1 Feb 2021 11:28:50 -0600 Subject: [PATCH 7/7] net: ipa: expand last transaction check Transactions to send data for a network device can be allocated at any time up until the point the TX queue is stopped. It is possible for ipa_start_xmit() to be called in one context just before a the transmit queue is stopped in another. Update gsi_channel_trans_last() so that for TX channels the allocated and pending transaction lists are checked--in addition to the completed and polled lists--to determine the "last" transaction. This means any transaction that has been allocated before the TX queue is stopped will be allowed to complete before we conclude the channel is quiesced. Rework the function a bit to use a list pointer and gotos. Signed-off-by: Alex Elder Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- drivers/net/ipa/gsi.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index dcc27b56102b..53640447bf12 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -725,22 +725,38 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) gsi_evt_ring_doorbell(gsi, evt_ring_id, 0); } -/* Return the last (most recent) transaction completed on a channel. */ +/* Find the transaction whose completion indicates a channel is quiesced */ static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel) { struct gsi_trans_info *trans_info = &channel->trans_info; + const struct list_head *list; struct gsi_trans *trans; spin_lock_bh(&trans_info->spinlock); - if (!list_empty(&trans_info->complete)) - trans = list_last_entry(&trans_info->complete, - struct gsi_trans, links); - else if (!list_empty(&trans_info->polled)) - trans = list_last_entry(&trans_info->polled, - struct gsi_trans, links); - else - trans = NULL; + /* There is a small chance a TX transaction got allocated just + * before we disabled transmits, so check for that. + */ + if (channel->toward_ipa) { + list = &trans_info->alloc; + if (!list_empty(list)) + goto done; + list = &trans_info->pending; + if (!list_empty(list)) + goto done; + } + + /* Otherwise (TX or RX) we want to wait for anything that + * has completed, or has been polled but not released yet. + */ + list = &trans_info->complete; + if (!list_empty(list)) + goto done; + list = &trans_info->polled; + if (list_empty(list)) + list = NULL; +done: + trans = list ? list_last_entry(list, struct gsi_trans, links) : NULL; /* Caller will wait for this, so take a reference */ if (trans)