diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 669ea945d3cd..4dfe0b66c5f7 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -331,16 +331,16 @@ static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) } void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - int idx) + u16 curr) { struct bnxt_napi *bnapi = txr->bnapi; if (bnapi->tx_fault) return; - netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_pkts:%d cons:%u prod:%u i:%d)", - txr->txq_index, bnapi->tx_pkts, - txr->tx_cons, txr->tx_prod, idx); + netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)", + txr->txq_index, txr->tx_hw_cons, + txr->tx_cons, txr->tx_prod, curr); WARN_ON_ONCE(1); bnapi->tx_fault = 1; bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); @@ -691,13 +691,13 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); + u16 hw_cons = txr->tx_hw_cons; u16 cons = txr->tx_cons; struct pci_dev *pdev = bp->pdev; - int nr_pkts = bnapi->tx_pkts; - int i; unsigned int tx_bytes = 0; + int tx_pkts = 0; - for (i = 0; i < nr_pkts; i++) { + while (cons != hw_cons) { struct bnxt_sw_tx_bd *tx_buf; struct sk_buff *skb; int j, last; @@ -708,10 +708,11 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) tx_buf->skb = NULL; if (unlikely(!skb)) { - bnxt_sched_reset_txr(bp, txr, i); + bnxt_sched_reset_txr(bp, txr, cons); return; } + tx_pkts++; tx_bytes += skb->len; if (tx_buf->is_push) { @@ -748,10 +749,10 @@ next_tx_int: dev_consume_skb_any(skb); } - bnapi->tx_pkts = 0; + bnapi->events &= ~BNXT_TX_CMP_EVENT; WRITE_ONCE(txr->tx_cons, cons); - __netif_txq_completed_wake(txq, nr_pkts, tx_bytes, + __netif_txq_completed_wake(txq, tx_pkts, tx_bytes, bnxt_tx_avail(bp, txr), bp->tx_wake_thresh, READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING); } @@ -2588,14 +2589,15 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, { struct bnxt_napi *bnapi = cpr->bnapi; u32 raw_cons = cpr->cp_raw_cons; + struct bnxt_tx_ring_info *txr; u32 cons; - int tx_pkts = 0; int rx_pkts = 0; u8 event = 0; struct tx_cmp *txcmp; cpr->has_more_work = 0; cpr->had_work_done = 1; + txr = bnapi->tx_ring; while (1) { int rc; @@ -2610,9 +2612,15 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, */ dma_rmb(); if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { - tx_pkts++; + u32 opaque = txcmp->tx_cmp_opaque; + u16 tx_freed; + + event |= BNXT_TX_CMP_EVENT; + txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque); + tx_freed = (txr->tx_hw_cons - txr->tx_cons) & + bp->tx_ring_mask; /* return full budget so NAPI will complete. */ - if (unlikely(tx_pkts >= bp->tx_wake_thresh)) { + if (unlikely(tx_freed >= bp->tx_wake_thresh)) { rx_pkts = budget; raw_cons = NEXT_RAW_CMP(raw_cons); if (budget) @@ -2666,7 +2674,6 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } cpr->cp_raw_cons = raw_cons; - bnapi->tx_pkts += tx_pkts; bnapi->events |= event; return rx_pkts; } @@ -2674,7 +2681,7 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) { - if (bnapi->tx_pkts && !bnapi->tx_fault) + if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault) bnapi->tx_int(bp, bnapi, budget); if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { @@ -2687,7 +2694,7 @@ static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi, bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); } - bnapi->events = 0; + bnapi->events &= BNXT_TX_CMP_EVENT; } static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, @@ -4515,6 +4522,7 @@ static void bnxt_clear_ring_indices(struct bnxt *bp) if (txr) { txr->tx_prod = 0; txr->tx_cons = 0; + txr->tx_hw_cons = 0; } rxr = bnapi->rx_ring; @@ -4524,6 +4532,7 @@ static void bnxt_clear_ring_indices(struct bnxt *bp) rxr->rx_sw_agg_prod = 0; rxr->rx_next_cons = 0; } + bnapi->events = 0; } } @@ -9528,8 +9537,6 @@ static void bnxt_enable_napi(struct bnxt *bp) cpr = &bnapi->cp_ring; bnapi->in_reset = false; - bnapi->tx_pkts = 0; - if (bnapi->rx_ring) { INIT_WORK(&cpr->dim.work, bnxt_dim_work); cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index c7895e7d78d5..cf22aae91f70 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -68,6 +68,12 @@ struct tx_bd { #define SET_TX_OPAQUE(bp, idx, bds) \ (((bds) << TX_OPAQUE_BDS_SHIFT) | ((idx) & (bp)->tx_ring_mask)) +#define TX_OPAQUE_IDX(opq) ((opq) & TX_OPAQUE_IDX_MASK) +#define TX_OPAQUE_BDS(opq) (((opq) & TX_OPAQUE_BDS_MASK) >> \ + TX_OPAQUE_BDS_SHIFT) +#define TX_OPAQUE_PROD(bp, opq) ((TX_OPAQUE_IDX(opq) + TX_OPAQUE_BDS(opq)) &\ + (bp)->tx_ring_mask) + struct tx_bd_ext { __le32 tx_bd_hsize_lflags; #define TX_BD_FLAGS_TCP_UDP_CHKSUM (1 << 0) @@ -709,6 +715,7 @@ struct nqe_cn { #define BNXT_AGG_EVENT 2 #define BNXT_TX_EVENT 4 #define BNXT_REDIRECT_EVENT 8 +#define BNXT_TX_CMP_EVENT 0x10 struct bnxt_sw_tx_bd { union { @@ -801,6 +808,7 @@ struct bnxt_tx_ring_info { struct bnxt_napi *bnapi; u16 tx_prod; u16 tx_cons; + u16 tx_hw_cons; u16 txq_index; u8 kick_pending; struct bnxt_db_info tx_db; @@ -1027,7 +1035,6 @@ struct bnxt_napi { void (*tx_int)(struct bnxt *, struct bnxt_napi *, int budget); - int tx_pkts; u8 events; u8 tx_fault:1; @@ -2367,7 +2374,7 @@ int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init); void bnxt_tx_disable(struct bnxt *bp); void bnxt_tx_enable(struct bnxt *bp); void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - int idx); + u16 curr); void bnxt_report_link(struct bnxt *bp); int bnxt_update_link(struct bnxt *bp, bool chng_link_state); int bnxt_hwrm_set_pause(struct bnxt *); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 3e5144aafb0c..23476100fad2 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -129,17 +129,17 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; + u16 tx_hw_cons = txr->tx_hw_cons; bool rx_doorbell_needed = false; - int nr_pkts = bnapi->tx_pkts; struct bnxt_sw_tx_bd *tx_buf; u16 tx_cons = txr->tx_cons; u16 last_tx_cons = tx_cons; - int i, j, frags; + int j, frags; if (!budget) return; - for (i = 0; i < nr_pkts; i++) { + while (tx_cons != tx_hw_cons) { tx_buf = &txr->tx_buf_ring[tx_cons]; if (tx_buf->action == XDP_REDIRECT) { @@ -164,13 +164,13 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) page_pool_recycle_direct(rxr->page_pool, tx_buf->page); } } else { - bnxt_sched_reset_txr(bp, txr, i); + bnxt_sched_reset_txr(bp, txr, tx_cons); return; } tx_cons = NEXT_TX(tx_cons); } - bnapi->tx_pkts = 0; + bnapi->events &= ~BNXT_TX_CMP_EVENT; WRITE_ONCE(txr->tx_cons, tx_cons); if (rx_doorbell_needed) { tx_buf = &txr->tx_buf_ring[last_tx_cons]; @@ -275,7 +275,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, case XDP_TX: rx_buf = &rxr->rx_buf_ring[cons]; mapping = rx_buf->mapping - bp->rx_dma_offset; - *event = 0; + *event &= BNXT_TX_CMP_EVENT; if (unlikely(xdp_buff_has_frags(&xdp))) { struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);