iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
linux/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
George Cherian f9e425e99b octeontx2-af: Add support for RSS hashing based on Transport protocol field 
Add support to choose RSS flow key algorithm with IPv4 transport protocol
field included in hashing input data. This will be enabled by default.
There-by enabling 3/5 tuple hash

Signed-off-by: Sunil Kovvuri Goutham <sgoutham@marvell.com>
Signed-off-by: George Cherian <george.cherian@marvell.com>
Link: https://lore.kernel.org/r/20201120093906.2873616-1-george.cherian@marvell.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2020-11-21 16:05:22 -08:00

3695 lines
98 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
/* Marvell OcteonTx2 RVU Admin Function driver
*
* Copyright (C) 2018 Marvell International Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/module.h>
#include <linux/pci.h>
#include "rvu_struct.h"
#include "rvu_reg.h"
#include "rvu.h"
#include "npc.h"
#include "cgx.h"
static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc);
static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
int type, int chan_id);
enum mc_tbl_sz {
MC_TBL_SZ_256,
MC_TBL_SZ_512,
MC_TBL_SZ_1K,
MC_TBL_SZ_2K,
MC_TBL_SZ_4K,
MC_TBL_SZ_8K,
MC_TBL_SZ_16K,
MC_TBL_SZ_32K,
MC_TBL_SZ_64K,
};
enum mc_buf_cnt {
MC_BUF_CNT_8,
MC_BUF_CNT_16,
MC_BUF_CNT_32,
MC_BUF_CNT_64,
MC_BUF_CNT_128,
MC_BUF_CNT_256,
MC_BUF_CNT_512,
MC_BUF_CNT_1024,
MC_BUF_CNT_2048,
};
enum nix_makr_fmt_indexes {
NIX_MARK_CFG_IP_DSCP_RED,
NIX_MARK_CFG_IP_DSCP_YELLOW,
NIX_MARK_CFG_IP_DSCP_YELLOW_RED,
NIX_MARK_CFG_IP_ECN_RED,
NIX_MARK_CFG_IP_ECN_YELLOW,
NIX_MARK_CFG_IP_ECN_YELLOW_RED,
NIX_MARK_CFG_VLAN_DEI_RED,
NIX_MARK_CFG_VLAN_DEI_YELLOW,
NIX_MARK_CFG_VLAN_DEI_YELLOW_RED,
NIX_MARK_CFG_MAX,
};
/* For now considering MC resources needed for broadcast
* pkt replication only. i.e 256 HWVFs + 12 PFs.
*/
#define MC_TBL_SIZE MC_TBL_SZ_512
#define MC_BUF_CNT MC_BUF_CNT_128
struct mce {
struct hlist_node node;
u16 pcifunc;
};
int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr)
{
int i = 0;
/*If blkaddr is 0, return the first nix block address*/
if (blkaddr == 0)
return rvu->nix_blkaddr[blkaddr];
while (i + 1 < MAX_NIX_BLKS) {
if (rvu->nix_blkaddr[i] == blkaddr)
return rvu->nix_blkaddr[i + 1];
i++;
}
return 0;
}
bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
int blkaddr;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return false;
return true;
}
int rvu_get_nixlf_count(struct rvu *rvu)
{
int blkaddr = 0, max = 0;
struct rvu_block *block;
blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
while (blkaddr) {
block = &rvu->hw->block[blkaddr];
max += block->lf.max;
blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
}
return max;
}
int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
struct rvu_hwinfo *hw = rvu->hw;
int blkaddr;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
*nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
if (*nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
if (nix_blkaddr)
*nix_blkaddr = blkaddr;
return 0;
}
static void nix_mce_list_init(struct nix_mce_list *list, int max)
{
INIT_HLIST_HEAD(&list->head);
list->count = 0;
list->max = max;
}
static u16 nix_alloc_mce_list(struct nix_mcast *mcast, int count)
{
int idx;
if (!mcast)
return 0;
idx = mcast->next_free_mce;
mcast->next_free_mce += count;
return idx;
}
struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr)
{
int nix_blkaddr = 0, i = 0;
struct rvu *rvu = hw->rvu;
nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr);
while (nix_blkaddr) {
if (blkaddr == nix_blkaddr && hw->nix)
return &hw->nix[i];
nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr);
i++;
}
return NULL;
}
static void nix_rx_sync(struct rvu *rvu, int blkaddr)
{
int err;
/*Sync all in flight RX packets to LLC/DRAM */
rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0));
err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true);
if (err)
dev_err(rvu->dev, "NIX RX software sync failed\n");
}
static bool is_valid_txschq(struct rvu *rvu, int blkaddr,
int lvl, u16 pcifunc, u16 schq)
{
struct rvu_hwinfo *hw = rvu->hw;
struct nix_txsch *txsch;
struct nix_hw *nix_hw;
u16 map_func;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return false;
txsch = &nix_hw->txsch[lvl];
/* Check out of bounds */
if (schq >= txsch->schq.max)
return false;
mutex_lock(&rvu->rsrc_lock);
map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]);
mutex_unlock(&rvu->rsrc_lock);
/* TLs aggegating traffic are shared across PF and VFs */
if (lvl >= hw->cap.nix_tx_aggr_lvl) {
if (rvu_get_pf(map_func) != rvu_get_pf(pcifunc))
return false;
else
return true;
}
if (map_func != pcifunc)
return false;
return true;
}
static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
int pkind, pf, vf, lbkid;
u8 cgx_id, lmac_id;
int err;
pf = rvu_get_pf(pcifunc);
if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
return 0;
switch (type) {
case NIX_INTF_TYPE_CGX:
pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf];
rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
pkind = rvu_npc_get_pkind(rvu, pf);
if (pkind < 0) {
dev_err(rvu->dev,
"PF_Func 0x%x: Invalid pkind\n", pcifunc);
return -EINVAL;
}
pfvf->rx_chan_base = NIX_CHAN_CGX_LMAC_CHX(cgx_id, lmac_id, 0);
pfvf->tx_chan_base = pfvf->rx_chan_base;
pfvf->rx_chan_cnt = 1;
pfvf->tx_chan_cnt = 1;
cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind);
rvu_npc_set_pkind(rvu, pkind, pfvf);
/* By default we enable pause frames */
if ((pcifunc & RVU_PFVF_FUNC_MASK) == 0)
cgx_lmac_set_pause_frm(rvu_cgx_pdata(cgx_id, rvu),
lmac_id, true, true);
break;
case NIX_INTF_TYPE_LBK:
vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
/* If NIX1 block is present on the silicon then NIXes are
* assigned alternatively for lbk interfaces. NIX0 should
* send packets on lbk link 1 channels and NIX1 should send
* on lbk link 0 channels for the communication between
* NIX0 and NIX1.
*/
lbkid = 0;
if (rvu->hw->lbk_links > 1)
lbkid = vf & 0x1 ? 0 : 1;
/* Note that AF's VFs work in pairs and talk over consecutive
* loopback channels.Therefore if odd number of AF VFs are
* enabled then the last VF remains with no pair.
*/
pfvf->rx_chan_base = NIX_CHAN_LBK_CHX(lbkid, vf);
pfvf->tx_chan_base = vf & 0x1 ?
NIX_CHAN_LBK_CHX(lbkid, vf - 1) :
NIX_CHAN_LBK_CHX(lbkid, vf + 1);
pfvf->rx_chan_cnt = 1;
pfvf->tx_chan_cnt = 1;
rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
pfvf->rx_chan_base, false);
break;
}
/* Add a UCAST forwarding rule in MCAM with this NIXLF attached
* RVU PF/VF's MAC address.
*/
rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
pfvf->rx_chan_base, pfvf->mac_addr);
/* Add this PF_FUNC to bcast pkt replication list */
err = nix_update_bcast_mce_list(rvu, pcifunc, true);
if (err) {
dev_err(rvu->dev,
"Bcast list, failed to enable PF_FUNC 0x%x\n",
pcifunc);
return err;
}
rvu_npc_install_bcast_match_entry(rvu, pcifunc,
nixlf, pfvf->rx_chan_base);
pfvf->maxlen = NIC_HW_MIN_FRS;
pfvf->minlen = NIC_HW_MIN_FRS;
return 0;
}
static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
int err;
pfvf->maxlen = 0;
pfvf->minlen = 0;
/* Remove this PF_FUNC from bcast pkt replication list */
err = nix_update_bcast_mce_list(rvu, pcifunc, false);
if (err) {
dev_err(rvu->dev,
"Bcast list, failed to disable PF_FUNC 0x%x\n",
pcifunc);
}
/* Free and disable any MCAM entries used by this NIX LF */
rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
}
int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu,
struct nix_bp_cfg_req *req,
struct msg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
struct rvu_pfvf *pfvf;
int blkaddr, pf, type;
u16 chan_base, chan;
u64 cfg;
pf = rvu_get_pf(pcifunc);
type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
return 0;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
chan_base = pfvf->rx_chan_base + req->chan_base;
for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
cfg & ~BIT_ULL(16));
}
return 0;
}
static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
int type, int chan_id)
{
int bpid, blkaddr, lmac_chan_cnt;
struct rvu_hwinfo *hw = rvu->hw;
u16 cgx_bpid_cnt, lbk_bpid_cnt;
struct rvu_pfvf *pfvf;
u8 cgx_id, lmac_id;
u64 cfg;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
lmac_chan_cnt = cfg & 0xFF;
cgx_bpid_cnt = hw->cgx_links * lmac_chan_cnt;
lbk_bpid_cnt = hw->lbk_links * ((cfg >> 16) & 0xFF);
pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
/* Backpressure IDs range division
* CGX channles are mapped to (0 - 191) BPIDs
* LBK channles are mapped to (192 - 255) BPIDs
* SDP channles are mapped to (256 - 511) BPIDs
*
* Lmac channles and bpids mapped as follows
* cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15)
* cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) ....
* cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) ....
*/
switch (type) {
case NIX_INTF_TYPE_CGX:
if ((req->chan_base + req->chan_cnt) > 15)
return -EINVAL;
rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
/* Assign bpid based on cgx, lmac and chan id */
bpid = (cgx_id * hw->lmac_per_cgx * lmac_chan_cnt) +
(lmac_id * lmac_chan_cnt) + req->chan_base;
if (req->bpid_per_chan)
bpid += chan_id;
if (bpid > cgx_bpid_cnt)
return -EINVAL;
break;
case NIX_INTF_TYPE_LBK:
if ((req->chan_base + req->chan_cnt) > 63)
return -EINVAL;
bpid = cgx_bpid_cnt + req->chan_base;
if (req->bpid_per_chan)
bpid += chan_id;
if (bpid > (cgx_bpid_cnt + lbk_bpid_cnt))
return -EINVAL;
break;
default:
return -EINVAL;
}
return bpid;
}
int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu,
struct nix_bp_cfg_req *req,
struct nix_bp_cfg_rsp *rsp)
{
int blkaddr, pf, type, chan_id = 0;
u16 pcifunc = req->hdr.pcifunc;
struct rvu_pfvf *pfvf;
u16 chan_base, chan;
s16 bpid, bpid_base;
u64 cfg;
pf = rvu_get_pf(pcifunc);
type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
/* Enable backpressure only for CGX mapped PFs and LBK interface */
if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
return 0;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id);
chan_base = pfvf->rx_chan_base + req->chan_base;
bpid = bpid_base;
for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
if (bpid < 0) {
dev_warn(rvu->dev, "Fail to enable backpressure\n");
return -EINVAL;
}
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
cfg | (bpid & 0xFF) | BIT_ULL(16));
chan_id++;
bpid = rvu_nix_get_bpid(rvu, req, type, chan_id);
}
for (chan = 0; chan < req->chan_cnt; chan++) {
/* Map channel and bpid assign to it */
rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 |
(bpid_base & 0x3FF);
if (req->bpid_per_chan)
bpid_base++;
}
rsp->chan_cnt = req->chan_cnt;
return 0;
}
static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr,
u64 format, bool v4, u64 *fidx)
{
struct nix_lso_format field = {0};
/* IP's Length field */
field.layer = NIX_TXLAYER_OL3;
/* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */
field.offset = v4 ? 2 : 4;
field.sizem1 = 1; /* i.e 2 bytes */
field.alg = NIX_LSOALG_ADD_PAYLEN;
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
*(u64 *)&field);
/* No ID field in IPv6 header */
if (!v4)
return;
/* IP's ID field */
field.layer = NIX_TXLAYER_OL3;
field.offset = 4;
field.sizem1 = 1; /* i.e 2 bytes */
field.alg = NIX_LSOALG_ADD_SEGNUM;
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
*(u64 *)&field);
}
static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr,
u64 format, u64 *fidx)
{
struct nix_lso_format field = {0};
/* TCP's sequence number field */
field.layer = NIX_TXLAYER_OL4;
field.offset = 4;
field.sizem1 = 3; /* i.e 4 bytes */
field.alg = NIX_LSOALG_ADD_OFFSET;
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
*(u64 *)&field);
/* TCP's flags field */
field.layer = NIX_TXLAYER_OL4;
field.offset = 12;
field.sizem1 = 1; /* 2 bytes */
field.alg = NIX_LSOALG_TCP_FLAGS;
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
*(u64 *)&field);
}
static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
{
u64 cfg, idx, fidx = 0;
/* Get max HW supported format indices */
cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF;
nix_hw->lso.total = cfg;
/* Enable LSO */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG);
/* For TSO, set first and middle segment flags to
* mask out PSH, RST & FIN flags in TCP packet
*/
cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16));
cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16);
rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63));
/* Setup default static LSO formats
*
* Configure format fields for TCPv4 segmentation offload
*/
idx = NIX_LSO_FORMAT_IDX_TSOV4;
nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx);
nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
/* Set rest of the fields to NOP */
for (; fidx < 8; fidx++) {
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
}
nix_hw->lso.in_use++;
/* Configure format fields for TCPv6 segmentation offload */
idx = NIX_LSO_FORMAT_IDX_TSOV6;
fidx = 0;
nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx);
nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
/* Set rest of the fields to NOP */
for (; fidx < 8; fidx++) {
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
}
nix_hw->lso.in_use++;
}
static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
{
kfree(pfvf->rq_bmap);
kfree(pfvf->sq_bmap);
kfree(pfvf->cq_bmap);
if (pfvf->rq_ctx)
qmem_free(rvu->dev, pfvf->rq_ctx);
if (pfvf->sq_ctx)
qmem_free(rvu->dev, pfvf->sq_ctx);
if (pfvf->cq_ctx)
qmem_free(rvu->dev, pfvf->cq_ctx);
if (pfvf->rss_ctx)
qmem_free(rvu->dev, pfvf->rss_ctx);
if (pfvf->nix_qints_ctx)
qmem_free(rvu->dev, pfvf->nix_qints_ctx);
if (pfvf->cq_ints_ctx)
qmem_free(rvu->dev, pfvf->cq_ints_ctx);
pfvf->rq_bmap = NULL;
pfvf->cq_bmap = NULL;
pfvf->sq_bmap = NULL;
pfvf->rq_ctx = NULL;
pfvf->sq_ctx = NULL;
pfvf->cq_ctx = NULL;
pfvf->rss_ctx = NULL;
pfvf->nix_qints_ctx = NULL;
pfvf->cq_ints_ctx = NULL;
}
static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr,
struct rvu_pfvf *pfvf, int nixlf,
int rss_sz, int rss_grps, int hwctx_size,
u64 way_mask)
{
int err, grp, num_indices;
/* RSS is not requested for this NIXLF */
if (!rss_sz)
return 0;
num_indices = rss_sz * rss_grps;
/* Alloc NIX RSS HW context memory and config the base */
err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size);
if (err)
return err;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf),
(u64)pfvf->rss_ctx->iova);
/* Config full RSS table size, enable RSS and caching */
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf),
BIT_ULL(36) | BIT_ULL(4) |
ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE) |
way_mask << 20);
/* Config RSS group offset and sizes */
for (grp = 0; grp < rss_grps; grp++)
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp),
((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp));
return 0;
}
static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
struct nix_aq_inst_s *inst)
{
struct admin_queue *aq = block->aq;
struct nix_aq_res_s *result;
int timeout = 1000;
u64 reg, head;
result = (struct nix_aq_res_s *)aq->res->base;
/* Get current head pointer where to append this instruction */
reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS);
head = (reg >> 4) & AQ_PTR_MASK;
memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
(void *)inst, aq->inst->entry_sz);
memset(result, 0, sizeof(*result));
/* sync into memory */
wmb();
/* Ring the doorbell and wait for result */
rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1);
while (result->compcode == NIX_AQ_COMP_NOTDONE) {
cpu_relax();
udelay(1);
timeout--;
if (!timeout)
return -EBUSY;
}
if (result->compcode != NIX_AQ_COMP_GOOD)
/* TODO: Replace this with some error code */
return -EBUSY;
return 0;
}
static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw,
struct nix_aq_enq_req *req,
struct nix_aq_enq_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
int nixlf, blkaddr, rc = 0;
struct nix_aq_inst_s inst;
struct rvu_block *block;
struct admin_queue *aq;
struct rvu_pfvf *pfvf;
void *ctx, *mask;
bool ena;
u64 cfg;
blkaddr = nix_hw->blkaddr;
block = &hw->block[blkaddr];
aq = block->aq;
if (!aq) {
dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__);
return NIX_AF_ERR_AQ_ENQUEUE;
}
pfvf = rvu_get_pfvf(rvu, pcifunc);
nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
/* Skip NIXLF check for broadcast MCE entry init */
if (!(!rsp && req->ctype == NIX_AQ_CTYPE_MCE)) {
if (!pfvf->nixlf || nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
}
switch (req->ctype) {
case NIX_AQ_CTYPE_RQ:
/* Check if index exceeds max no of queues */
if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize)
rc = NIX_AF_ERR_AQ_ENQUEUE;
break;
case NIX_AQ_CTYPE_SQ:
if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize)
rc = NIX_AF_ERR_AQ_ENQUEUE;
break;
case NIX_AQ_CTYPE_CQ:
if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize)
rc = NIX_AF_ERR_AQ_ENQUEUE;
break;
case NIX_AQ_CTYPE_RSS:
/* Check if RSS is enabled and qidx is within range */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf));
if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx ||
(req->qidx >= (256UL << (cfg & 0xF))))
rc = NIX_AF_ERR_AQ_ENQUEUE;
break;
case NIX_AQ_CTYPE_MCE:
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG);
/* Check if index exceeds MCE list length */
if (!nix_hw->mcast.mce_ctx ||
(req->qidx >= (256UL << (cfg & 0xF))))
rc = NIX_AF_ERR_AQ_ENQUEUE;
/* Adding multicast lists for requests from PF/VFs is not
* yet supported, so ignore this.
*/
if (rsp)
rc = NIX_AF_ERR_AQ_ENQUEUE;
break;
default:
rc = NIX_AF_ERR_AQ_ENQUEUE;
}
if (rc)
return rc;
/* Check if SQ pointed SMQ belongs to this PF/VF or not */
if (req->ctype == NIX_AQ_CTYPE_SQ &&
((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) ||
(req->op == NIX_AQ_INSTOP_WRITE &&
req->sq_mask.ena && req->sq_mask.smq && req->sq.ena))) {
if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ,
pcifunc, req->sq.smq))
return NIX_AF_ERR_AQ_ENQUEUE;
}
memset(&inst, 0, sizeof(struct nix_aq_inst_s));
inst.lf = nixlf;
inst.cindex = req->qidx;
inst.ctype = req->ctype;
inst.op = req->op;
/* Currently we are not supporting enqueuing multiple instructions,
* so always choose first entry in result memory.
*/
inst.res_addr = (u64)aq->res->iova;
/* Hardware uses same aq->res->base for updating result of
* previous instruction hence wait here till it is done.
*/
spin_lock(&aq->lock);
/* Clean result + context memory */
memset(aq->res->base, 0, aq->res->entry_sz);
/* Context needs to be written at RES_ADDR + 128 */
ctx = aq->res->base + 128;
/* Mask needs to be written at RES_ADDR + 256 */
mask = aq->res->base + 256;
switch (req->op) {
case NIX_AQ_INSTOP_WRITE:
if (req->ctype == NIX_AQ_CTYPE_RQ)
memcpy(mask, &req->rq_mask,
sizeof(struct nix_rq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_SQ)
memcpy(mask, &req->sq_mask,
sizeof(struct nix_sq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_CQ)
memcpy(mask, &req->cq_mask,
sizeof(struct nix_cq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_RSS)
memcpy(mask, &req->rss_mask,
sizeof(struct nix_rsse_s));
else if (req->ctype == NIX_AQ_CTYPE_MCE)
memcpy(mask, &req->mce_mask,
sizeof(struct nix_rx_mce_s));
fallthrough;
case NIX_AQ_INSTOP_INIT:
if (req->ctype == NIX_AQ_CTYPE_RQ)
memcpy(ctx, &req->rq, sizeof(struct nix_rq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_SQ)
memcpy(ctx, &req->sq, sizeof(struct nix_sq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_CQ)
memcpy(ctx, &req->cq, sizeof(struct nix_cq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_RSS)
memcpy(ctx, &req->rss, sizeof(struct nix_rsse_s));
else if (req->ctype == NIX_AQ_CTYPE_MCE)
memcpy(ctx, &req->mce, sizeof(struct nix_rx_mce_s));
break;
case NIX_AQ_INSTOP_NOP:
case NIX_AQ_INSTOP_READ:
case NIX_AQ_INSTOP_LOCK:
case NIX_AQ_INSTOP_UNLOCK:
break;
default:
rc = NIX_AF_ERR_AQ_ENQUEUE;
spin_unlock(&aq->lock);
return rc;
}
/* Submit the instruction to AQ */
rc = nix_aq_enqueue_wait(rvu, block, &inst);
if (rc) {
spin_unlock(&aq->lock);
return rc;
}
/* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */
if (req->op == NIX_AQ_INSTOP_INIT) {
if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena)
__set_bit(req->qidx, pfvf->rq_bmap);
if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena)
__set_bit(req->qidx, pfvf->sq_bmap);
if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena)
__set_bit(req->qidx, pfvf->cq_bmap);
}
if (req->op == NIX_AQ_INSTOP_WRITE) {
if (req->ctype == NIX_AQ_CTYPE_RQ) {
ena = (req->rq.ena & req->rq_mask.ena) |
(test_bit(req->qidx, pfvf->rq_bmap) &
~req->rq_mask.ena);
if (ena)
__set_bit(req->qidx, pfvf->rq_bmap);
else
__clear_bit(req->qidx, pfvf->rq_bmap);
}
if (req->ctype == NIX_AQ_CTYPE_SQ) {
ena = (req->rq.ena & req->sq_mask.ena) |
(test_bit(req->qidx, pfvf->sq_bmap) &
~req->sq_mask.ena);
if (ena)
__set_bit(req->qidx, pfvf->sq_bmap);
else
__clear_bit(req->qidx, pfvf->sq_bmap);
}
if (req->ctype == NIX_AQ_CTYPE_CQ) {
ena = (req->rq.ena & req->cq_mask.ena) |
(test_bit(req->qidx, pfvf->cq_bmap) &
~req->cq_mask.ena);
if (ena)
__set_bit(req->qidx, pfvf->cq_bmap);
else
__clear_bit(req->qidx, pfvf->cq_bmap);
}
}
if (rsp) {
/* Copy read context into mailbox */
if (req->op == NIX_AQ_INSTOP_READ) {
if (req->ctype == NIX_AQ_CTYPE_RQ)
memcpy(&rsp->rq, ctx,
sizeof(struct nix_rq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_SQ)
memcpy(&rsp->sq, ctx,
sizeof(struct nix_sq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_CQ)
memcpy(&rsp->cq, ctx,
sizeof(struct nix_cq_ctx_s));
else if (req->ctype == NIX_AQ_CTYPE_RSS)
memcpy(&rsp->rss, ctx,
sizeof(struct nix_rsse_s));
else if (req->ctype == NIX_AQ_CTYPE_MCE)
memcpy(&rsp->mce, ctx,
sizeof(struct nix_rx_mce_s));
}
}
spin_unlock(&aq->lock);
return 0;
}
static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req,
struct nix_aq_enq_rsp *rsp)
{
struct nix_hw *nix_hw;
int blkaddr;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
if (blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp);
}
static const char *nix_get_ctx_name(int ctype)
{
switch (ctype) {
case NIX_AQ_CTYPE_CQ:
return "CQ";
case NIX_AQ_CTYPE_SQ:
return "SQ";
case NIX_AQ_CTYPE_RQ:
return "RQ";
case NIX_AQ_CTYPE_RSS:
return "RSS";
}
return "";
}
static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
struct nix_aq_enq_req aq_req;
unsigned long *bmap;
int qidx, q_cnt = 0;
int err = 0, rc;
if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx)
return NIX_AF_ERR_AQ_ENQUEUE;
memset(&aq_req, 0, sizeof(struct nix_aq_enq_req));
aq_req.hdr.pcifunc = req->hdr.pcifunc;
if (req->ctype == NIX_AQ_CTYPE_CQ) {
aq_req.cq.ena = 0;
aq_req.cq_mask.ena = 1;
aq_req.cq.bp_ena = 0;
aq_req.cq_mask.bp_ena = 1;
q_cnt = pfvf->cq_ctx->qsize;
bmap = pfvf->cq_bmap;
}
if (req->ctype == NIX_AQ_CTYPE_SQ) {
aq_req.sq.ena = 0;
aq_req.sq_mask.ena = 1;
q_cnt = pfvf->sq_ctx->qsize;
bmap = pfvf->sq_bmap;
}
if (req->ctype == NIX_AQ_CTYPE_RQ) {
aq_req.rq.ena = 0;
aq_req.rq_mask.ena = 1;
q_cnt = pfvf->rq_ctx->qsize;
bmap = pfvf->rq_bmap;
}
aq_req.ctype = req->ctype;
aq_req.op = NIX_AQ_INSTOP_WRITE;
for (qidx = 0; qidx < q_cnt; qidx++) {
if (!test_bit(qidx, bmap))
continue;
aq_req.qidx = qidx;
rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL);
if (rc) {
err = rc;
dev_err(rvu->dev, "Failed to disable %s:%d context\n",
nix_get_ctx_name(req->ctype), qidx);
}
}
return err;
}
#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req)
{
struct nix_aq_enq_req lock_ctx_req;
int err;
if (req->op != NIX_AQ_INSTOP_INIT)
return 0;
if (req->ctype == NIX_AQ_CTYPE_MCE ||
req->ctype == NIX_AQ_CTYPE_DYNO)
return 0;
memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req));
lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
lock_ctx_req.ctype = req->ctype;
lock_ctx_req.op = NIX_AQ_INSTOP_LOCK;
lock_ctx_req.qidx = req->qidx;
err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL);
if (err)
dev_err(rvu->dev,
"PFUNC 0x%x: Failed to lock NIX %s:%d context\n",
req->hdr.pcifunc,
nix_get_ctx_name(req->ctype), req->qidx);
return err;
}
int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
struct nix_aq_enq_req *req,
struct nix_aq_enq_rsp *rsp)
{
int err;
err = rvu_nix_aq_enq_inst(rvu, req, rsp);
if (!err)
err = nix_lf_hwctx_lockdown(rvu, req);
return err;
}
#else
int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
struct nix_aq_enq_req *req,
struct nix_aq_enq_rsp *rsp)
{
return rvu_nix_aq_enq_inst(rvu, req, rsp);
}
#endif
int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu,
struct hwctx_disable_req *req,
struct msg_rsp *rsp)
{
return nix_lf_hwctx_disable(rvu, req);
}
int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu,
struct nix_lf_alloc_req *req,
struct nix_lf_alloc_rsp *rsp)
{
int nixlf, qints, hwctx_size, intf, err, rc = 0;
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
struct rvu_block *block;
struct rvu_pfvf *pfvf;
u64 cfg, ctx_cfg;
int blkaddr;
if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt)
return NIX_AF_ERR_PARAM;
if (req->way_mask)
req->way_mask &= 0xFFFF;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
block = &hw->block[blkaddr];
nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
if (nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
/* Check if requested 'NIXLF <=> NPALF' mapping is valid */
if (req->npa_func) {
/* If default, use 'this' NIXLF's PFFUNC */
if (req->npa_func == RVU_DEFAULT_PF_FUNC)
req->npa_func = pcifunc;
if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA))
return NIX_AF_INVAL_NPA_PF_FUNC;
}
/* Check if requested 'NIXLF <=> SSOLF' mapping is valid */
if (req->sso_func) {
/* If default, use 'this' NIXLF's PFFUNC */
if (req->sso_func == RVU_DEFAULT_PF_FUNC)
req->sso_func = pcifunc;
if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO))
return NIX_AF_INVAL_SSO_PF_FUNC;
}
/* If RSS is being enabled, check if requested config is valid.
* RSS table size should be power of two, otherwise
* RSS_GRP::OFFSET + adder might go beyond that group or
* won't be able to use entire table.
*/
if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE ||
!is_power_of_2(req->rss_sz)))
return NIX_AF_ERR_RSS_SIZE_INVALID;
if (req->rss_sz &&
(!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS))
return NIX_AF_ERR_RSS_GRPS_INVALID;
/* Reset this NIX LF */
err = rvu_lf_reset(rvu, block, nixlf);
if (err) {
dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
block->addr - BLKADDR_NIX0, nixlf);
return NIX_AF_ERR_LF_RESET;
}
ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3);
/* Alloc NIX RQ HW context memory and config the base */
hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size);
if (err)
goto free_mem;
pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL);
if (!pfvf->rq_bmap)
goto free_mem;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf),
(u64)pfvf->rq_ctx->iova);
/* Set caching and queue count in HW */
cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg);
/* Alloc NIX SQ HW context memory and config the base */
hwctx_size = 1UL << (ctx_cfg & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size);
if (err)
goto free_mem;
pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL);
if (!pfvf->sq_bmap)
goto free_mem;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf),
(u64)pfvf->sq_ctx->iova);
cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg);
/* Alloc NIX CQ HW context memory and config the base */
hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size);
if (err)
goto free_mem;
pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL);
if (!pfvf->cq_bmap)
goto free_mem;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf),
(u64)pfvf->cq_ctx->iova);
cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg);
/* Initialize receive side scaling (RSS) */
hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF);
err = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz,
req->rss_grps, hwctx_size, req->way_mask);
if (err)
goto free_mem;
/* Alloc memory for CQINT's HW contexts */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
qints = (cfg >> 24) & 0xFFF;
hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size);
if (err)
goto free_mem;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf),
(u64)pfvf->cq_ints_ctx->iova);
rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf),
BIT_ULL(36) | req->way_mask << 20);
/* Alloc memory for QINT's HW contexts */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
qints = (cfg >> 12) & 0xFFF;
hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size);
if (err)
goto free_mem;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf),
(u64)pfvf->nix_qints_ctx->iova);
rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf),
BIT_ULL(36) | req->way_mask << 20);
/* Setup VLANX TPID's.
* Use VLAN1 for 802.1Q
* and VLAN0 for 802.1AD.
*/
cfg = (0x8100ULL << 16) | 0x88A8ULL;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
/* Enable LMTST for this NIX LF */
rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0));
/* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */
if (req->npa_func)
cfg = req->npa_func;
if (req->sso_func)
cfg |= (u64)req->sso_func << 16;
cfg |= (u64)req->xqe_sz << 33;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg);
/* Config Rx pkt length, csum checks and apad enable / disable */
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg);
/* Configure pkind for TX parse config */
cfg = NPC_TX_DEF_PKIND;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg);
intf = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
err = nix_interface_init(rvu, pcifunc, intf, nixlf);
if (err)
goto free_mem;
/* Disable NPC entries as NIXLF's contexts are not initialized yet */
rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
/* Configure RX VTAG Type 7 (strip) for vf vlan */
rvu_write64(rvu, blkaddr,
NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, NIX_AF_LFX_RX_VTAG_TYPE7),
VTAGSIZE_T4 | VTAG_STRIP);
goto exit;
free_mem:
nix_ctx_free(rvu, pfvf);
rc = -ENOMEM;
exit:
/* Set macaddr of this PF/VF */
ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
/* set SQB size info */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST);
rsp->sqb_size = (cfg >> 34) & 0xFFFF;
rsp->rx_chan_base = pfvf->rx_chan_base;
rsp->tx_chan_base = pfvf->tx_chan_base;
rsp->rx_chan_cnt = pfvf->rx_chan_cnt;
rsp->tx_chan_cnt = pfvf->tx_chan_cnt;
rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4;
rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6;
/* Get HW supported stat count */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
rsp->lf_rx_stats = ((cfg >> 32) & 0xFF);
rsp->lf_tx_stats = ((cfg >> 24) & 0xFF);
/* Get count of CQ IRQs and error IRQs supported per LF */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
rsp->qints = ((cfg >> 12) & 0xFFF);
rsp->cints = ((cfg >> 24) & 0xFFF);
rsp->cgx_links = hw->cgx_links;
rsp->lbk_links = hw->lbk_links;
rsp->sdp_links = hw->sdp_links;
return rc;
}
int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct nix_lf_free_req *req,
struct msg_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
struct rvu_block *block;
int blkaddr, nixlf, err;
struct rvu_pfvf *pfvf;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
block = &hw->block[blkaddr];
nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
if (nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
if (req->flags & NIX_LF_DISABLE_FLOWS)
rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
else
rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf);
/* Free any tx vtag def entries used by this NIX LF */
if (!(req->flags & NIX_LF_DONT_FREE_TX_VTAG))
nix_free_tx_vtag_entries(rvu, pcifunc);
nix_interface_deinit(rvu, pcifunc, nixlf);
/* Reset this NIX LF */
err = rvu_lf_reset(rvu, block, nixlf);
if (err) {
dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
block->addr - BLKADDR_NIX0, nixlf);
return NIX_AF_ERR_LF_RESET;
}
nix_ctx_free(rvu, pfvf);
return 0;
}
int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu,
struct nix_mark_format_cfg *req,
struct nix_mark_format_cfg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
struct nix_hw *nix_hw;
struct rvu_pfvf *pfvf;
int blkaddr, rc;
u32 cfg;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
cfg = (((u32)req->offset & 0x7) << 16) |
(((u32)req->y_mask & 0xF) << 12) |
(((u32)req->y_val & 0xF) << 8) |
(((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF);
rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg);
if (rc < 0) {
dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)",
rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
return NIX_AF_ERR_MARK_CFG_FAIL;
}
rsp->mark_format_idx = rc;
return 0;
}
/* Disable shaping of pkts by a scheduler queue
* at a given scheduler level.
*/
static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr,
int lvl, int schq)
{
u64 cir_reg = 0, pir_reg = 0;
u64 cfg;
switch (lvl) {
case NIX_TXSCH_LVL_TL1:
cir_reg = NIX_AF_TL1X_CIR(schq);
pir_reg = 0; /* PIR not available at TL1 */
break;
case NIX_TXSCH_LVL_TL2:
cir_reg = NIX_AF_TL2X_CIR(schq);
pir_reg = NIX_AF_TL2X_PIR(schq);
break;
case NIX_TXSCH_LVL_TL3:
cir_reg = NIX_AF_TL3X_CIR(schq);
pir_reg = NIX_AF_TL3X_PIR(schq);
break;
case NIX_TXSCH_LVL_TL4:
cir_reg = NIX_AF_TL4X_CIR(schq);
pir_reg = NIX_AF_TL4X_PIR(schq);
break;
}
if (!cir_reg)
return;
cfg = rvu_read64(rvu, blkaddr, cir_reg);
rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0));
if (!pir_reg)
return;
cfg = rvu_read64(rvu, blkaddr, pir_reg);
rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0));
}
static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr,
int lvl, int schq)
{
struct rvu_hwinfo *hw = rvu->hw;
int link;
if (lvl >= hw->cap.nix_tx_aggr_lvl)
return;
/* Reset TL4's SDP link config */
if (lvl == NIX_TXSCH_LVL_TL4)
rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00);
if (lvl != NIX_TXSCH_LVL_TL2)
return;
/* Reset TL2's CGX or LBK link config */
for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++)
rvu_write64(rvu, blkaddr,
NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00);
}
static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc)
{
struct rvu_hwinfo *hw = rvu->hw;
int pf = rvu_get_pf(pcifunc);
u8 cgx_id = 0, lmac_id = 0;
if (is_afvf(pcifunc)) {/* LBK links */
return hw->cgx_links;
} else if (is_pf_cgxmapped(rvu, pf)) {
rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
return (cgx_id * hw->lmac_per_cgx) + lmac_id;
}
/* SDP link */
return hw->cgx_links + hw->lbk_links;
}
static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc,
int link, int *start, int *end)
{
struct rvu_hwinfo *hw = rvu->hw;
int pf = rvu_get_pf(pcifunc);
if (is_afvf(pcifunc)) { /* LBK links */
*start = hw->cap.nix_txsch_per_cgx_lmac * link;
*end = *start + hw->cap.nix_txsch_per_lbk_lmac;
} else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */
*start = hw->cap.nix_txsch_per_cgx_lmac * link;
*end = *start + hw->cap.nix_txsch_per_cgx_lmac;
} else { /* SDP link */
*start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) +
(hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links);
*end = *start + hw->cap.nix_txsch_per_sdp_lmac;
}
}
static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc,
struct nix_hw *nix_hw,
struct nix_txsch_alloc_req *req)
{
struct rvu_hwinfo *hw = rvu->hw;
int schq, req_schq, free_cnt;
struct nix_txsch *txsch;
int link, start, end;
txsch = &nix_hw->txsch[lvl];
req_schq = req->schq_contig[lvl] + req->schq[lvl];
if (!req_schq)
return 0;
link = nix_get_tx_link(rvu, pcifunc);
/* For traffic aggregating scheduler level, one queue is enough */
if (lvl >= hw->cap.nix_tx_aggr_lvl) {
if (req_schq != 1)
return NIX_AF_ERR_TLX_ALLOC_FAIL;
return 0;
}
/* Get free SCHQ count and check if request can be accomodated */
if (hw->cap.nix_fixed_txschq_mapping) {
nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
schq = start + (pcifunc & RVU_PFVF_FUNC_MASK);
if (end <= txsch->schq.max && schq < end &&
!test_bit(schq, txsch->schq.bmap))
free_cnt = 1;
else
free_cnt = 0;
} else {
free_cnt = rvu_rsrc_free_count(&txsch->schq);
}
if (free_cnt < req_schq || req_schq > MAX_TXSCHQ_PER_FUNC)
return NIX_AF_ERR_TLX_ALLOC_FAIL;
/* If contiguous queues are needed, check for availability */
if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] &&
!rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl]))
return NIX_AF_ERR_TLX_ALLOC_FAIL;
return 0;
}
static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch,
struct nix_txsch_alloc_rsp *rsp,
int lvl, int start, int end)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = rsp->hdr.pcifunc;
int idx, schq;
/* For traffic aggregating levels, queue alloc is based
* on transmit link to which PF_FUNC is mapped to.
*/
if (lvl >= hw->cap.nix_tx_aggr_lvl) {
/* A single TL queue is allocated */
if (rsp->schq_contig[lvl]) {
rsp->schq_contig[lvl] = 1;
rsp->schq_contig_list[lvl][0] = start;
}
/* Both contig and non-contig reqs doesn't make sense here */
if (rsp->schq_contig[lvl])
rsp->schq[lvl] = 0;
if (rsp->schq[lvl]) {
rsp->schq[lvl] = 1;
rsp->schq_list[lvl][0] = start;
}
return;
}
/* Adjust the queue request count if HW supports
* only one queue per level configuration.
*/
if (hw->cap.nix_fixed_txschq_mapping) {
idx = pcifunc & RVU_PFVF_FUNC_MASK;
schq = start + idx;
if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) {
rsp->schq_contig[lvl] = 0;
rsp->schq[lvl] = 0;
return;
}
if (rsp->schq_contig[lvl]) {
rsp->schq_contig[lvl] = 1;
set_bit(schq, txsch->schq.bmap);
rsp->schq_contig_list[lvl][0] = schq;
rsp->schq[lvl] = 0;
} else if (rsp->schq[lvl]) {
rsp->schq[lvl] = 1;
set_bit(schq, txsch->schq.bmap);
rsp->schq_list[lvl][0] = schq;
}
return;
}
/* Allocate contiguous queue indices requesty first */
if (rsp->schq_contig[lvl]) {
schq = bitmap_find_next_zero_area(txsch->schq.bmap,
txsch->schq.max, start,
rsp->schq_contig[lvl], 0);
if (schq >= end)
rsp->schq_contig[lvl] = 0;
for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) {
set_bit(schq, txsch->schq.bmap);
rsp->schq_contig_list[lvl][idx] = schq;
schq++;
}
}
/* Allocate non-contiguous queue indices */
if (rsp->schq[lvl]) {
idx = 0;
for (schq = start; schq < end; schq++) {
if (!test_bit(schq, txsch->schq.bmap)) {
set_bit(schq, txsch->schq.bmap);
rsp->schq_list[lvl][idx++] = schq;
}
if (idx == rsp->schq[lvl])
break;
}
/* Update how many were allocated */
rsp->schq[lvl] = idx;
}
}
int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu,
struct nix_txsch_alloc_req *req,
struct nix_txsch_alloc_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
int link, blkaddr, rc = 0;
int lvl, idx, start, end;
struct nix_txsch *txsch;
struct rvu_pfvf *pfvf;
struct nix_hw *nix_hw;
u32 *pfvf_map;
u16 schq;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
mutex_lock(&rvu->rsrc_lock);
/* Check if request is valid as per HW capabilities
* and can be accomodated.
*/
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req);
if (rc)
goto err;
}
/* Allocate requested Tx scheduler queues */
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
txsch = &nix_hw->txsch[lvl];
pfvf_map = txsch->pfvf_map;
if (!req->schq[lvl] && !req->schq_contig[lvl])
continue;
rsp->schq[lvl] = req->schq[lvl];
rsp->schq_contig[lvl] = req->schq_contig[lvl];
link = nix_get_tx_link(rvu, pcifunc);
if (lvl >= hw->cap.nix_tx_aggr_lvl) {
start = link;
end = link;
} else if (hw->cap.nix_fixed_txschq_mapping) {
nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
} else {
start = 0;
end = txsch->schq.max;
}
nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end);
/* Reset queue config */
for (idx = 0; idx < req->schq_contig[lvl]; idx++) {
schq = rsp->schq_contig_list[lvl][idx];
if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
NIX_TXSCHQ_CFG_DONE))
pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
nix_reset_tx_shaping(rvu, blkaddr, lvl, schq);
}
for (idx = 0; idx < req->schq[lvl]; idx++) {
schq = rsp->schq_list[lvl][idx];
if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
NIX_TXSCHQ_CFG_DONE))
pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
nix_reset_tx_shaping(rvu, blkaddr, lvl, schq);
}
}
rsp->aggr_level = hw->cap.nix_tx_aggr_lvl;
rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO;
rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr,
NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ?
NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2;
goto exit;
err:
rc = NIX_AF_ERR_TLX_ALLOC_FAIL;
exit:
mutex_unlock(&rvu->rsrc_lock);
return rc;
}
static void nix_smq_flush(struct rvu *rvu, int blkaddr,
int smq, u16 pcifunc, int nixlf)
{
int pf = rvu_get_pf(pcifunc);
u8 cgx_id = 0, lmac_id = 0;
int err, restore_tx_en = 0;
u64 cfg;
/* enable cgx tx if disabled */
if (is_pf_cgxmapped(rvu, pf)) {
rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu),
lmac_id, true);
}
cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq));
/* Do SMQ flush and set enqueue xoff */
cfg |= BIT_ULL(50) | BIT_ULL(49);
rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg);
/* Disable backpressure from physical link,
* otherwise SMQ flush may stall.
*/
rvu_cgx_enadis_rx_bp(rvu, pf, false);
/* Wait for flush to complete */
err = rvu_poll_reg(rvu, blkaddr,
NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true);
if (err)
dev_err(rvu->dev,
"NIXLF%d: SMQ%d flush failed\n", nixlf, smq);
rvu_cgx_enadis_rx_bp(rvu, pf, true);
/* restore cgx tx state */
if (restore_tx_en)
cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
}
static int nix_txschq_free(struct rvu *rvu, u16 pcifunc)
{
int blkaddr, nixlf, lvl, schq, err;
struct rvu_hwinfo *hw = rvu->hw;
struct nix_txsch *txsch;
struct nix_hw *nix_hw;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
if (nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
/* Disable TL2/3 queue links before SMQ flush*/
mutex_lock(&rvu->rsrc_lock);
for (lvl = NIX_TXSCH_LVL_TL4; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
if (lvl != NIX_TXSCH_LVL_TL2 && lvl != NIX_TXSCH_LVL_TL4)
continue;
txsch = &nix_hw->txsch[lvl];
for (schq = 0; schq < txsch->schq.max; schq++) {
if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
continue;
nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
}
}
/* Flush SMQs */
txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
for (schq = 0; schq < txsch->schq.max; schq++) {
if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
continue;
nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
}
/* Now free scheduler queues to free pool */
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
/* TLs above aggregation level are shared across all PF
* and it's VFs, hence skip freeing them.
*/
if (lvl >= hw->cap.nix_tx_aggr_lvl)
continue;
txsch = &nix_hw->txsch[lvl];
for (schq = 0; schq < txsch->schq.max; schq++) {
if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
continue;
rvu_free_rsrc(&txsch->schq, schq);
txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
}
}
mutex_unlock(&rvu->rsrc_lock);
/* Sync cached info for this LF in NDC-TX to LLC/DRAM */
rvu_write64(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12) | nixlf);
err = rvu_poll_reg(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12), true);
if (err)
dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf);
return 0;
}
static int nix_txschq_free_one(struct rvu *rvu,
struct nix_txsch_free_req *req)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
int lvl, schq, nixlf, blkaddr;
struct nix_txsch *txsch;
struct nix_hw *nix_hw;
u32 *pfvf_map;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
if (nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
lvl = req->schq_lvl;
schq = req->schq;
txsch = &nix_hw->txsch[lvl];
if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max)
return 0;
pfvf_map = txsch->pfvf_map;
mutex_lock(&rvu->rsrc_lock);
if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) {
mutex_unlock(&rvu->rsrc_lock);
goto err;
}
/* Flush if it is a SMQ. Onus of disabling
* TL2/3 queue links before SMQ flush is on user
*/
if (lvl == NIX_TXSCH_LVL_SMQ)
nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
/* Free the resource */
rvu_free_rsrc(&txsch->schq, schq);
txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
mutex_unlock(&rvu->rsrc_lock);
return 0;
err:
return NIX_AF_ERR_TLX_INVALID;
}
int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu,
struct nix_txsch_free_req *req,
struct msg_rsp *rsp)
{
if (req->flags & TXSCHQ_FREE_ALL)
return nix_txschq_free(rvu, req->hdr.pcifunc);
else
return nix_txschq_free_one(rvu, req);
}
static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr,
int lvl, u64 reg, u64 regval)
{
u64 regbase = reg & 0xFFFF;
u16 schq, parent;
if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg))
return false;
schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
/* Check if this schq belongs to this PF/VF or not */
if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq))
return false;
parent = (regval >> 16) & 0x1FF;
/* Validate MDQ's TL4 parent */
if (regbase == NIX_AF_MDQX_PARENT(0) &&
!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent))
return false;
/* Validate TL4's TL3 parent */
if (regbase == NIX_AF_TL4X_PARENT(0) &&
!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent))
return false;
/* Validate TL3's TL2 parent */
if (regbase == NIX_AF_TL3X_PARENT(0) &&
!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent))
return false;
/* Validate TL2's TL1 parent */
if (regbase == NIX_AF_TL2X_PARENT(0) &&
!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent))
return false;
return true;
}
static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg)
{
u64 regbase;
if (hw->cap.nix_shaping)
return true;
/* If shaping and coloring is not supported, then
* *_CIR and *_PIR registers should not be configured.
*/
regbase = reg & 0xFFFF;
switch (lvl) {
case NIX_TXSCH_LVL_TL1:
if (regbase == NIX_AF_TL1X_CIR(0))
return false;
break;
case NIX_TXSCH_LVL_TL2:
if (regbase == NIX_AF_TL2X_CIR(0) ||
regbase == NIX_AF_TL2X_PIR(0))
return false;
break;
case NIX_TXSCH_LVL_TL3:
if (regbase == NIX_AF_TL3X_CIR(0) ||
regbase == NIX_AF_TL3X_PIR(0))
return false;
break;
case NIX_TXSCH_LVL_TL4:
if (regbase == NIX_AF_TL4X_CIR(0) ||
regbase == NIX_AF_TL4X_PIR(0))
return false;
break;
}
return true;
}
static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw,
u16 pcifunc, int blkaddr)
{
u32 *pfvf_map;
int schq;
schq = nix_get_tx_link(rvu, pcifunc);
pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map;
/* Skip if PF has already done the config */
if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE)
return;
rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq),
(TXSCH_TL1_DFLT_RR_PRIO << 1));
rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq),
TXSCH_TL1_DFLT_RR_QTM);
rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00);
pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE);
}
int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu,
struct nix_txschq_config *req,
struct msg_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
u64 reg, regval, schq_regbase;
struct nix_txsch *txsch;
struct nix_hw *nix_hw;
int blkaddr, idx, err;
int nixlf, schq;
u32 *pfvf_map;
if (req->lvl >= NIX_TXSCH_LVL_CNT ||
req->num_regs > MAX_REGS_PER_MBOX_MSG)
return NIX_AF_INVAL_TXSCHQ_CFG;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
if (err)
return err;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
txsch = &nix_hw->txsch[req->lvl];
pfvf_map = txsch->pfvf_map;
if (req->lvl >= hw->cap.nix_tx_aggr_lvl &&
pcifunc & RVU_PFVF_FUNC_MASK) {
mutex_lock(&rvu->rsrc_lock);
if (req->lvl == NIX_TXSCH_LVL_TL1)
nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr);
mutex_unlock(&rvu->rsrc_lock);
return 0;
}
for (idx = 0; idx < req->num_regs; idx++) {
reg = req->reg[idx];
regval = req->regval[idx];
schq_regbase = reg & 0xFFFF;
if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr,
txsch->lvl, reg, regval))
return NIX_AF_INVAL_TXSCHQ_CFG;
/* Check if shaping and coloring is supported */
if (!is_txschq_shaping_valid(hw, req->lvl, reg))
continue;
/* Replace PF/VF visible NIXLF slot with HW NIXLF id */
if (schq_regbase == NIX_AF_SMQX_CFG(0)) {
nixlf = rvu_get_lf(rvu, &hw->block[blkaddr],
pcifunc, 0);
regval &= ~(0x7FULL << 24);
regval |= ((u64)nixlf << 24);
}
/* Clear 'BP_ENA' config, if it's not allowed */
if (!hw->cap.nix_tx_link_bp) {
if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) ||
(schq_regbase & 0xFF00) ==
NIX_AF_TL3_TL2X_LINKX_CFG(0, 0))
regval &= ~BIT_ULL(13);
}
/* Mark config as done for TL1 by PF */
if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) &&
schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) {
schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
mutex_lock(&rvu->rsrc_lock);
pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq],
NIX_TXSCHQ_CFG_DONE);
mutex_unlock(&rvu->rsrc_lock);
}
/* SMQ flush is special hence split register writes such
* that flush first and write rest of the bits later.
*/
if (schq_regbase == NIX_AF_SMQX_CFG(0) &&
(regval & BIT_ULL(49))) {
schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
regval &= ~BIT_ULL(49);
}
rvu_write64(rvu, blkaddr, reg, regval);
}
return 0;
}
static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr,
struct nix_vtag_config *req)
{
u64 regval = req->vtag_size;
if (req->rx.vtag_type > NIX_AF_LFX_RX_VTAG_TYPE7 ||
req->vtag_size > VTAGSIZE_T8)
return -EINVAL;
/* RX VTAG Type 7 reserved for vf vlan */
if (req->rx.vtag_type == NIX_AF_LFX_RX_VTAG_TYPE7)
return NIX_AF_ERR_RX_VTAG_INUSE;
if (req->rx.capture_vtag)
regval |= BIT_ULL(5);
if (req->rx.strip_vtag)
regval |= BIT_ULL(4);
rvu_write64(rvu, blkaddr,
NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval);
return 0;
}
static int nix_tx_vtag_free(struct rvu *rvu, int blkaddr,
u16 pcifunc, int index)
{
struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
struct nix_txvlan *vlan = &nix_hw->txvlan;
if (vlan->entry2pfvf_map[index] != pcifunc)
return NIX_AF_ERR_PARAM;
rvu_write64(rvu, blkaddr,
NIX_AF_TX_VTAG_DEFX_DATA(index), 0x0ull);
rvu_write64(rvu, blkaddr,
NIX_AF_TX_VTAG_DEFX_CTL(index), 0x0ull);
vlan->entry2pfvf_map[index] = 0;
rvu_free_rsrc(&vlan->rsrc, index);
return 0;
}
static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc)
{
struct nix_txvlan *vlan;
struct nix_hw *nix_hw;
int index, blkaddr;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (blkaddr < 0)
return;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
vlan = &nix_hw->txvlan;
mutex_lock(&vlan->rsrc_lock);
/* Scan all the entries and free the ones mapped to 'pcifunc' */
for (index = 0; index < vlan->rsrc.max; index++) {
if (vlan->entry2pfvf_map[index] == pcifunc)
nix_tx_vtag_free(rvu, blkaddr, pcifunc, index);
}
mutex_unlock(&vlan->rsrc_lock);
}
static int nix_tx_vtag_alloc(struct rvu *rvu, int blkaddr,
u64 vtag, u8 size)
{
struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
struct nix_txvlan *vlan = &nix_hw->txvlan;
u64 regval;
int index;
mutex_lock(&vlan->rsrc_lock);
index = rvu_alloc_rsrc(&vlan->rsrc);
if (index < 0) {
mutex_unlock(&vlan->rsrc_lock);
return index;
}
mutex_unlock(&vlan->rsrc_lock);
regval = size ? vtag : vtag << 32;
rvu_write64(rvu, blkaddr,
NIX_AF_TX_VTAG_DEFX_DATA(index), regval);
rvu_write64(rvu, blkaddr,
NIX_AF_TX_VTAG_DEFX_CTL(index), size);
return index;
}
static int nix_tx_vtag_decfg(struct rvu *rvu, int blkaddr,
struct nix_vtag_config *req)
{
struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
struct nix_txvlan *vlan = &nix_hw->txvlan;
u16 pcifunc = req->hdr.pcifunc;
int idx0 = req->tx.vtag0_idx;
int idx1 = req->tx.vtag1_idx;
int err = 0;
if (req->tx.free_vtag0 && req->tx.free_vtag1)
if (vlan->entry2pfvf_map[idx0] != pcifunc ||
vlan->entry2pfvf_map[idx1] != pcifunc)
return NIX_AF_ERR_PARAM;
mutex_lock(&vlan->rsrc_lock);
if (req->tx.free_vtag0) {
err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx0);
if (err)
goto exit;
}
if (req->tx.free_vtag1)
err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx1);
exit:
mutex_unlock(&vlan->rsrc_lock);
return err;
}
static int nix_tx_vtag_cfg(struct rvu *rvu, int blkaddr,
struct nix_vtag_config *req,
struct nix_vtag_config_rsp *rsp)
{
struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
struct nix_txvlan *vlan = &nix_hw->txvlan;
u16 pcifunc = req->hdr.pcifunc;
if (req->tx.cfg_vtag0) {
rsp->vtag0_idx =
nix_tx_vtag_alloc(rvu, blkaddr,
req->tx.vtag0, req->vtag_size);
if (rsp->vtag0_idx < 0)
return NIX_AF_ERR_TX_VTAG_NOSPC;
vlan->entry2pfvf_map[rsp->vtag0_idx] = pcifunc;
}
if (req->tx.cfg_vtag1) {
rsp->vtag1_idx =
nix_tx_vtag_alloc(rvu, blkaddr,
req->tx.vtag1, req->vtag_size);
if (rsp->vtag1_idx < 0)
goto err_free;
vlan->entry2pfvf_map[rsp->vtag1_idx] = pcifunc;
}
return 0;
err_free:
if (req->tx.cfg_vtag0)
nix_tx_vtag_free(rvu, blkaddr, pcifunc, rsp->vtag0_idx);
return NIX_AF_ERR_TX_VTAG_NOSPC;
}
int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu,
struct nix_vtag_config *req,
struct nix_vtag_config_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
int blkaddr, nixlf, err;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
if (err)
return err;
if (req->cfg_type) {
/* rx vtag configuration */
err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req);
if (err)
return NIX_AF_ERR_PARAM;
} else {
/* tx vtag configuration */
if ((req->tx.cfg_vtag0 || req->tx.cfg_vtag1) &&
(req->tx.free_vtag0 || req->tx.free_vtag1))
return NIX_AF_ERR_PARAM;
if (req->tx.cfg_vtag0 || req->tx.cfg_vtag1)
return nix_tx_vtag_cfg(rvu, blkaddr, req, rsp);
if (req->tx.free_vtag0 || req->tx.free_vtag1)
return nix_tx_vtag_decfg(rvu, blkaddr, req);
}
return 0;
}
static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw,
int mce, u8 op, u16 pcifunc, int next, bool eol)
{
struct nix_aq_enq_req aq_req;
int err;
aq_req.hdr.pcifunc = 0;
aq_req.ctype = NIX_AQ_CTYPE_MCE;
aq_req.op = op;
aq_req.qidx = mce;
/* Forward bcast pkts to RQ0, RSS not needed */
aq_req.mce.op = 0;
aq_req.mce.index = 0;
aq_req.mce.eol = eol;
aq_req.mce.pf_func = pcifunc;
aq_req.mce.next = next;
/* All fields valid */
*(u64 *)(&aq_req.mce_mask) = ~0ULL;
err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, &aq_req, NULL);
if (err) {
dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n",
rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
return err;
}
return 0;
}
static int nix_update_mce_list(struct nix_mce_list *mce_list,
u16 pcifunc, bool add)
{
struct mce *mce, *tail = NULL;
bool delete = false;
/* Scan through the current list */
hlist_for_each_entry(mce, &mce_list->head, node) {
/* If already exists, then delete */
if (mce->pcifunc == pcifunc && !add) {
delete = true;
break;
}
tail = mce;
}
if (delete) {
hlist_del(&mce->node);
kfree(mce);
mce_list->count--;
return 0;
}
if (!add)
return 0;
/* Add a new one to the list, at the tail */
mce = kzalloc(sizeof(*mce), GFP_KERNEL);
if (!mce)
return -ENOMEM;
mce->pcifunc = pcifunc;
if (!tail)
hlist_add_head(&mce->node, &mce_list->head);
else
hlist_add_behind(&mce->node, &tail->node);
mce_list->count++;
return 0;
}
int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add)
{
int err = 0, idx, next_idx, last_idx;
struct nix_mce_list *mce_list;
struct nix_mcast *mcast;
struct nix_hw *nix_hw;
struct rvu_pfvf *pfvf;
struct mce *mce;
int blkaddr;
/* Broadcast pkt replication is not needed for AF's VFs, hence skip */
if (is_afvf(pcifunc))
return 0;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (blkaddr < 0)
return 0;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return 0;
mcast = &nix_hw->mcast;
/* Get this PF/VF func's MCE index */
pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
idx = pfvf->bcast_mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK);
mce_list = &pfvf->bcast_mce_list;
if (idx > (pfvf->bcast_mce_idx + mce_list->max)) {
dev_err(rvu->dev,
"%s: Idx %d > max MCE idx %d, for PF%d bcast list\n",
__func__, idx, mce_list->max,
pcifunc >> RVU_PFVF_PF_SHIFT);
return -EINVAL;
}
mutex_lock(&mcast->mce_lock);
err = nix_update_mce_list(mce_list, pcifunc, add);
if (err)
goto end;
/* Disable MCAM entry in NPC */
if (!mce_list->count) {
rvu_npc_enable_bcast_entry(rvu, pcifunc, false);
goto end;
}
/* Dump the updated list to HW */
idx = pfvf->bcast_mce_idx;
last_idx = idx + mce_list->count - 1;
hlist_for_each_entry(mce, &mce_list->head, node) {
if (idx > last_idx)
break;
next_idx = idx + 1;
/* EOL should be set in last MCE */
err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE,
mce->pcifunc, next_idx,
(next_idx > last_idx) ? true : false);
if (err)
goto end;
idx++;
}
end:
mutex_unlock(&mcast->mce_lock);
return err;
}
static int nix_setup_bcast_tables(struct rvu *rvu, struct nix_hw *nix_hw)
{
struct nix_mcast *mcast = &nix_hw->mcast;
int err, pf, numvfs, idx;
struct rvu_pfvf *pfvf;
u16 pcifunc;
u64 cfg;
/* Skip PF0 (i.e AF) */
for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) {
cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
/* If PF is not enabled, nothing to do */
if (!((cfg >> 20) & 0x01))
continue;
/* Get numVFs attached to this PF */
numvfs = (cfg >> 12) & 0xFF;
pfvf = &rvu->pf[pf];
/* This NIX0/1 block mapped to PF ? */
if (pfvf->nix_blkaddr != nix_hw->blkaddr)
continue;
/* Save the start MCE */
pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1);
nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1);
for (idx = 0; idx < (numvfs + 1); idx++) {
/* idx-0 is for PF, followed by VFs */
pcifunc = (pf << RVU_PFVF_PF_SHIFT);
pcifunc |= idx;
/* Add dummy entries now, so that we don't have to check
* for whether AQ_OP should be INIT/WRITE later on.
* Will be updated when a NIXLF is attached/detached to
* these PF/VFs.
*/
err = nix_blk_setup_mce(rvu, nix_hw,
pfvf->bcast_mce_idx + idx,
NIX_AQ_INSTOP_INIT,
pcifunc, 0, true);
if (err)
return err;
}
}
return 0;
}
static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
{
struct nix_mcast *mcast = &nix_hw->mcast;
struct rvu_hwinfo *hw = rvu->hw;
int err, size;
size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F;
size = (1ULL << size);
/* Alloc memory for multicast/mirror replication entries */
err = qmem_alloc(rvu->dev, &mcast->mce_ctx,
(256UL << MC_TBL_SIZE), size);
if (err)
return -ENOMEM;
rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE,
(u64)mcast->mce_ctx->iova);
/* Set max list length equal to max no of VFs per PF + PF itself */
rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG,
BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE);
/* Alloc memory for multicast replication buffers */
size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF;
err = qmem_alloc(rvu->dev, &mcast->mcast_buf,
(8UL << MC_BUF_CNT), size);
if (err)
return -ENOMEM;
rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE,
(u64)mcast->mcast_buf->iova);
/* Alloc pkind for NIX internal RX multicast/mirror replay */
mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc);
rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG,
BIT_ULL(63) | (mcast->replay_pkind << 24) |
BIT_ULL(20) | MC_BUF_CNT);
mutex_init(&mcast->mce_lock);
return nix_setup_bcast_tables(rvu, nix_hw);
}
static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw)
{
struct nix_txvlan *vlan = &nix_hw->txvlan;
int err;
/* Allocate resource bimap for tx vtag def registers*/
vlan->rsrc.max = NIX_TX_VTAG_DEF_MAX;
err = rvu_alloc_bitmap(&vlan->rsrc);
if (err)
return -ENOMEM;
/* Alloc memory for saving entry to RVU PFFUNC allocation mapping */
vlan->entry2pfvf_map = devm_kcalloc(rvu->dev, vlan->rsrc.max,
sizeof(u16), GFP_KERNEL);
if (!vlan->entry2pfvf_map)
goto free_mem;
mutex_init(&vlan->rsrc_lock);
return 0;
free_mem:
kfree(vlan->rsrc.bmap);
return -ENOMEM;
}
static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
{
struct nix_txsch *txsch;
int err, lvl, schq;
u64 cfg, reg;
/* Get scheduler queue count of each type and alloc
* bitmap for each for alloc/free/attach operations.
*/
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
txsch = &nix_hw->txsch[lvl];
txsch->lvl = lvl;
switch (lvl) {
case NIX_TXSCH_LVL_SMQ:
reg = NIX_AF_MDQ_CONST;
break;
case NIX_TXSCH_LVL_TL4:
reg = NIX_AF_TL4_CONST;
break;
case NIX_TXSCH_LVL_TL3:
reg = NIX_AF_TL3_CONST;
break;
case NIX_TXSCH_LVL_TL2:
reg = NIX_AF_TL2_CONST;
break;
case NIX_TXSCH_LVL_TL1:
reg = NIX_AF_TL1_CONST;
break;
}
cfg = rvu_read64(rvu, blkaddr, reg);
txsch->schq.max = cfg & 0xFFFF;
err = rvu_alloc_bitmap(&txsch->schq);
if (err)
return err;
/* Allocate memory for scheduler queues to
* PF/VF pcifunc mapping info.
*/
txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max,
sizeof(u32), GFP_KERNEL);
if (!txsch->pfvf_map)
return -ENOMEM;
for (schq = 0; schq < txsch->schq.max; schq++)
txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
}
return 0;
}
int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw,
int blkaddr, u32 cfg)
{
int fmt_idx;
for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) {
if (nix_hw->mark_format.cfg[fmt_idx] == cfg)
return fmt_idx;
}
if (fmt_idx >= nix_hw->mark_format.total)
return -ERANGE;
rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg);
nix_hw->mark_format.cfg[fmt_idx] = cfg;
nix_hw->mark_format.in_use++;
return fmt_idx;
}
static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw,
int blkaddr)
{
u64 cfgs[] = {
[NIX_MARK_CFG_IP_DSCP_RED] = 0x10003,
[NIX_MARK_CFG_IP_DSCP_YELLOW] = 0x11200,
[NIX_MARK_CFG_IP_DSCP_YELLOW_RED] = 0x11203,
[NIX_MARK_CFG_IP_ECN_RED] = 0x6000c,
[NIX_MARK_CFG_IP_ECN_YELLOW] = 0x60c00,
[NIX_MARK_CFG_IP_ECN_YELLOW_RED] = 0x60c0c,
[NIX_MARK_CFG_VLAN_DEI_RED] = 0x30008,
[NIX_MARK_CFG_VLAN_DEI_YELLOW] = 0x30800,
[NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808,
};
int i, rc;
u64 total;
total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8;
nix_hw->mark_format.total = (u8)total;
nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32),
GFP_KERNEL);
if (!nix_hw->mark_format.cfg)
return -ENOMEM;
for (i = 0; i < NIX_MARK_CFG_MAX; i++) {
rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]);
if (rc < 0)
dev_err(rvu->dev, "Err %d in setup mark format %d\n",
i, rc);
}
return 0;
}
int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
int i, nixlf, blkaddr, err;
u64 stats;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
if (err)
return err;
/* Get stats count supported by HW */
stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
/* Reset tx stats */
for (i = 0; i < ((stats >> 24) & 0xFF); i++)
rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0);
/* Reset rx stats */
for (i = 0; i < ((stats >> 32) & 0xFF); i++)
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0);
return 0;
}
/* Returns the ALG index to be set into NPC_RX_ACTION */
static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg)
{
int i;
/* Scan over exiting algo entries to find a match */
for (i = 0; i < nix_hw->flowkey.in_use; i++)
if (nix_hw->flowkey.flowkey[i] == flow_cfg)
return i;
return -ERANGE;
}
static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg)
{
int idx, nr_field, key_off, field_marker, keyoff_marker;
int max_key_off, max_bit_pos, group_member;
struct nix_rx_flowkey_alg *field;
struct nix_rx_flowkey_alg tmp;
u32 key_type, valid_key;
if (!alg)
return -EINVAL;
#define FIELDS_PER_ALG 5
#define MAX_KEY_OFF 40
/* Clear all fields */
memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG);
/* Each of the 32 possible flow key algorithm definitions should
* fall into above incremental config (except ALG0). Otherwise a
* single NPC MCAM entry is not sufficient for supporting RSS.
*
* If a different definition or combination needed then NPC MCAM
* has to be programmed to filter such pkts and it's action should
* point to this definition to calculate flowtag or hash.
*
* The `for loop` goes over _all_ protocol field and the following
* variables depicts the state machine forward progress logic.
*
* keyoff_marker - Enabled when hash byte length needs to be accounted
* in field->key_offset update.
* field_marker - Enabled when a new field needs to be selected.
* group_member - Enabled when protocol is part of a group.
*/
keyoff_marker = 0; max_key_off = 0; group_member = 0;
nr_field = 0; key_off = 0; field_marker = 1;
field = &tmp; max_bit_pos = fls(flow_cfg);
for (idx = 0;
idx < max_bit_pos && nr_field < FIELDS_PER_ALG &&
key_off < MAX_KEY_OFF; idx++) {
key_type = BIT(idx);
valid_key = flow_cfg & key_type;
/* Found a field marker, reset the field values */
if (field_marker)
memset(&tmp, 0, sizeof(tmp));
field_marker = true;
keyoff_marker = true;
switch (key_type) {
case NIX_FLOW_KEY_TYPE_PORT:
field->sel_chan = true;
/* This should be set to 1, when SEL_CHAN is set */
field->bytesm1 = 1;
break;
case NIX_FLOW_KEY_TYPE_IPV4_PROTO:
field->lid = NPC_LID_LC;
field->hdr_offset = 9; /* offset */
field->bytesm1 = 0; /* 1 byte */
field->ltype_match = NPC_LT_LC_IP;
field->ltype_mask = 0xF;
break;
case NIX_FLOW_KEY_TYPE_IPV4:
case NIX_FLOW_KEY_TYPE_INNR_IPV4:
field->lid = NPC_LID_LC;
field->ltype_match = NPC_LT_LC_IP;
if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) {
field->lid = NPC_LID_LG;
field->ltype_match = NPC_LT_LG_TU_IP;
}
field->hdr_offset = 12; /* SIP offset */
field->bytesm1 = 7; /* SIP + DIP, 8 bytes */
field->ltype_mask = 0xF; /* Match only IPv4 */
keyoff_marker = false;
break;
case NIX_FLOW_KEY_TYPE_IPV6:
case NIX_FLOW_KEY_TYPE_INNR_IPV6:
field->lid = NPC_LID_LC;
field->ltype_match = NPC_LT_LC_IP6;
if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) {
field->lid = NPC_LID_LG;
field->ltype_match = NPC_LT_LG_TU_IP6;
}
field->hdr_offset = 8; /* SIP offset */
field->bytesm1 = 31; /* SIP + DIP, 32 bytes */
field->ltype_mask = 0xF; /* Match only IPv6 */
break;
case NIX_FLOW_KEY_TYPE_TCP:
case NIX_FLOW_KEY_TYPE_UDP:
case NIX_FLOW_KEY_TYPE_SCTP:
case NIX_FLOW_KEY_TYPE_INNR_TCP:
case NIX_FLOW_KEY_TYPE_INNR_UDP:
case NIX_FLOW_KEY_TYPE_INNR_SCTP:
field->lid = NPC_LID_LD;
if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP ||
key_type == NIX_FLOW_KEY_TYPE_INNR_UDP ||
key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP)
field->lid = NPC_LID_LH;
field->bytesm1 = 3; /* Sport + Dport, 4 bytes */
/* Enum values for NPC_LID_LD and NPC_LID_LG are same,
* so no need to change the ltype_match, just change
* the lid for inner protocols
*/
BUILD_BUG_ON((int)NPC_LT_LD_TCP !=
(int)NPC_LT_LH_TU_TCP);
BUILD_BUG_ON((int)NPC_LT_LD_UDP !=
(int)NPC_LT_LH_TU_UDP);
BUILD_BUG_ON((int)NPC_LT_LD_SCTP !=
(int)NPC_LT_LH_TU_SCTP);
if ((key_type == NIX_FLOW_KEY_TYPE_TCP ||
key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) &&
valid_key) {
field->ltype_match |= NPC_LT_LD_TCP;
group_member = true;
} else if ((key_type == NIX_FLOW_KEY_TYPE_UDP ||
key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) &&
valid_key) {
field->ltype_match |= NPC_LT_LD_UDP;
group_member = true;
} else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP ||
key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) &&
valid_key) {
field->ltype_match |= NPC_LT_LD_SCTP;
group_member = true;
}
field->ltype_mask = ~field->ltype_match;
if (key_type == NIX_FLOW_KEY_TYPE_SCTP ||
key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) {
/* Handle the case where any of the group item
* is enabled in the group but not the final one
*/
if (group_member) {
valid_key = true;
group_member = false;
}
} else {
field_marker = false;
keyoff_marker = false;
}
break;
case NIX_FLOW_KEY_TYPE_NVGRE:
field->lid = NPC_LID_LD;
field->hdr_offset = 4; /* VSID offset */
field->bytesm1 = 2;
field->ltype_match = NPC_LT_LD_NVGRE;
field->ltype_mask = 0xF;
break;
case NIX_FLOW_KEY_TYPE_VXLAN:
case NIX_FLOW_KEY_TYPE_GENEVE:
field->lid = NPC_LID_LE;
field->bytesm1 = 2;
field->hdr_offset = 4;
field->ltype_mask = 0xF;
field_marker = false;
keyoff_marker = false;
if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) {
field->ltype_match |= NPC_LT_LE_VXLAN;
group_member = true;
}
if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) {
field->ltype_match |= NPC_LT_LE_GENEVE;
group_member = true;
}
if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) {
if (group_member) {
field->ltype_mask = ~field->ltype_match;
field_marker = true;
keyoff_marker = true;
valid_key = true;
group_member = false;
}
}
break;
case NIX_FLOW_KEY_TYPE_ETH_DMAC:
case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC:
field->lid = NPC_LID_LA;
field->ltype_match = NPC_LT_LA_ETHER;
if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) {
field->lid = NPC_LID_LF;
field->ltype_match = NPC_LT_LF_TU_ETHER;
}
field->hdr_offset = 0;
field->bytesm1 = 5; /* DMAC 6 Byte */
field->ltype_mask = 0xF;
break;
case NIX_FLOW_KEY_TYPE_IPV6_EXT:
field->lid = NPC_LID_LC;
field->hdr_offset = 40; /* IPV6 hdr */
field->bytesm1 = 0; /* 1 Byte ext hdr*/
field->ltype_match = NPC_LT_LC_IP6_EXT;
field->ltype_mask = 0xF;
break;
case NIX_FLOW_KEY_TYPE_GTPU:
field->lid = NPC_LID_LE;
field->hdr_offset = 4;
field->bytesm1 = 3; /* 4 bytes TID*/
field->ltype_match = NPC_LT_LE_GTPU;
field->ltype_mask = 0xF;
break;
case NIX_FLOW_KEY_TYPE_VLAN:
field->lid = NPC_LID_LB;
field->hdr_offset = 2; /* Skip TPID (2-bytes) */
field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */
field->ltype_match = NPC_LT_LB_CTAG;
field->ltype_mask = 0xF;
field->fn_mask = 1; /* Mask out the first nibble */
break;
}
field->ena = 1;
/* Found a valid flow key type */
if (valid_key) {
field->key_offset = key_off;
memcpy(&alg[nr_field], field, sizeof(*field));
max_key_off = max(max_key_off, field->bytesm1 + 1);
/* Found a field marker, get the next field */
if (field_marker)
nr_field++;
}
/* Found a keyoff marker, update the new key_off */
if (keyoff_marker) {
key_off += max_key_off;
max_key_off = 0;
}
}
/* Processed all the flow key types */
if (idx == max_bit_pos && key_off <= MAX_KEY_OFF)
return 0;
else
return NIX_AF_ERR_RSS_NOSPC_FIELD;
}
static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg)
{
u64 field[FIELDS_PER_ALG];
struct nix_hw *hw;
int fid, rc;
hw = get_nix_hw(rvu->hw, blkaddr);
if (!hw)
return -EINVAL;
/* No room to add new flow hash algoritham */
if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX)
return NIX_AF_ERR_RSS_NOSPC_ALGO;
/* Generate algo fields for the given flow_cfg */
rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg);
if (rc)
return rc;
/* Update ALGX_FIELDX register with generated fields */
for (fid = 0; fid < FIELDS_PER_ALG; fid++)
rvu_write64(rvu, blkaddr,
NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use,
fid), field[fid]);
/* Store the flow_cfg for futher lookup */
rc = hw->flowkey.in_use;
hw->flowkey.flowkey[rc] = flow_cfg;
hw->flowkey.in_use++;
return rc;
}
int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu,
struct nix_rss_flowkey_cfg *req,
struct nix_rss_flowkey_cfg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
int alg_idx, nixlf, blkaddr;
struct nix_hw *nix_hw;
int err;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
if (err)
return err;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg);
/* Failed to get algo index from the exiting list, reserve new */
if (alg_idx < 0) {
alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr,
req->flowkey_cfg);
if (alg_idx < 0)
return alg_idx;
}
rsp->alg_idx = alg_idx;
rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group,
alg_idx, req->mcam_index);
return 0;
}
static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr)
{
u32 flowkey_cfg, minkey_cfg;
int alg, fid, rc;
/* Disable all flow key algx fieldx */
for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) {
for (fid = 0; fid < FIELDS_PER_ALG; fid++)
rvu_write64(rvu, blkaddr,
NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid),
0);
}
/* IPv4/IPv6 SIP/DIPs */
flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
minkey_cfg = flowkey_cfg;
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
NIX_FLOW_KEY_TYPE_UDP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
NIX_FLOW_KEY_TYPE_SCTP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP |
NIX_FLOW_KEY_TYPE_SCTP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
/* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP;
rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
if (rc < 0)
return rc;
return 0;
}
int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu,
struct nix_set_mac_addr *req,
struct msg_rsp *rsp)
{
bool from_vf = req->hdr.pcifunc & RVU_PFVF_FUNC_MASK;
u16 pcifunc = req->hdr.pcifunc;
int blkaddr, nixlf, err;
struct rvu_pfvf *pfvf;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
if (err)
return err;
pfvf = rvu_get_pfvf(rvu, pcifunc);
/* VF can't overwrite admin(PF) changes */
if (from_vf && pfvf->pf_set_vf_cfg)
return -EPERM;
ether_addr_copy(pfvf->mac_addr, req->mac_addr);
rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
pfvf->rx_chan_base, req->mac_addr);
return 0;
}
int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu,
struct msg_req *req,
struct nix_get_mac_addr_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
struct rvu_pfvf *pfvf;
if (!is_nixlf_attached(rvu, pcifunc))
return NIX_AF_ERR_AF_LF_INVALID;
pfvf = rvu_get_pfvf(rvu, pcifunc);
ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
return 0;
}
int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req,
struct msg_rsp *rsp)
{
bool allmulti = false, disable_promisc = false;
u16 pcifunc = req->hdr.pcifunc;
int blkaddr, nixlf, err;
struct rvu_pfvf *pfvf;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
if (err)
return err;
pfvf = rvu_get_pfvf(rvu, pcifunc);
if (req->mode & NIX_RX_MODE_PROMISC)
allmulti = false;
else if (req->mode & NIX_RX_MODE_ALLMULTI)
allmulti = true;
else
disable_promisc = true;
if (disable_promisc)
rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf);
else
rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
pfvf->rx_chan_base, allmulti);
return 0;
}
static void nix_find_link_frs(struct rvu *rvu,
struct nix_frs_cfg *req, u16 pcifunc)
{
int pf = rvu_get_pf(pcifunc);
struct rvu_pfvf *pfvf;
int maxlen, minlen;
int numvfs, hwvf;
int vf;
/* Update with requester's min/max lengths */
pfvf = rvu_get_pfvf(rvu, pcifunc);
pfvf->maxlen = req->maxlen;
if (req->update_minlen)
pfvf->minlen = req->minlen;
maxlen = req->maxlen;
minlen = req->update_minlen ? req->minlen : 0;
/* Get this PF's numVFs and starting hwvf */
rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
/* For each VF, compare requested max/minlen */
for (vf = 0; vf < numvfs; vf++) {
pfvf = &rvu->hwvf[hwvf + vf];
if (pfvf->maxlen > maxlen)
maxlen = pfvf->maxlen;
if (req->update_minlen &&
pfvf->minlen && pfvf->minlen < minlen)
minlen = pfvf->minlen;
}
/* Compare requested max/minlen with PF's max/minlen */
pfvf = &rvu->pf[pf];
if (pfvf->maxlen > maxlen)
maxlen = pfvf->maxlen;
if (req->update_minlen &&
pfvf->minlen && pfvf->minlen < minlen)
minlen = pfvf->minlen;
/* Update the request with max/min PF's and it's VF's max/min */
req->maxlen = maxlen;
if (req->update_minlen)
req->minlen = minlen;
}
int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req,
struct msg_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
int pf = rvu_get_pf(pcifunc);
int blkaddr, schq, link = -1;
struct nix_txsch *txsch;
u64 cfg, lmac_fifo_len;
struct nix_hw *nix_hw;
u8 cgx = 0, lmac = 0;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
if (!req->sdp_link && req->maxlen > NIC_HW_MAX_FRS)
return NIX_AF_ERR_FRS_INVALID;
if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS)
return NIX_AF_ERR_FRS_INVALID;
/* Check if requester wants to update SMQ's */
if (!req->update_smq)
goto rx_frscfg;
/* Update min/maxlen in each of the SMQ attached to this PF/VF */
txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
mutex_lock(&rvu->rsrc_lock);
for (schq = 0; schq < txsch->schq.max; schq++) {
if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
continue;
cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq));
cfg = (cfg & ~(0xFFFFULL << 8)) | ((u64)req->maxlen << 8);
if (req->update_minlen)
cfg = (cfg & ~0x7FULL) | ((u64)req->minlen & 0x7F);
rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq), cfg);
}
mutex_unlock(&rvu->rsrc_lock);
rx_frscfg:
/* Check if config is for SDP link */
if (req->sdp_link) {
if (!hw->sdp_links)
return NIX_AF_ERR_RX_LINK_INVALID;
link = hw->cgx_links + hw->lbk_links;
goto linkcfg;
}
/* Check if the request is from CGX mapped RVU PF */
if (is_pf_cgxmapped(rvu, pf)) {
/* Get CGX and LMAC to which this PF is mapped and find link */
rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac);
link = (cgx * hw->lmac_per_cgx) + lmac;
} else if (pf == 0) {
/* For VFs of PF0 ingress is LBK port, so config LBK link */
link = hw->cgx_links;
}
if (link < 0)
return NIX_AF_ERR_RX_LINK_INVALID;
nix_find_link_frs(rvu, req, pcifunc);
linkcfg:
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link));
cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16);
if (req->update_minlen)
cfg = (cfg & ~0xFFFFULL) | req->minlen;
rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg);
if (req->sdp_link || pf == 0)
return 0;
/* Update transmit credits for CGX links */
lmac_fifo_len =
CGX_FIFO_LEN / cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
cfg = rvu_read64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link));
cfg &= ~(0xFFFFFULL << 12);
cfg |= ((lmac_fifo_len - req->maxlen) / 16) << 12;
rvu_write64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg);
return 0;
}
int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req,
struct msg_rsp *rsp)
{
int nixlf, blkaddr, err;
u64 cfg;
err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr);
if (err)
return err;
cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf));
/* Set the interface configuration */
if (req->len_verify & BIT(0))
cfg |= BIT_ULL(41);
else
cfg &= ~BIT_ULL(41);
if (req->len_verify & BIT(1))
cfg |= BIT_ULL(40);
else
cfg &= ~BIT_ULL(40);
if (req->csum_verify & BIT(0))
cfg |= BIT_ULL(37);
else
cfg &= ~BIT_ULL(37);
rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg);
return 0;
}
static void nix_link_config(struct rvu *rvu, int blkaddr)
{
struct rvu_hwinfo *hw = rvu->hw;
int cgx, lmac_cnt, slink, link;
u64 tx_credits;
/* Set default min/max packet lengths allowed on NIX Rx links.
*
* With HW reset minlen value of 60byte, HW will treat ARP pkts
* as undersize and report them to SW as error pkts, hence
* setting it to 40 bytes.
*/
for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++) {
rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
NIC_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS);
}
if (hw->sdp_links) {
link = hw->cgx_links + hw->lbk_links;
rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
SDP_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS);
}
/* Set credits for Tx links assuming max packet length allowed.
* This will be reconfigured based on MTU set for PF/VF.
*/
for (cgx = 0; cgx < hw->cgx; cgx++) {
lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
tx_credits = ((CGX_FIFO_LEN / lmac_cnt) - NIC_HW_MAX_FRS) / 16;
/* Enable credits and set credit pkt count to max allowed */
tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
slink = cgx * hw->lmac_per_cgx;
for (link = slink; link < (slink + lmac_cnt); link++) {
rvu_write64(rvu, blkaddr,
NIX_AF_TX_LINKX_NORM_CREDIT(link),
tx_credits);
}
}
/* Set Tx credits for LBK link */
slink = hw->cgx_links;
for (link = slink; link < (slink + hw->lbk_links); link++) {
tx_credits = 1000; /* 10 * max LBK datarate = 10 * 100Gbps */
/* Enable credits and set credit pkt count to max allowed */
tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
rvu_write64(rvu, blkaddr,
NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits);
}
}
static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr)
{
int idx, err;
u64 status;
/* Start X2P bus calibration */
rvu_write64(rvu, blkaddr, NIX_AF_CFG,
rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9));
/* Wait for calibration to complete */
err = rvu_poll_reg(rvu, blkaddr,
NIX_AF_STATUS, BIT_ULL(10), false);
if (err) {
dev_err(rvu->dev, "NIX X2P bus calibration failed\n");
return err;
}
status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS);
/* Check if CGX devices are ready */
for (idx = 0; idx < rvu->cgx_cnt_max; idx++) {
/* Skip when cgx port is not available */
if (!rvu_cgx_pdata(idx, rvu) ||
(status & (BIT_ULL(16 + idx))))
continue;
dev_err(rvu->dev,
"CGX%d didn't respond to NIX X2P calibration\n", idx);
err = -EBUSY;
}
/* Check if LBK is ready */
if (!(status & BIT_ULL(19))) {
dev_err(rvu->dev,
"LBK didn't respond to NIX X2P calibration\n");
err = -EBUSY;
}
/* Clear 'calibrate_x2p' bit */
rvu_write64(rvu, blkaddr, NIX_AF_CFG,
rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9));
if (err || (status & 0x3FFULL))
dev_err(rvu->dev,
"NIX X2P calibration failed, status 0x%llx\n", status);
if (err)
return err;
return 0;
}
static int nix_aq_init(struct rvu *rvu, struct rvu_block *block)
{
u64 cfg;
int err;
/* Set admin queue endianness */
cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG);
#ifdef __BIG_ENDIAN
cfg |= BIT_ULL(8);
rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
#else
cfg &= ~BIT_ULL(8);
rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
#endif
/* Do not bypass NDC cache */
cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG);
cfg &= ~0x3FFEULL;
#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
/* Disable caching of SQB aka SQEs */
cfg |= 0x04ULL;
#endif
rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg);
/* Result structure can be followed by RQ/SQ/CQ context at
* RES + 128bytes and a write mask at RES + 256 bytes, depending on
* operation type. Alloc sufficient result memory for all operations.
*/
err = rvu_aq_alloc(rvu, &block->aq,
Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s),
ALIGN(sizeof(struct nix_aq_res_s), 128) + 256);
if (err)
return err;
rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE);
rvu_write64(rvu, block->addr,
NIX_AF_AQ_BASE, (u64)block->aq->inst->iova);
return 0;
}
static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw)
{
const struct npc_lt_def_cfg *ltdefs;
struct rvu_hwinfo *hw = rvu->hw;
int blkaddr = nix_hw->blkaddr;
struct rvu_block *block;
int err;
u64 cfg;
block = &hw->block[blkaddr];
if (is_rvu_96xx_B0(rvu)) {
/* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt
* internal state when conditional clocks are turned off.
* Hence enable them.
*/
rvu_write64(rvu, blkaddr, NIX_AF_CFG,
rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL);
/* Set chan/link to backpressure TL3 instead of TL2 */
rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01);
/* Disable SQ manager's sticky mode operation (set TM6 = 0)
* This sticky mode is known to cause SQ stalls when multiple
* SQs are mapped to same SMQ and transmitting pkts at a time.
*/
cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS);
cfg &= ~BIT_ULL(15);
rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg);
}
ltdefs = rvu->kpu.lt_def;
/* Calibrate X2P bus to check if CGX/LBK links are fine */
err = nix_calibrate_x2p(rvu, blkaddr);
if (err)
return err;
/* Set num of links of each type */
cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
hw->cgx = (cfg >> 12) & 0xF;
hw->lmac_per_cgx = (cfg >> 8) & 0xF;
hw->cgx_links = hw->cgx * hw->lmac_per_cgx;
hw->lbk_links = (cfg >> 24) & 0xF;
hw->sdp_links = 1;
/* Initialize admin queue */
err = nix_aq_init(rvu, block);
if (err)
return err;
/* Restore CINT timer delay to HW reset values */
rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL);
if (is_block_implemented(hw, blkaddr)) {
err = nix_setup_txschq(rvu, nix_hw, blkaddr);
if (err)
return err;
err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr);
if (err)
return err;
err = nix_setup_mcast(rvu, nix_hw, blkaddr);
if (err)
return err;
err = nix_setup_txvlan(rvu, nix_hw);
if (err)
return err;
/* Configure segmentation offload formats */
nix_setup_lso(rvu, nix_hw, blkaddr);
/* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info.
* This helps HW protocol checker to identify headers
* and validate length and checksums.
*/
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2,
(ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) |
ltdefs->rx_ol2.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4,
(ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) |
ltdefs->rx_oip4.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4,
(ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) |
ltdefs->rx_iip4.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6,
(ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) |
ltdefs->rx_oip6.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6,
(ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) |
ltdefs->rx_iip6.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP,
(ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) |
ltdefs->rx_otcp.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP,
(ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) |
ltdefs->rx_itcp.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP,
(ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) |
ltdefs->rx_oudp.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP,
(ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) |
ltdefs->rx_iudp.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP,
(ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) |
ltdefs->rx_osctp.ltype_mask);
rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP,
(ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) |
ltdefs->rx_isctp.ltype_mask);
err = nix_rx_flowkey_alg_cfg(rvu, blkaddr);
if (err)
return err;
/* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */
nix_link_config(rvu, blkaddr);
/* Enable Channel backpressure */
rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0));
}
return 0;
}
int rvu_nix_init(struct rvu *rvu)
{
struct rvu_hwinfo *hw = rvu->hw;
struct nix_hw *nix_hw;
int blkaddr = 0, err;
int i = 0;
hw->nix = devm_kcalloc(rvu->dev, MAX_NIX_BLKS, sizeof(struct nix_hw),
GFP_KERNEL);
if (!hw->nix)
return -ENOMEM;
blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
while (blkaddr) {
nix_hw = &hw->nix[i];
nix_hw->rvu = rvu;
nix_hw->blkaddr = blkaddr;
err = rvu_nix_block_init(rvu, nix_hw);
if (err)
return err;
blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
i++;
}
return 0;
}
static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr,
struct rvu_block *block)
{
struct nix_txsch *txsch;
struct nix_mcast *mcast;
struct nix_txvlan *vlan;
struct nix_hw *nix_hw;
int lvl;
rvu_aq_free(rvu, block->aq);
if (is_block_implemented(rvu->hw, blkaddr)) {
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return;
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
txsch = &nix_hw->txsch[lvl];
kfree(txsch->schq.bmap);
}
vlan = &nix_hw->txvlan;
kfree(vlan->rsrc.bmap);
mutex_destroy(&vlan->rsrc_lock);
devm_kfree(rvu->dev, vlan->entry2pfvf_map);
mcast = &nix_hw->mcast;
qmem_free(rvu->dev, mcast->mce_ctx);
qmem_free(rvu->dev, mcast->mcast_buf);
mutex_destroy(&mcast->mce_lock);
}
}
void rvu_nix_freemem(struct rvu *rvu)
{
struct rvu_hwinfo *hw = rvu->hw;
struct rvu_block *block;
int blkaddr = 0;
blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
while (blkaddr) {
block = &hw->block[blkaddr];
rvu_nix_block_freemem(rvu, blkaddr, block);
blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
}
}
int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
int nixlf, err;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
if (err)
return err;
rvu_npc_enable_default_entries(rvu, pcifunc, nixlf);
npc_mcam_enable_flows(rvu, pcifunc);
return rvu_cgx_start_stop_io(rvu, pcifunc, true);
}
int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
int nixlf, err;
err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
if (err)
return err;
rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
npc_mcam_disable_flows(rvu, pcifunc);
return rvu_cgx_start_stop_io(rvu, pcifunc, false);
}
void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
struct hwctx_disable_req ctx_req;
int err;
ctx_req.hdr.pcifunc = pcifunc;
/* Cleanup NPC MCAM entries, free Tx scheduler queues being used */
rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf);
nix_interface_deinit(rvu, pcifunc, nixlf);
nix_rx_sync(rvu, blkaddr);
nix_txschq_free(rvu, pcifunc);
rvu_cgx_start_stop_io(rvu, pcifunc, false);
if (pfvf->sq_ctx) {
ctx_req.ctype = NIX_AQ_CTYPE_SQ;
err = nix_lf_hwctx_disable(rvu, &ctx_req);
if (err)
dev_err(rvu->dev, "SQ ctx disable failed\n");
}
if (pfvf->rq_ctx) {
ctx_req.ctype = NIX_AQ_CTYPE_RQ;
err = nix_lf_hwctx_disable(rvu, &ctx_req);
if (err)
dev_err(rvu->dev, "RQ ctx disable failed\n");
}
if (pfvf->cq_ctx) {
ctx_req.ctype = NIX_AQ_CTYPE_CQ;
err = nix_lf_hwctx_disable(rvu, &ctx_req);
if (err)
dev_err(rvu->dev, "CQ ctx disable failed\n");
}
nix_ctx_free(rvu, pfvf);
}
#define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32)
static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable)
{
struct rvu_hwinfo *hw = rvu->hw;
struct rvu_block *block;
int blkaddr;
int nixlf;
u64 cfg;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
block = &hw->block[blkaddr];
nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
if (nixlf < 0)
return NIX_AF_ERR_AF_LF_INVALID;
cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf));
if (enable)
cfg |= NIX_AF_LFX_TX_CFG_PTP_EN;
else
cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN;
rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
return 0;
}
int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true);
}
int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false);
}
int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu,
struct nix_lso_format_cfg *req,
struct nix_lso_format_cfg_rsp *rsp)
{
u16 pcifunc = req->hdr.pcifunc;
struct nix_hw *nix_hw;
struct rvu_pfvf *pfvf;
int blkaddr, idx, f;
u64 reg;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
if (!pfvf->nixlf || blkaddr < 0)
return NIX_AF_ERR_AF_LF_INVALID;
nix_hw = get_nix_hw(rvu->hw, blkaddr);
if (!nix_hw)
return -EINVAL;
/* Find existing matching LSO format, if any */
for (idx = 0; idx < nix_hw->lso.in_use; idx++) {
for (f = 0; f < NIX_LSO_FIELD_MAX; f++) {
reg = rvu_read64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(idx, f));
if (req->fields[f] != (reg & req->field_mask))
break;
}
if (f == NIX_LSO_FIELD_MAX)
break;
}
if (idx < nix_hw->lso.in_use) {
/* Match found */
rsp->lso_format_idx = idx;
return 0;
}
if (nix_hw->lso.in_use == nix_hw->lso.total)
return NIX_AF_ERR_LSO_CFG_FAIL;
rsp->lso_format_idx = nix_hw->lso.in_use++;
for (f = 0; f < NIX_LSO_FIELD_MAX; f++)
rvu_write64(rvu, blkaddr,
NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f),
req->fields[f]);
return 0;
}
void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc)
{
bool from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK);
/* overwrite vf mac address with default_mac */
if (from_vf)
ether_addr_copy(pfvf->mac_addr, pfvf->default_mac);
}
Reference in New Issue View Git Blame Copy Permalink