5e37d8254e
Add header buffers and ethtool support to enable header split via the tcp-data-split flag in ethtool's ringparam config. A coherent dma memory is allocated for the header buffers. There is one header buffer per ring entry by calculating the offset to the header-buffers starting address. The header buffer is always copied directly into the skb and payload is always added as frags. When there is a header buffer overflow or the header length is 0, the driver places the whole unsplit packet in frags. When toggling header split, the driver will call gve_adjust_config to set its queues appropriately. If header split is enabled by the user and the max packet buffer size is no less than 4KB, driver will set the packet buffer size as 4KB to support TCP_ZEROCOPY_RECEIVE. Otherwise the driver will use the default 2KB as the packet buffer size. `ethtool -G <dev> tcp-data-split on/off` is the command to toggle header split. `ethtool -g <dev>` will show the status of header split with the field of `tcp-data-split`. Co-developed-by: Ziwei Xiao <ziweixiao@google.com> Signed-off-by: Ziwei Xiao <ziweixiao@google.com> Signed-off-by: Jeroen de Borst <jeroendb@google.com> Reviewed-by: Praveen Kaligineedi <pkaligineedi@google.com> Reviewed-by: Harshitha Ramamurthy <hramamurthy@google.com> Reviewed-by: Willem de Bruijn <willemb@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
122 lines
3.0 KiB
C
122 lines
3.0 KiB
C
// SPDX-License-Identifier: (GPL-2.0 OR MIT)
|
|
/* Google virtual Ethernet (gve) driver
|
|
*
|
|
* Copyright (C) 2015-2021 Google, Inc.
|
|
*/
|
|
|
|
#include "gve.h"
|
|
#include "gve_adminq.h"
|
|
#include "gve_utils.h"
|
|
|
|
bool gve_tx_was_added_to_block(struct gve_priv *priv, int queue_idx)
|
|
{
|
|
struct gve_notify_block *block =
|
|
&priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)];
|
|
|
|
return block->tx != NULL;
|
|
}
|
|
|
|
void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx)
|
|
{
|
|
struct gve_notify_block *block =
|
|
&priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)];
|
|
|
|
block->tx = NULL;
|
|
}
|
|
|
|
void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx)
|
|
{
|
|
unsigned int active_cpus = min_t(int, priv->num_ntfy_blks / 2,
|
|
num_online_cpus());
|
|
int ntfy_idx = gve_tx_idx_to_ntfy(priv, queue_idx);
|
|
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
|
|
struct gve_tx_ring *tx = &priv->tx[queue_idx];
|
|
|
|
block->tx = tx;
|
|
tx->ntfy_id = ntfy_idx;
|
|
netif_set_xps_queue(priv->dev, get_cpu_mask(ntfy_idx % active_cpus),
|
|
queue_idx);
|
|
}
|
|
|
|
bool gve_rx_was_added_to_block(struct gve_priv *priv, int queue_idx)
|
|
{
|
|
struct gve_notify_block *block =
|
|
&priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)];
|
|
|
|
return block->rx != NULL;
|
|
}
|
|
|
|
void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx)
|
|
{
|
|
struct gve_notify_block *block =
|
|
&priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)];
|
|
|
|
block->rx = NULL;
|
|
}
|
|
|
|
void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)
|
|
{
|
|
u32 ntfy_idx = gve_rx_idx_to_ntfy(priv, queue_idx);
|
|
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
|
|
struct gve_rx_ring *rx = &priv->rx[queue_idx];
|
|
|
|
block->rx = rx;
|
|
rx->ntfy_id = ntfy_idx;
|
|
}
|
|
|
|
struct sk_buff *gve_rx_copy_data(struct net_device *dev, struct napi_struct *napi,
|
|
u8 *data, u16 len)
|
|
{
|
|
struct sk_buff *skb;
|
|
|
|
skb = napi_alloc_skb(napi, len);
|
|
if (unlikely(!skb))
|
|
return NULL;
|
|
|
|
__skb_put(skb, len);
|
|
skb_copy_to_linear_data_offset(skb, 0, data, len);
|
|
skb->protocol = eth_type_trans(skb, dev);
|
|
|
|
return skb;
|
|
}
|
|
|
|
struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi,
|
|
struct gve_rx_slot_page_info *page_info, u16 len)
|
|
{
|
|
void *va = page_info->page_address + page_info->page_offset +
|
|
page_info->pad;
|
|
|
|
return gve_rx_copy_data(dev, napi, va, len);
|
|
}
|
|
|
|
void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info)
|
|
{
|
|
page_info->pagecnt_bias--;
|
|
if (page_info->pagecnt_bias == 0) {
|
|
int pagecount = page_count(page_info->page);
|
|
|
|
/* If we have run out of bias - set it back up to INT_MAX
|
|
* minus the existing refs.
|
|
*/
|
|
page_info->pagecnt_bias = INT_MAX - pagecount;
|
|
|
|
/* Set pagecount back up to max. */
|
|
page_ref_add(page_info->page, INT_MAX - pagecount);
|
|
}
|
|
}
|
|
|
|
void gve_add_napi(struct gve_priv *priv, int ntfy_idx,
|
|
int (*gve_poll)(struct napi_struct *, int))
|
|
{
|
|
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
|
|
|
|
netif_napi_add(priv->dev, &block->napi, gve_poll);
|
|
}
|
|
|
|
void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
|
|
{
|
|
struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
|
|
|
|
netif_napi_del(&block->napi);
|
|
}
|