iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'remove-page-frag-implementation-in-vhost_net'

Browse Source 
Yunsheng Lin says:

====================
remove page frag implementation in vhost_net

Currently there are three implementations for page frag:

1. mm/page_alloc.c: net stack seems to be using it in the
   rx part with 'struct page_frag_cache' and the main API
   being page_frag_alloc_align().
2. net/core/sock.c: net stack seems to be using it in the
   tx part with 'struct page_frag' and the main API being
   skb_page_frag_refill().
3. drivers/vhost/net.c: vhost seems to be using it to build
   xdp frame, and it's implementation seems to be a mix of
   the above two.

This patchset tries to unfiy the page frag implementation a
little bit by unifying gfp bit for order 3 page allocation
and replacing page frag implementation in vhost.c with the
one in page_alloc.c.

After this patchset, we are not only able to unify the page
frag implementation a little, but also able to have about
0.5% performance boost testing by using the vhost_net_test
introduced in the last patch.

Before this patchset:
Performance counter stats for './vhost_net_test' (10 runs):

     305325.78 msec task-clock                       #    1.738 CPUs utilized               ( +-  0.12% )
       1048668      context-switches                 #    3.435 K/sec                       ( +-  0.00% )
            11      cpu-migrations                   #    0.036 /sec                        ( +- 17.64% )
            33      page-faults                      #    0.108 /sec                        ( +-  0.49% )
  244651819491      cycles                           #    0.801 GHz                         ( +-  0.43% )  (64)
   64714638024      stalled-cycles-frontend          #   26.45% frontend cycles idle        ( +-  2.19% )  (67)
   30774313491      stalled-cycles-backend           #   12.58% backend cycles idle         ( +-  7.68% )  (70)
  201749748680      instructions                     #    0.82  insn per cycle
                                              #    0.32  stalled cycles per insn     ( +-  0.41% )  (66.76%)
   65494787909      branches                         #  214.508 M/sec                       ( +-  0.35% )  (64)
    4284111313      branch-misses                    #    6.54% of all branches             ( +-  0.45% )  (66)

       175.699 +- 0.189 seconds time elapsed  ( +-  0.11% )

After this patchset:
Performance counter stats for './vhost_net_test' (10 runs):

     303974.38 msec task-clock                       #    1.739 CPUs utilized               ( +-  0.14% )
       1048807      context-switches                 #    3.450 K/sec                       ( +-  0.00% )
            14      cpu-migrations                   #    0.046 /sec                        ( +- 12.86% )
            33      page-faults                      #    0.109 /sec                        ( +-  0.46% )
  251289376347      cycles                           #    0.827 GHz                         ( +-  0.32% )  (60)
   67885175415      stalled-cycles-frontend          #   27.01% frontend cycles idle        ( +-  0.48% )  (63)
   27809282600      stalled-cycles-backend           #   11.07% backend cycles idle         ( +-  0.36% )  (71)
  195543234672      instructions                     #    0.78  insn per cycle
                                              #    0.35  stalled cycles per insn     ( +-  0.29% )  (69.04%)
   62423183552      branches                         #  205.357 M/sec                       ( +-  0.48% )  (67)
    4135666632      branch-misses                    #    6.63% of all branches             ( +-  0.63% )  (67)

       174.764 +- 0.214 seconds time elapsed  ( +-  0.12% )

Changelog:
V6: Add timeout for poll() and simplify some logic as suggested
    by Jason.

V5: Address the comment from jason in vhost_net_test.c and the
    comment about leaving out the gfp change for page frag in
    sock.c as suggested by Paolo.

V4: Resend based on latest net-next branch.

V3:
1. Add __page_frag_alloc_align() which is passed with the align mask
   the original function expected as suggested by Alexander.
2. Drop patch 3 in v2 suggested by Alexander.
3. Reorder patch 4 & 5 in v2 suggested by Alexander.

Note that placing this gfp flags handing for order 3 page in an inline
function is not considered, as we may be able to unify the page_frag
and page_frag_cache handling.

V2: Change 'xor'd' to 'masked off', add vhost tx testing for
    vhost_net_test.

V1: Fix some typo, drop RFC tag and rebase on latest net-next.
====================

Link: https://lore.kernel.org/r/20240228093013.8263-1-linyunsheng@huawei.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni 2024-03-05 11:38:16 +01:00
commit 6702d60d3c
12 changed files with 609 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
drivers/net/ethernet/google/gve/gve_main.c
View File

@ -1276,17 +1276,10 @@ static void gve_unreg_xdp_info(struct gve_priv *priv)
static void gve_drain_page_cache(struct gve_priv *priv)
{
struct page_frag_cache *nc;
int i;
for (i = 0; i < priv->rx_cfg.num_queues; i++) {
nc = &priv->rx[i].page_cache;
if (nc->va) {
__page_frag_cache_drain(virt_to_page(nc->va),
nc->pagecnt_bias);
nc->va = NULL;
}
}
for (i = 0; i < priv->rx_cfg.num_queues; i++)
page_frag_cache_drain(&priv->rx[i].page_cache);
}
static void gve_qpls_get_curr_alloc_cfg(struct gve_priv *priv,

17
drivers/net/ethernet/mediatek/mtk_wed_wo.c
View File

@ -286,7 +286,6 @@ mtk_wed_wo_queue_free(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
static void
mtk_wed_wo_queue_tx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
struct page *page;
int i;
for (i = 0; i < q->n_desc; i++) {
@ -301,19 +300,12 @@ mtk_wed_wo_queue_tx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
entry->buf = NULL;
}
if (!q->cache.va)
return;
page = virt_to_page(q->cache.va);
__page_frag_cache_drain(page, q->cache.pagecnt_bias);
memset(&q->cache, 0, sizeof(q->cache));
page_frag_cache_drain(&q->cache);
}
static void
mtk_wed_wo_queue_rx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
struct page *page;
for (;;) {
void *buf = mtk_wed_wo_dequeue(wo, q, NULL, true);
@ -323,12 +315,7 @@ mtk_wed_wo_queue_rx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
skb_free_frag(buf);
}
if (!q->cache.va)
return;
page = virt_to_page(q->cache.va);
__page_frag_cache_drain(page, q->cache.pagecnt_bias);
memset(&q->cache, 0, sizeof(q->cache));
page_frag_cache_drain(&q->cache);
}
static void

7
drivers/nvme/host/tcp.c
View File

@ -1344,7 +1344,6 @@ static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
{
struct page *page;
struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
struct nvme_tcp_queue *queue = &ctrl->queues[qid];
unsigned int noreclaim_flag;
@ -1355,11 +1354,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
if (queue->hdr_digest || queue->data_digest)
nvme_tcp_free_crypto(queue);
if (queue->pf_cache.va) {
page = virt_to_head_page(queue->pf_cache.va);
__page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
queue->pf_cache.va = NULL;
}
page_frag_cache_drain(&queue->pf_cache);
noreclaim_flag = memalloc_noreclaim_save();
/* ->sock will be released by fput() */

4
drivers/nvme/target/tcp.c
View File

@ -1591,7 +1591,6 @@ static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue)
static void nvmet_tcp_release_queue_work(struct work_struct *w)
{
struct page *page;
struct nvmet_tcp_queue *queue =
container_of(w, struct nvmet_tcp_queue, release_work);
@ -1615,8 +1614,7 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w)
if (queue->hdr_digest || queue->data_digest)
nvmet_tcp_free_crypto(queue);
ida_free(&nvmet_tcp_queue_ida, queue->idx);
page = virt_to_head_page(queue->pf_cache.va);
__page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
page_frag_cache_drain(&queue->pf_cache);
kfree(queue);
}

91
drivers/vhost/net.c
View File

@ -141,10 +141,8 @@ struct vhost_net {
unsigned tx_zcopy_err;
/* Flush in progress. Protected by tx vq lock. */
bool tx_flush;
/* Private page frag */
struct page_frag page_frag;
/* Refcount bias of page frag */
int refcnt_bias;
/* Private page frag cache */
struct page_frag_cache pf_cache;
};
static unsigned vhost_net_zcopy_mask __read_mostly;
@ -655,41 +653,6 @@ static bool tx_can_batch(struct vhost_virtqueue *vq, size_t total_len)
!vhost_vq_avail_empty(vq->dev, vq);
}
static bool vhost_net_page_frag_refill(struct vhost_net *net, unsigned int sz,
struct page_frag *pfrag, gfp_t gfp)
{
if (pfrag->page) {
if (pfrag->offset + sz <= pfrag->size)
return true;
__page_frag_cache_drain(pfrag->page, net->refcnt_bias);
}
pfrag->offset = 0;
net->refcnt_bias = 0;
if (SKB_FRAG_PAGE_ORDER) {
/* Avoid direct reclaim but allow kswapd to wake */
pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
__GFP_COMP | __GFP_NOWARN |
__GFP_NORETRY,
SKB_FRAG_PAGE_ORDER);
if (likely(pfrag->page)) {
pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
goto done;
}
}
pfrag->page = alloc_page(gfp);
if (likely(pfrag->page)) {
pfrag->size = PAGE_SIZE;
goto done;
}
return false;
done:
net->refcnt_bias = USHRT_MAX;
page_ref_add(pfrag->page, USHRT_MAX - 1);
return true;
}
#define VHOST_NET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
@ -699,7 +662,6 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
struct vhost_net *net = container_of(vq->dev, struct vhost_net,
dev);
struct socket *sock = vhost_vq_get_backend(vq);
struct page_frag *alloc_frag = &net->page_frag;
struct virtio_net_hdr *gso;
struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];
struct tun_xdp_hdr *hdr;
@ -710,6 +672,7 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
int sock_hlen = nvq->sock_hlen;
void *buf;
int copied;
int ret;
if (unlikely(len < nvq->sock_hlen))
return -EFAULT;
@ -719,18 +682,17 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
return -ENOSPC;
buflen += SKB_DATA_ALIGN(len + pad);
alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
if (unlikely(!vhost_net_page_frag_refill(net, buflen,
alloc_frag, GFP_KERNEL)))
buf = page_frag_alloc_align(&net->pf_cache, buflen, GFP_KERNEL,
SMP_CACHE_BYTES);
if (unlikely(!buf))
return -ENOMEM;
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
copied = copy_page_from_iter(alloc_frag->page,
alloc_frag->offset +
offsetof(struct tun_xdp_hdr, gso),
sock_hlen, from);
if (copied != sock_hlen)
return -EFAULT;
copied = copy_from_iter(buf + offsetof(struct tun_xdp_hdr, gso),
sock_hlen, from);
if (copied != sock_hlen) {
ret = -EFAULT;
goto err;
}
hdr = buf;
gso = &hdr->gso;
@ -743,27 +705,30 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
vhost16_to_cpu(vq, gso->csum_start) +
vhost16_to_cpu(vq, gso->csum_offset) + 2);
if (vhost16_to_cpu(vq, gso->hdr_len) > len)
return -EINVAL;
if (vhost16_to_cpu(vq, gso->hdr_len) > len) {
ret = -EINVAL;
goto err;
}
}
len -= sock_hlen;
copied = copy_page_from_iter(alloc_frag->page,
alloc_frag->offset + pad,
len, from);
if (copied != len)
return -EFAULT;
copied = copy_from_iter(buf + pad, len, from);
if (copied != len) {
ret = -EFAULT;
goto err;
}
xdp_init_buff(xdp, buflen, NULL);
xdp_prepare_buff(xdp, buf, pad, len, true);
hdr->buflen = buflen;
--net->refcnt_bias;
alloc_frag->offset += buflen;
++nvq->batched_xdp;
return 0;
err:
page_frag_free(buf);
return ret;
}
static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
@ -1353,8 +1318,7 @@ static int vhost_net_open(struct inode *inode, struct file *f)
vqs[VHOST_NET_VQ_RX]);
f->private_data = n;
n->page_frag.page = NULL;
n->refcnt_bias = 0;
n->pf_cache.va = NULL;
return 0;
}
@ -1422,8 +1386,7 @@ static int vhost_net_release(struct inode *inode, struct file *f)
kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue);
kfree(n->vqs[VHOST_NET_VQ_TX].xdp);
kfree(n->dev.vqs);
if (n->page_frag.page)
__page_frag_cache_drain(n->page_frag.page, n->refcnt_bias);
page_frag_cache_drain(&n->pf_cache);
kvfree(n);
return 0;
}

16
include/linux/gfp.h
View File

@ -311,15 +311,23 @@ extern void __free_pages(struct page *page, unsigned int order);
extern void free_pages(unsigned long addr, unsigned int order);
struct page_frag_cache;
void page_frag_cache_drain(struct page_frag_cache *nc);
extern void __page_frag_cache_drain(struct page *page, unsigned int count);
extern void *page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
unsigned int align_mask);
void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz,
gfp_t gfp_mask, unsigned int align_mask);
static inline void *page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
unsigned int align)
{
WARN_ON_ONCE(!is_power_of_2(align));
return __page_frag_alloc_align(nc, fragsz, gfp_mask, -align);
}
static inline void *page_frag_alloc(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask)
{
return page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u);
return __page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u);
}
extern void page_frag_free(void *addr);

22
mm/page_alloc.c
View File

@ -4685,8 +4685,8 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
gfp_t gfp = gfp_mask;
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
__GFP_NOMEMALLOC;
gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP |
__GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC;
page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
PAGE_FRAG_CACHE_MAX_ORDER);
nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
@ -4699,6 +4699,16 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
return page;
}
void page_frag_cache_drain(struct page_frag_cache *nc)
{
if (!nc->va)
return;
__page_frag_cache_drain(virt_to_head_page(nc->va), nc->pagecnt_bias);
nc->va = NULL;
}
EXPORT_SYMBOL(page_frag_cache_drain);
void __page_frag_cache_drain(struct page *page, unsigned int count)
{
VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
@ -4708,9 +4718,9 @@ void __page_frag_cache_drain(struct page *page, unsigned int count)
}
EXPORT_SYMBOL(__page_frag_cache_drain);
void *page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
unsigned int align_mask)
void *__page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
unsigned int align_mask)
{
unsigned int size = PAGE_SIZE;
struct page *page;
@ -4779,7 +4789,7 @@ refill:
return nc->va + offset;
}
EXPORT_SYMBOL(page_frag_alloc_align);
EXPORT_SYMBOL(__page_frag_alloc_align);
/*
* Frees a page fragment allocated out of either a compound or order 0 page.

9
net/core/skbuff.c
View File

@ -315,7 +315,8 @@ void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
fragsz = SKB_DATA_ALIGN(fragsz);
return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask);
return __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC,
align_mask);
}
EXPORT_SYMBOL(__napi_alloc_frag_align);
@ -327,13 +328,15 @@ void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
if (in_hardirq() || irqs_disabled()) {
struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask);
data = __page_frag_alloc_align(nc, fragsz, GFP_ATOMIC,
align_mask);
} else {
struct napi_alloc_cache *nc;
local_bh_disable();
nc = this_cpu_ptr(&napi_alloc_cache);
data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask);
data = __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC,
align_mask);
local_bh_enable();
}
return data;

1
tools/virtio/.gitignore vendored
View File

@ -1,5 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
*.d
virtio_test
vhost_net_test
vringh_test
virtio-trace/trace-agent

8
tools/virtio/Makefile
View File

@ -1,8 +1,9 @@
# SPDX-License-Identifier: GPL-2.0
all: test mod
test: virtio_test vringh_test
test: virtio_test vringh_test vhost_net_test
virtio_test: virtio_ring.o virtio_test.o
vringh_test: vringh_test.o vringh.o virtio_ring.o
vhost_net_test: virtio_ring.o vhost_net_test.o
try-run = $(shell set -e; \
if ($(1)) >/dev/null 2>&1; \
@ -49,6 +50,7 @@ oot-clean: OOT_BUILD+=clean
.PHONY: all test mod clean vhost oot oot-clean oot-build
clean:
${RM} *.o vringh_test virtio_test vhost_test/*.o vhost_test/.*.cmd \
vhost_test/Module.symvers vhost_test/modules.order *.d
${RM} *.o vringh_test virtio_test vhost_net_test vhost_test/*.o \
vhost_test/.*.cmd vhost_test/Module.symvers \
vhost_test/modules.order *.d
-include *.d

4
tools/virtio/linux/virtio_config.h
View File

@ -1,4 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef LINUX_VIRTIO_CONFIG_H
#define LINUX_VIRTIO_CONFIG_H
#include <linux/virtio_byteorder.h>
#include <linux/virtio.h>
#include <uapi/linux/virtio_config.h>
@ -95,3 +97,5 @@ static inline __virtio64 cpu_to_virtio64(struct virtio_device *vdev, u64 val)
{
return __cpu_to_virtio64(virtio_is_little_endian(vdev), val);
}
#endif

532
tools/virtio/vhost_net_test.c Normal file
View File

@ -0,0 +1,532 @@
// SPDX-License-Identifier: GPL-2.0
#define _GNU_SOURCE
#include <getopt.h>
#include <limits.h>
#include <string.h>
#include <poll.h>
#include <sys/eventfd.h>
#include <stdlib.h>
#include <assert.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdbool.h>
#include <linux/vhost.h>
#include <linux/if.h>
#include <linux/if_tun.h>
#include <linux/in.h>
#include <linux/if_packet.h>
#include <linux/virtio_net.h>
#include <netinet/ether.h>
#define HDR_LEN sizeof(struct virtio_net_hdr_mrg_rxbuf)
#define TEST_BUF_LEN 256
#define TEST_PTYPE ETH_P_LOOPBACK
#define DESC_NUM 256
/* Used by implementation of kmalloc() in tools/virtio/linux/kernel.h */
void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end;
struct vq_info {
int kick;
int call;
int idx;
long started;
long completed;
struct pollfd fds;
void *ring;
/* copy used for control */
struct vring vring;
struct virtqueue *vq;
};
struct vdev_info {
struct virtio_device vdev;
int control;
struct vq_info vqs[2];
int nvqs;
void *buf;
size_t buf_size;
char *test_buf;
char *res_buf;
struct vhost_memory *mem;
int sock;
int ifindex;
unsigned char mac[ETHER_ADDR_LEN];
};
static int tun_alloc(struct vdev_info *dev, char *tun_name)
{
struct ifreq ifr;
int len = HDR_LEN;
int fd, e;
fd = open("/dev/net/tun", O_RDWR);
if (fd < 0) {
perror("Cannot open /dev/net/tun");
return fd;
}
memset(&ifr, 0, sizeof(ifr));
ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR;
strncpy(ifr.ifr_name, tun_name, IFNAMSIZ);
e = ioctl(fd, TUNSETIFF, &ifr);
if (e < 0) {
perror("ioctl[TUNSETIFF]");
close(fd);
return e;
}
e = ioctl(fd, TUNSETVNETHDRSZ, &len);
if (e < 0) {
perror("ioctl[TUNSETVNETHDRSZ]");
close(fd);
return e;
}
e = ioctl(fd, SIOCGIFHWADDR, &ifr);
if (e < 0) {
perror("ioctl[SIOCGIFHWADDR]");
close(fd);
return e;
}
memcpy(dev->mac, &ifr.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
return fd;
}
static void vdev_create_socket(struct vdev_info *dev, char *tun_name)
{
struct ifreq ifr;
dev->sock = socket(AF_PACKET, SOCK_RAW, htons(TEST_PTYPE));
assert(dev->sock != -1);
strncpy(ifr.ifr_name, tun_name, IFNAMSIZ);
assert(ioctl(dev->sock, SIOCGIFINDEX, &ifr) >= 0);
dev->ifindex = ifr.ifr_ifindex;
/* Set the flags that bring the device up */
assert(ioctl(dev->sock, SIOCGIFFLAGS, &ifr) >= 0);
ifr.ifr_flags |= (IFF_UP | IFF_RUNNING);
assert(ioctl(dev->sock, SIOCSIFFLAGS, &ifr) >= 0);
}
static void vdev_send_packet(struct vdev_info *dev)
{
char *sendbuf = dev->test_buf + HDR_LEN;
struct sockaddr_ll saddrll = {0};
int sockfd = dev->sock;
int ret;
saddrll.sll_family = PF_PACKET;
saddrll.sll_ifindex = dev->ifindex;
saddrll.sll_halen = ETH_ALEN;
saddrll.sll_protocol = htons(TEST_PTYPE);
ret = sendto(sockfd, sendbuf, TEST_BUF_LEN, 0,
(struct sockaddr *)&saddrll,
sizeof(struct sockaddr_ll));
assert(ret >= 0);
}
static bool vq_notify(struct virtqueue *vq)
{
struct vq_info *info = vq->priv;
unsigned long long v = 1;
int r;
r = write(info->kick, &v, sizeof(v));
assert(r == sizeof(v));
return true;
}
static void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info)
{
struct vhost_vring_addr addr = {
.index = info->idx,
.desc_user_addr = (uint64_t)(unsigned long)info->vring.desc,
.avail_user_addr = (uint64_t)(unsigned long)info->vring.avail,
.used_user_addr = (uint64_t)(unsigned long)info->vring.used,
};
struct vhost_vring_state state = { .index = info->idx };
struct vhost_vring_file file = { .index = info->idx };
int r;
state.num = info->vring.num;
r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
assert(r >= 0);
state.num = 0;
r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
assert(r >= 0);
r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
assert(r >= 0);
file.fd = info->kick;
r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
assert(r >= 0);
}
static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev)
{
if (info->vq)
vring_del_virtqueue(info->vq);
memset(info->ring, 0, vring_size(num, 4096));
vring_init(&info->vring, num, info->ring, 4096);
info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false,
info->ring, vq_notify, NULL, "test");
assert(info->vq);
info->vq->priv = info;
}
static void vq_info_add(struct vdev_info *dev, int idx, int num, int fd)
{
struct vhost_vring_file backend = { .index = idx, .fd = fd };
struct vq_info *info = &dev->vqs[idx];
int r;
info->idx = idx;
info->kick = eventfd(0, EFD_NONBLOCK);
r = posix_memalign(&info->ring, 4096, vring_size(num, 4096));
assert(r >= 0);
vq_reset(info, num, &dev->vdev);
vhost_vq_setup(dev, info);
r = ioctl(dev->control, VHOST_NET_SET_BACKEND, &backend);
assert(!r);
}
static void vdev_info_init(struct vdev_info *dev, unsigned long long features)
{
struct ether_header *eh;
int i, r;
dev->vdev.features = features;
INIT_LIST_HEAD(&dev->vdev.vqs);
spin_lock_init(&dev->vdev.vqs_list_lock);
dev->buf_size = (HDR_LEN + TEST_BUF_LEN) * 2;
dev->buf = malloc(dev->buf_size);
assert(dev->buf);
dev->test_buf = dev->buf;
dev->res_buf = dev->test_buf + HDR_LEN + TEST_BUF_LEN;
memset(dev->test_buf, 0, HDR_LEN + TEST_BUF_LEN);
eh = (struct ether_header *)(dev->test_buf + HDR_LEN);
eh->ether_type = htons(TEST_PTYPE);
memcpy(eh->ether_dhost, dev->mac, ETHER_ADDR_LEN);
memcpy(eh->ether_shost, dev->mac, ETHER_ADDR_LEN);
for (i = sizeof(*eh); i < TEST_BUF_LEN; i++)
dev->test_buf[i + HDR_LEN] = (char)i;
dev->control = open("/dev/vhost-net", O_RDWR);
assert(dev->control >= 0);
r = ioctl(dev->control, VHOST_SET_OWNER, NULL);
assert(r >= 0);
dev->mem = malloc(offsetof(struct vhost_memory, regions) +
sizeof(dev->mem->regions[0]));
assert(dev->mem);
memset(dev->mem, 0, offsetof(struct vhost_memory, regions) +
sizeof(dev->mem->regions[0]));
dev->mem->nregions = 1;
dev->mem->regions[0].guest_phys_addr = (long)dev->buf;
dev->mem->regions[0].userspace_addr = (long)dev->buf;
dev->mem->regions[0].memory_size = dev->buf_size;
r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
assert(r >= 0);
r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
assert(r >= 0);
dev->nvqs = 2;
}
static void wait_for_interrupt(struct vq_info *vq)
{
unsigned long long val;
poll(&vq->fds, 1, 100);
if (vq->fds.revents & POLLIN)
read(vq->fds.fd, &val, sizeof(val));
}
static void verify_res_buf(char *res_buf)
{
int i;
for (i = ETHER_HDR_LEN; i < TEST_BUF_LEN; i++)
assert(res_buf[i] == (char)i);
}
static void run_tx_test(struct vdev_info *dev, struct vq_info *vq,
bool delayed, int bufs)
{
long long spurious = 0;
struct scatterlist sl;
unsigned int len;
int r;
for (;;) {
long started_before = vq->started;
long completed_before = vq->completed;
virtqueue_disable_cb(vq->vq);
do {
while (vq->started < bufs &&
(vq->started - vq->completed) < 1) {
sg_init_one(&sl, dev->test_buf, HDR_LEN + TEST_BUF_LEN);
r = virtqueue_add_outbuf(vq->vq, &sl, 1,
dev->test_buf + vq->started,
GFP_ATOMIC);
if (unlikely(r != 0))
break;
++vq->started;
if (unlikely(!virtqueue_kick(vq->vq))) {
r = -1;
break;
}
}
if (vq->started >= bufs)
r = -1;
/* Flush out completed bufs if any */
while (virtqueue_get_buf(vq->vq, &len)) {
int n;
n = recvfrom(dev->sock, dev->res_buf, TEST_BUF_LEN, 0, NULL, NULL);
assert(n == TEST_BUF_LEN);
verify_res_buf(dev->res_buf);
++vq->completed;
r = 0;
}
} while (r == 0);
if (vq->completed == completed_before && vq->started == started_before)
++spurious;
assert(vq->completed <= bufs);
assert(vq->started <= bufs);
if (vq->completed == bufs)
break;
if (delayed) {
if (virtqueue_enable_cb_delayed(vq->vq))
wait_for_interrupt(vq);
} else {
if (virtqueue_enable_cb(vq->vq))
wait_for_interrupt(vq);
}
}
printf("TX spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n",
spurious, vq->started, vq->completed);
}
static void run_rx_test(struct vdev_info *dev, struct vq_info *vq,
bool delayed, int bufs)
{
long long spurious = 0;
struct scatterlist sl;
unsigned int len;
int r;
for (;;) {
long started_before = vq->started;
long completed_before = vq->completed;
do {
while (vq->started < bufs &&
(vq->started - vq->completed) < 1) {
sg_init_one(&sl, dev->res_buf, HDR_LEN + TEST_BUF_LEN);
r = virtqueue_add_inbuf(vq->vq, &sl, 1,
dev->res_buf + vq->started,
GFP_ATOMIC);
if (unlikely(r != 0))
break;
++vq->started;
vdev_send_packet(dev);
if (unlikely(!virtqueue_kick(vq->vq))) {
r = -1;
break;
}
}
if (vq->started >= bufs)
r = -1;
/* Flush out completed bufs if any */
while (virtqueue_get_buf(vq->vq, &len)) {
struct ether_header *eh;
eh = (struct ether_header *)(dev->res_buf + HDR_LEN);
/* tun netdev is up and running, only handle the
* TEST_PTYPE packet.
*/
if (eh->ether_type == htons(TEST_PTYPE)) {
assert(len == TEST_BUF_LEN + HDR_LEN);
verify_res_buf(dev->res_buf + HDR_LEN);
}
++vq->completed;
r = 0;
}
} while (r == 0);
if (vq->completed == completed_before && vq->started == started_before)
++spurious;
assert(vq->completed <= bufs);
assert(vq->started <= bufs);
if (vq->completed == bufs)
break;
}
printf("RX spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n",
spurious, vq->started, vq->completed);
}
static const char optstring[] = "h";
static const struct option longopts[] = {
{
.name = "help",
.val = 'h',
},
{
.name = "event-idx",
.val = 'E',
},
{
.name = "no-event-idx",
.val = 'e',
},
{
.name = "indirect",
.val = 'I',
},
{
.name = "no-indirect",
.val = 'i',
},
{
.name = "virtio-1",
.val = '1',
},
{
.name = "no-virtio-1",
.val = '0',
},
{
.name = "delayed-interrupt",
.val = 'D',
},
{
.name = "no-delayed-interrupt",
.val = 'd',
},
{
.name = "buf-num",
.val = 'n',
.has_arg = required_argument,
},
{
.name = "batch",
.val = 'b',
.has_arg = required_argument,
},
{
}
};
static void help(int status)
{
fprintf(stderr, "Usage: vhost_net_test [--help]"
" [--no-indirect]"
" [--no-event-idx]"
" [--no-virtio-1]"
" [--delayed-interrupt]"
" [--buf-num]"
"\n");
exit(status);
}
int main(int argc, char **argv)
{
unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
(1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1);
char tun_name[IFNAMSIZ];
long nbufs = 0x100000;
struct vdev_info dev;
bool delayed = false;
int o, fd;
for (;;) {
o = getopt_long(argc, argv, optstring, longopts, NULL);
switch (o) {
case -1:
goto done;
case '?':
help(2);
case 'e':
features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX);
break;
case 'h':
help(0);
case 'i':
features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC);
break;
case '0':
features &= ~(1ULL << VIRTIO_F_VERSION_1);
break;
case 'D':
delayed = true;
break;
case 'n':
nbufs = strtol(optarg, NULL, 10);
assert(nbufs > 0);
break;
default:
assert(0);
break;
}
}
done:
memset(&dev, 0, sizeof(dev));
snprintf(tun_name, IFNAMSIZ, "tun_%d", getpid());
fd = tun_alloc(&dev, tun_name);
assert(fd >= 0);
vdev_info_init(&dev, features);
vq_info_add(&dev, 0, DESC_NUM, fd);
vq_info_add(&dev, 1, DESC_NUM, fd);
vdev_create_socket(&dev, tun_name);
run_rx_test(&dev, &dev.vqs[0], delayed, nbufs);
run_tx_test(&dev, &dev.vqs[1], delayed, nbufs);
return 0;
}