a4ea4c4776
Change the way the Tx queueing works to make the following ends easier to
achieve:
(1) The filling of packets, the encryption of packets and the transmission
of packets can be handled in parallel by separate threads, rather than
rxrpc_sendmsg() allocating, filling, encrypting and transmitting each
packet before moving onto the next one.
(2) Get rid of the fixed-size ring which sets a hard limit on the number
of packets that can be retained in the ring. This allows the number
of packets to increase without having to allocate a very large ring or
having variable-sized rings.
[Note: the downside of this is that it's then less efficient to locate
a packet for retransmission as we then have to step through a list and
examine each buffer in the list.]
(3) Allow the filler/encrypter to run ahead of the transmission window.
(4) Make it easier to do zero copy UDP from the packet buffers.
(5) Make it easier to do zero copy from userspace to the packet buffers -
and thence to UDP (only if for unauthenticated connections).
To that end, the following changes are made:
(1) Use the new rxrpc_txbuf struct instead of sk_buff for keeping packets
to be transmitted in. This allows them to be placed on multiple
queues simultaneously. An sk_buff isn't really necessary as it's
never passed on to lower-level networking code.
(2) Keep the transmissable packets in a linked list on the call struct
rather than in a ring. As a consequence, the annotation buffer isn't
used either; rather a flag is set on the packet to indicate ackedness.
(3) Use the RXRPC_CALL_TX_LAST flag to indicate that the last packet to be
transmitted has been queued. Add RXRPC_CALL_TX_ALL_ACKED to indicate
that all packets up to and including the last got hard acked.
(4) Wire headers are now stored in the txbuf rather than being concocted
on the stack and they're stored immediately before the data, thereby
allowing zerocopy of a single span.
(5) Don't bother with instant-resend on transmission failure; rather,
leave it for a timer or an ACK packet to trigger.
Signed-off-by: David Howells <dhowells@redhat.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: linux-afs@lists.infradead.org
136 lines
3.4 KiB
C
136 lines
3.4 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/* RxRPC Tx data buffering.
|
|
*
|
|
* Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
*/
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
#include <linux/slab.h>
|
|
#include "ar-internal.h"
|
|
|
|
static atomic_t rxrpc_txbuf_debug_ids;
|
|
atomic_t rxrpc_nr_txbuf;
|
|
|
|
/*
|
|
* Allocate and partially initialise an I/O request structure.
|
|
*/
|
|
struct rxrpc_txbuf *rxrpc_alloc_txbuf(struct rxrpc_call *call, u8 packet_type,
|
|
gfp_t gfp)
|
|
{
|
|
struct rxrpc_txbuf *txb;
|
|
|
|
txb = kmalloc(sizeof(*txb), gfp);
|
|
if (txb) {
|
|
INIT_LIST_HEAD(&txb->call_link);
|
|
INIT_LIST_HEAD(&txb->tx_link);
|
|
refcount_set(&txb->ref, 1);
|
|
txb->call = call;
|
|
txb->call_debug_id = call->debug_id;
|
|
txb->debug_id = atomic_inc_return(&rxrpc_txbuf_debug_ids);
|
|
txb->space = sizeof(txb->data);
|
|
txb->len = 0;
|
|
txb->offset = 0;
|
|
txb->flags = 0;
|
|
txb->ack_why = 0;
|
|
txb->seq = call->tx_top + 1;
|
|
txb->wire.epoch = htonl(call->conn->proto.epoch);
|
|
txb->wire.cid = htonl(call->cid);
|
|
txb->wire.callNumber = htonl(call->call_id);
|
|
txb->wire.seq = htonl(txb->seq);
|
|
txb->wire.type = packet_type;
|
|
txb->wire.flags = call->conn->out_clientflag;
|
|
txb->wire.userStatus = 0;
|
|
txb->wire.securityIndex = call->security_ix;
|
|
txb->wire._rsvd = 0;
|
|
txb->wire.serviceId = htons(call->service_id);
|
|
|
|
trace_rxrpc_txbuf(txb->debug_id,
|
|
txb->call_debug_id, txb->seq, 1,
|
|
packet_type == RXRPC_PACKET_TYPE_DATA ?
|
|
rxrpc_txbuf_alloc_data :
|
|
rxrpc_txbuf_alloc_ack);
|
|
atomic_inc(&rxrpc_nr_txbuf);
|
|
}
|
|
|
|
return txb;
|
|
}
|
|
|
|
void rxrpc_get_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
|
|
{
|
|
int r;
|
|
|
|
__refcount_inc(&txb->ref, &r);
|
|
trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r + 1, what);
|
|
}
|
|
|
|
void rxrpc_see_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
|
|
{
|
|
int r = refcount_read(&txb->ref);
|
|
|
|
trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r, what);
|
|
}
|
|
|
|
static void rxrpc_free_txbuf(struct rcu_head *rcu)
|
|
{
|
|
struct rxrpc_txbuf *txb = container_of(rcu, struct rxrpc_txbuf, rcu);
|
|
|
|
trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 0,
|
|
rxrpc_txbuf_free);
|
|
kfree(txb);
|
|
atomic_dec(&rxrpc_nr_txbuf);
|
|
}
|
|
|
|
void rxrpc_put_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
|
|
{
|
|
unsigned int debug_id, call_debug_id;
|
|
rxrpc_seq_t seq;
|
|
bool dead;
|
|
int r;
|
|
|
|
if (txb) {
|
|
debug_id = txb->debug_id;
|
|
call_debug_id = txb->call_debug_id;
|
|
seq = txb->seq;
|
|
dead = __refcount_dec_and_test(&txb->ref, &r);
|
|
trace_rxrpc_txbuf(debug_id, call_debug_id, seq, r - 1, what);
|
|
if (dead)
|
|
call_rcu(&txb->rcu, rxrpc_free_txbuf);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Shrink the transmit buffer.
|
|
*/
|
|
void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *call)
|
|
{
|
|
struct rxrpc_txbuf *txb;
|
|
rxrpc_seq_t hard_ack = smp_load_acquire(&call->acks_hard_ack);
|
|
|
|
_enter("%x/%x/%x", call->tx_bottom, call->acks_hard_ack, call->tx_top);
|
|
|
|
for (;;) {
|
|
spin_lock(&call->tx_lock);
|
|
txb = list_first_entry_or_null(&call->tx_buffer,
|
|
struct rxrpc_txbuf, call_link);
|
|
if (!txb)
|
|
break;
|
|
hard_ack = smp_load_acquire(&call->acks_hard_ack);
|
|
if (before(hard_ack, txb->seq))
|
|
break;
|
|
|
|
ASSERTCMP(txb->seq, ==, call->tx_bottom + 1);
|
|
call->tx_bottom++;
|
|
list_del_rcu(&txb->call_link);
|
|
|
|
trace_rxrpc_txqueue(call, rxrpc_txqueue_dequeue);
|
|
|
|
spin_unlock(&call->tx_lock);
|
|
|
|
rxrpc_put_txbuf(txb, rxrpc_txbuf_put_rotated);
|
|
}
|
|
|
|
spin_unlock(&call->tx_lock);
|
|
}
|