2023-08-23 12:29:12 +03:00
/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2016-02-14 23:44:34 +03:00
/*
* Copyright ( c ) 2016 Intel Corporation .
*/
# ifndef HFI1_SDMA_TXREQ_H
# define HFI1_SDMA_TXREQ_H
/* increased for AHG */
# define NUM_DESC 6
/*
* struct sdma_desc - canonical fragment descriptor
*
* This is the descriptor carried in the tx request
* corresponding to each fragment .
*
*/
struct sdma_desc {
/* private: don't use directly */
u64 qw [ 2 ] ;
IB/hfi1: Fix bugs with non-PAGE_SIZE-end multi-iovec user SDMA requests
hfi1 user SDMA request processing has two bugs that can cause data
corruption for user SDMA requests that have multiple payload iovecs
where an iovec other than the tail iovec does not run up to the page
boundary for the buffer pointed to by that iovec.a
Here are the specific bugs:
1. user_sdma_txadd() does not use struct user_sdma_iovec->iov.iov_len.
Rather, user_sdma_txadd() will add up to PAGE_SIZE bytes from iovec
to the packet, even if some of those bytes are past
iovec->iov.iov_len and are thus not intended to be in the packet.
2. user_sdma_txadd() and user_sdma_send_pkts() fail to advance to the
next iovec in user_sdma_request->iovs when the current iovec
is not PAGE_SIZE and does not contain enough data to complete the
packet. The transmitted packet will contain the wrong data from the
iovec pages.
This has not been an issue with SDMA packets from hfi1 Verbs or PSM2
because they only produce iovecs that end short of PAGE_SIZE as the tail
iovec of an SDMA request.
Fixing these bugs exposes other bugs with the SDMA pin cache
(struct mmu_rb_handler) that get in way of supporting user SDMA requests
with multiple payload iovecs whose buffers do not end at PAGE_SIZE. So
this commit fixes those issues as well.
Here are the mmu_rb_handler bugs that non-PAGE_SIZE-end multi-iovec
payload user SDMA requests can hit:
1. Overlapping memory ranges in mmu_rb_handler will result in duplicate
pinnings.
2. When extending an existing mmu_rb_handler entry (struct mmu_rb_node),
the mmu_rb code (1) removes the existing entry under a lock, (2)
releases that lock, pins the new pages, (3) then reacquires the lock
to insert the extended mmu_rb_node.
If someone else comes in and inserts an overlapping entry between (2)
and (3), insert in (3) will fail.
The failure path code in this case unpins _all_ pages in either the
original mmu_rb_node or the new mmu_rb_node that was inserted between
(2) and (3).
3. In hfi1_mmu_rb_remove_unless_exact(), mmu_rb_node->refcount is
incremented outside of mmu_rb_handler->lock. As a result, mmu_rb_node
could be evicted by another thread that gets mmu_rb_handler->lock and
checks mmu_rb_node->refcount before mmu_rb_node->refcount is
incremented.
4. Related to #2 above, SDMA request submission failure path does not
check mmu_rb_node->refcount before freeing mmu_rb_node object.
If there are other SDMA requests in progress whose iovecs have
pointers to the now-freed mmu_rb_node(s), those pointers to the
now-freed mmu_rb nodes will be dereferenced when those SDMA requests
complete.
Fixes: 7be85676f1d1 ("IB/hfi1: Don't remove RB entry when not needed.")
Fixes: 7724105686e7 ("IB/hfi1: add driver files")
Signed-off-by: Brendan Cunningham <bcunningham@cornelisnetworks.com>
Signed-off-by: Patrick Kelsey <pat.kelsey@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
Link: https://lore.kernel.org/r/168088636445.3027109.10054635277810177889.stgit@252.162.96.66.static.eigbox.net
Signed-off-by: Leon Romanovsky <leon@kernel.org>
2023-04-07 19:52:44 +03:00
void * pinning_ctx ;
2023-05-19 19:32:16 +03:00
/* Release reference to @pinning_ctx. May be called in interrupt context. Must not sleep. */
void ( * ctx_put ) ( void * ctx ) ;
2016-02-14 23:44:34 +03:00
} ;
/**
* struct sdma_txreq - the sdma_txreq structure ( one per packet )
* @ list : for use by user and by queuing for wait
*
* This is the representation of a packet which consists of some
* number of fragments . Storage is provided to within the structure .
* for all fragments .
*
* The storage for the descriptors are automatically extended as needed
* when the currently allocation is exceeded .
*
* The user ( Verbs or PSM ) may overload this structure with fields
* specific to their use by putting this struct first in their struct .
* The method of allocation of the overloaded structure is user dependent
*
* The list is the only public field in the structure .
*
*/
# define SDMA_TXREQ_S_OK 0
# define SDMA_TXREQ_S_SENDERROR 1
# define SDMA_TXREQ_S_ABORTED 2
# define SDMA_TXREQ_S_SHUTDOWN 3
/* flags bits */
# define SDMA_TXREQ_F_URGENT 0x0001
# define SDMA_TXREQ_F_AHG_COPY 0x0002
# define SDMA_TXREQ_F_USE_AHG 0x0004
2019-01-24 08:52:19 +03:00
# define SDMA_TXREQ_F_VIP 0x0010
2016-02-14 23:44:34 +03:00
struct sdma_txreq ;
2016-02-14 23:45:53 +03:00
typedef void ( * callback_t ) ( struct sdma_txreq * , int ) ;
2016-02-14 23:44:34 +03:00
struct iowait ;
struct sdma_txreq {
struct list_head list ;
/* private: */
struct sdma_desc * descp ;
/* private: */
void * coalesce_buf ;
/* private: */
struct iowait * wait ;
/* private: */
callback_t complete ;
# ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
u64 sn ;
# endif
/* private: - used in coalesce/pad processing */
u16 packet_len ;
/* private: - down-counted to trigger last */
u16 tlen ;
/* private: */
u16 num_desc ;
/* private: */
u16 desc_limit ;
/* private: */
u16 next_descq_idx ;
/* private: */
u16 coalesce_idx ;
/* private: flags */
u16 flags ;
/* private: */
struct sdma_desc descs [ NUM_DESC ] ;
} ;
2016-02-14 23:45:18 +03:00
static inline int sdma_txreq_built ( struct sdma_txreq * tx )
{
return tx - > num_desc ;
}
2016-02-14 23:44:34 +03:00
# endif /* HFI1_SDMA_TXREQ_H */