2023-04-13 10:33:36 +03:00
// SPDX-License-Identifier: GPL-2.0-only
/*
* SolidRun DPU driver for control plane
*
* Copyright ( C ) 2022 - 2023 SolidRun
*
* Author : Alvaro Karsz < alvaro . karsz @ solid - run . com >
*
*/
# include <linux/iopoll.h>
# include "snet_vdpa.h"
enum snet_ctrl_opcodes {
SNET_CTRL_OP_DESTROY = 1 ,
SNET_CTRL_OP_READ_VQ_STATE ,
2023-04-13 10:33:37 +03:00
SNET_CTRL_OP_SUSPEND ,
2023-05-02 16:10:48 +03:00
SNET_CTRL_OP_RESUME ,
2023-04-13 10:33:36 +03:00
} ;
# define SNET_CTRL_TIMEOUT 2000000
# define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF
# define SNET_CTRL_IN_PROCESS_MASK 0x00010000
# define SNET_CTRL_CHUNK_RDY_MASK 0x00020000
# define SNET_CTRL_ERROR_MASK 0x0FFC0000
# define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
# define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \
! ( ( val ) & SNET_CTRL_IN_PROCESS_MASK ) )
# define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
/* Control register used to read data from the DPU */
struct snet_ctrl_reg_ctrl {
/* Chunk size in 4B words */
u16 data_size ;
/* We are in the middle of a command */
u16 in_process : 1 ;
/* A data chunk is ready and can be consumed */
u16 chunk_ready : 1 ;
/* Error code */
u16 error : 10 ;
/* Saved for future usage */
u16 rsvd : 4 ;
} ;
/* Opcode register */
struct snet_ctrl_reg_op {
u16 opcode ;
/* Only if VQ index is relevant for the command */
u16 vq_idx ;
} ;
struct snet_ctrl_regs {
struct snet_ctrl_reg_op op ;
struct snet_ctrl_reg_ctrl ctrl ;
u32 rsvd ;
u32 data [ ] ;
} ;
static struct snet_ctrl_regs __iomem * snet_get_ctrl ( struct snet * snet )
{
return snet - > bar + snet - > psnet - > cfg . ctrl_off ;
}
static int snet_wait_for_empty_ctrl ( struct snet_ctrl_regs __iomem * regs )
{
u32 val ;
return readx_poll_timeout ( ioread32 , & regs - > ctrl , val , SNET_EMPTY_CTRL ( val ) , 10 ,
SNET_CTRL_TIMEOUT ) ;
}
static int snet_wait_for_empty_op ( struct snet_ctrl_regs __iomem * regs )
{
u32 val ;
return readx_poll_timeout ( ioread32 , & regs - > op , val , ! val , 10 , SNET_CTRL_TIMEOUT ) ;
}
static int snet_wait_for_data ( struct snet_ctrl_regs __iomem * regs )
{
u32 val ;
return readx_poll_timeout ( ioread32 , & regs - > ctrl , val , SNET_DATA_READY ( val ) , 10 ,
SNET_CTRL_TIMEOUT ) ;
}
static u32 snet_read32_word ( struct snet_ctrl_regs __iomem * ctrl_regs , u16 word_idx )
{
return ioread32 ( & ctrl_regs - > data [ word_idx ] ) ;
}
static u32 snet_read_ctrl ( struct snet_ctrl_regs __iomem * ctrl_regs )
{
return ioread32 ( & ctrl_regs - > ctrl ) ;
}
static void snet_write_ctrl ( struct snet_ctrl_regs __iomem * ctrl_regs , u32 val )
{
iowrite32 ( val , & ctrl_regs - > ctrl ) ;
}
static void snet_write_op ( struct snet_ctrl_regs __iomem * ctrl_regs , u32 val )
{
iowrite32 ( val , & ctrl_regs - > op ) ;
}
static int snet_wait_for_dpu_completion ( struct snet_ctrl_regs __iomem * ctrl_regs )
{
/* Wait until the DPU finishes completely.
* It will clear the opcode register .
*/
return snet_wait_for_empty_op ( ctrl_regs ) ;
}
/* Reading ctrl from the DPU:
* buf_size must be 4 B aligned
*
* Steps :
*
* ( 1 ) Verify that the DPU is not in the middle of another operation by
* reading the in_process and error bits in the control register .
* ( 2 ) Write the request opcode and the VQ idx in the opcode register
* and write the buffer size in the control register .
* ( 3 ) Start readind chunks of data , chunk_ready bit indicates that a
* data chunk is available , we signal that we read the data by clearing the bit .
* ( 4 ) Detect that the transfer is completed when the in_process bit
* in the control register is cleared or when the an error appears .
*/
static int snet_ctrl_read_from_dpu ( struct snet * snet , u16 opcode , u16 vq_idx , void * buffer ,
u32 buf_size )
{
struct pci_dev * pdev = snet - > pdev ;
struct snet_ctrl_regs __iomem * regs = snet_get_ctrl ( snet ) ;
u32 * bfr_ptr = ( u32 * ) buffer ;
u32 val ;
u16 buf_words ;
int ret ;
u16 words , i , tot_words = 0 ;
/* Supported for config 2+ */
if ( ! SNET_CFG_VER ( snet , 2 ) )
return - EOPNOTSUPP ;
if ( ! IS_ALIGNED ( buf_size , 4 ) )
return - EINVAL ;
mutex_lock ( & snet - > ctrl_lock ) ;
buf_words = buf_size / 4 ;
/* Make sure control register is empty */
ret = snet_wait_for_empty_ctrl ( regs ) ;
if ( ret ) {
SNET_WARN ( pdev , " Timeout waiting for previous control data to be consumed \n " ) ;
goto exit ;
}
/* We need to write the buffer size in the control register, and the opcode + vq index in
* the opcode register .
* We use a spinlock to serialize the writes .
*/
spin_lock ( & snet - > ctrl_spinlock ) ;
snet_write_ctrl ( regs , buf_words ) ;
snet_write_op ( regs , opcode | ( vq_idx < < 16 ) ) ;
spin_unlock ( & snet - > ctrl_spinlock ) ;
while ( buf_words ! = tot_words ) {
ret = snet_wait_for_data ( regs ) ;
if ( ret ) {
SNET_WARN ( pdev , " Timeout waiting for control data \n " ) ;
goto exit ;
}
val = snet_read_ctrl ( regs ) ;
/* Error? */
if ( val & SNET_CTRL_ERROR_MASK ) {
ret = SNET_VAL_TO_ERR ( val ) ;
SNET_WARN ( pdev , " Error while reading control data from DPU, err %d \n " , ret ) ;
goto exit ;
}
words = min_t ( u16 , val & SNET_CTRL_DATA_SIZE_MASK , buf_words - tot_words ) ;
for ( i = 0 ; i < words ; i + + ) {
* bfr_ptr = snet_read32_word ( regs , i ) ;
bfr_ptr + + ;
}
tot_words + = words ;
/* Is the job completed? */
if ( ! ( val & SNET_CTRL_IN_PROCESS_MASK ) )
break ;
/* Clear the chunk ready bit and continue */
val & = ~ SNET_CTRL_CHUNK_RDY_MASK ;
snet_write_ctrl ( regs , val ) ;
}
ret = snet_wait_for_dpu_completion ( regs ) ;
if ( ret )
SNET_WARN ( pdev , " Timeout waiting for the DPU to complete a control command \n " ) ;
exit :
mutex_unlock ( & snet - > ctrl_lock ) ;
return ret ;
}
/* Send a control message to the DPU using the old mechanism
* used with config version 1.
*/
static int snet_send_ctrl_msg_old ( struct snet * snet , u32 opcode )
{
struct pci_dev * pdev = snet - > pdev ;
struct snet_ctrl_regs __iomem * regs = snet_get_ctrl ( snet ) ;
int ret ;
mutex_lock ( & snet - > ctrl_lock ) ;
/* Old mechanism uses just 1 register, the opcode register.
* Make sure that the opcode register is empty , and that the DPU isn ' t
* processing an old message .
*/
ret = snet_wait_for_empty_op ( regs ) ;
if ( ret ) {
SNET_WARN ( pdev , " Timeout waiting for previous control message to be ACKed \n " ) ;
goto exit ;
}
/* Write the message */
snet_write_op ( regs , opcode ) ;
/* DPU ACKs the message by clearing the opcode register */
ret = snet_wait_for_empty_op ( regs ) ;
if ( ret )
SNET_WARN ( pdev , " Timeout waiting for a control message to be ACKed \n " ) ;
exit :
mutex_unlock ( & snet - > ctrl_lock ) ;
return ret ;
}
/* Send a control message to the DPU.
* A control message is a message without payload .
*/
static int snet_send_ctrl_msg ( struct snet * snet , u16 opcode , u16 vq_idx )
{
struct pci_dev * pdev = snet - > pdev ;
struct snet_ctrl_regs __iomem * regs = snet_get_ctrl ( snet ) ;
u32 val ;
int ret ;
/* If config version is not 2+, use the old mechanism */
if ( ! SNET_CFG_VER ( snet , 2 ) )
return snet_send_ctrl_msg_old ( snet , opcode ) ;
mutex_lock ( & snet - > ctrl_lock ) ;
/* Make sure control register is empty */
ret = snet_wait_for_empty_ctrl ( regs ) ;
if ( ret ) {
SNET_WARN ( pdev , " Timeout waiting for previous control data to be consumed \n " ) ;
goto exit ;
}
/* We need to clear the control register and write the opcode + vq index in the opcode
* register .
* We use a spinlock to serialize the writes .
*/
spin_lock ( & snet - > ctrl_spinlock ) ;
snet_write_ctrl ( regs , 0 ) ;
snet_write_op ( regs , opcode | ( vq_idx < < 16 ) ) ;
spin_unlock ( & snet - > ctrl_spinlock ) ;
/* The DPU ACKs control messages by setting the chunk ready bit
* without data .
*/
ret = snet_wait_for_data ( regs ) ;
if ( ret ) {
SNET_WARN ( pdev , " Timeout waiting for control message to be ACKed \n " ) ;
goto exit ;
}
/* Check for errors */
val = snet_read_ctrl ( regs ) ;
ret = SNET_VAL_TO_ERR ( val ) ;
/* Clear the chunk ready bit */
val & = ~ SNET_CTRL_CHUNK_RDY_MASK ;
snet_write_ctrl ( regs , val ) ;
ret = snet_wait_for_dpu_completion ( regs ) ;
if ( ret )
SNET_WARN ( pdev , " Timeout waiting for DPU to complete a control command, err %d \n " ,
ret ) ;
exit :
mutex_unlock ( & snet - > ctrl_lock ) ;
return ret ;
}
void snet_ctrl_clear ( struct snet * snet )
{
struct snet_ctrl_regs __iomem * regs = snet_get_ctrl ( snet ) ;
snet_write_op ( regs , 0 ) ;
}
int snet_destroy_dev ( struct snet * snet )
{
return snet_send_ctrl_msg ( snet , SNET_CTRL_OP_DESTROY , 0 ) ;
}
int snet_read_vq_state ( struct snet * snet , u16 idx , struct vdpa_vq_state * state )
{
return snet_ctrl_read_from_dpu ( snet , SNET_CTRL_OP_READ_VQ_STATE , idx , state ,
sizeof ( * state ) ) ;
}
2023-04-13 10:33:37 +03:00
int snet_suspend_dev ( struct snet * snet )
{
return snet_send_ctrl_msg ( snet , SNET_CTRL_OP_SUSPEND , 0 ) ;
}
2023-05-02 16:10:48 +03:00
int snet_resume_dev ( struct snet * snet )
{
return snet_send_ctrl_msg ( snet , SNET_CTRL_OP_RESUME , 0 ) ;
}