From d63cc24933c774ea464090af1998a7b63f11c166 Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Wed, 8 Apr 2020 12:42:09 +0300 Subject: [PATCH 01/20] net/mlx5: Export resource dump interface Export some of the resource dump API. mlx5_ib driver will use it in downstream patches. Signed-off-by: Maor Gottlieb Signed-off-by: Leon Romanovsky --- .../mellanox/mlx5/core/diag/rsc_dump.c | 3 ++ .../mellanox/mlx5/core/diag/rsc_dump.h | 33 +------------ include/linux/mlx5/rsc_dump.h | 48 +++++++++++++++++++ 3 files changed, 52 insertions(+), 32 deletions(-) create mode 100644 include/linux/mlx5/rsc_dump.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c index 17ab7efe693d..10218c2324cc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c @@ -130,11 +130,13 @@ struct mlx5_rsc_dump_cmd *mlx5_rsc_dump_cmd_create(struct mlx5_core_dev *dev, cmd->mem_size = key->size; return cmd; } +EXPORT_SYMBOL(mlx5_rsc_dump_cmd_create); void mlx5_rsc_dump_cmd_destroy(struct mlx5_rsc_dump_cmd *cmd) { kfree(cmd); } +EXPORT_SYMBOL(mlx5_rsc_dump_cmd_destroy); int mlx5_rsc_dump_next(struct mlx5_core_dev *dev, struct mlx5_rsc_dump_cmd *cmd, struct page *page, int *size) @@ -155,6 +157,7 @@ int mlx5_rsc_dump_next(struct mlx5_core_dev *dev, struct mlx5_rsc_dump_cmd *cmd, return more_dump; } +EXPORT_SYMBOL(mlx5_rsc_dump_next); #define MLX5_RSC_DUMP_MENU_SEGMENT 0xffff static int mlx5_rsc_dump_menu(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h index 148270073e71..64c4956db6d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h @@ -4,41 +4,10 @@ #ifndef __MLX5_RSC_DUMP_H #define __MLX5_RSC_DUMP_H +#include #include #include "mlx5_core.h" -enum mlx5_sgmt_type { - MLX5_SGMT_TYPE_HW_CQPC, - MLX5_SGMT_TYPE_HW_SQPC, - MLX5_SGMT_TYPE_HW_RQPC, - MLX5_SGMT_TYPE_FULL_SRQC, - MLX5_SGMT_TYPE_FULL_CQC, - MLX5_SGMT_TYPE_FULL_EQC, - MLX5_SGMT_TYPE_FULL_QPC, - MLX5_SGMT_TYPE_SND_BUFF, - MLX5_SGMT_TYPE_RCV_BUFF, - MLX5_SGMT_TYPE_SRQ_BUFF, - MLX5_SGMT_TYPE_CQ_BUFF, - MLX5_SGMT_TYPE_EQ_BUFF, - MLX5_SGMT_TYPE_SX_SLICE, - MLX5_SGMT_TYPE_SX_SLICE_ALL, - MLX5_SGMT_TYPE_RDB, - MLX5_SGMT_TYPE_RX_SLICE_ALL, - MLX5_SGMT_TYPE_MENU, - MLX5_SGMT_TYPE_TERMINATE, - - MLX5_SGMT_TYPE_NUM, /* Keep last */ -}; - -struct mlx5_rsc_key { - enum mlx5_sgmt_type rsc; - int index1; - int index2; - int num_of_obj1; - int num_of_obj2; - int size; -}; - #define MLX5_RSC_DUMP_ALL 0xFFFF struct mlx5_rsc_dump_cmd; struct mlx5_rsc_dump; diff --git a/include/linux/mlx5/rsc_dump.h b/include/linux/mlx5/rsc_dump.h new file mode 100644 index 000000000000..87415fa754fe --- /dev/null +++ b/include/linux/mlx5/rsc_dump.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies inc. */ + +#include + +#ifndef __MLX5_RSC_DUMP +#define __MLX5_RSC_DUMP + +enum mlx5_sgmt_type { + MLX5_SGMT_TYPE_HW_CQPC, + MLX5_SGMT_TYPE_HW_SQPC, + MLX5_SGMT_TYPE_HW_RQPC, + MLX5_SGMT_TYPE_FULL_SRQC, + MLX5_SGMT_TYPE_FULL_CQC, + MLX5_SGMT_TYPE_FULL_EQC, + MLX5_SGMT_TYPE_FULL_QPC, + MLX5_SGMT_TYPE_SND_BUFF, + MLX5_SGMT_TYPE_RCV_BUFF, + MLX5_SGMT_TYPE_SRQ_BUFF, + MLX5_SGMT_TYPE_CQ_BUFF, + MLX5_SGMT_TYPE_EQ_BUFF, + MLX5_SGMT_TYPE_SX_SLICE, + MLX5_SGMT_TYPE_SX_SLICE_ALL, + MLX5_SGMT_TYPE_RDB, + MLX5_SGMT_TYPE_RX_SLICE_ALL, + MLX5_SGMT_TYPE_MENU, + MLX5_SGMT_TYPE_TERMINATE, + + MLX5_SGMT_TYPE_NUM, /* Keep last */ +}; + +struct mlx5_rsc_key { + enum mlx5_sgmt_type rsc; + int index1; + int index2; + int num_of_obj1; + int num_of_obj2; + int size; +}; + +struct mlx5_rsc_dump_cmd; + +struct mlx5_rsc_dump_cmd *mlx5_rsc_dump_cmd_create(struct mlx5_core_dev *dev, + struct mlx5_rsc_key *key); +void mlx5_rsc_dump_cmd_destroy(struct mlx5_rsc_dump_cmd *cmd); +int mlx5_rsc_dump_next(struct mlx5_core_dev *dev, struct mlx5_rsc_dump_cmd *cmd, + struct page *page, int *size); +#endif /* __MLX5_RSC_DUMP */ From 608ca553c9a2008908120e0e45b1cfc4aefcfd49 Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Wed, 8 Apr 2020 12:36:20 +0300 Subject: [PATCH 02/20] net/mlx5: Add support in query QP, CQ and MKEY segments Introduce new resource dump segments - PRM_QUERY_QP, PRM_QUERY_CQ and PRM_QUERY_MKEY. These segments contains the resource dump in PRM query format. Signed-off-by: Maor Gottlieb Signed-off-by: Leon Romanovsky --- drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c | 3 +++ include/linux/mlx5/rsc_dump.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c index 10218c2324cc..4924a5658853 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c @@ -23,6 +23,9 @@ static const char *const mlx5_rsc_sgmt_name[] = { MLX5_SGMT_STR_ASSING(SX_SLICE_ALL), MLX5_SGMT_STR_ASSING(RDB), MLX5_SGMT_STR_ASSING(RX_SLICE_ALL), + MLX5_SGMT_STR_ASSING(PRM_QUERY_QP), + MLX5_SGMT_STR_ASSING(PRM_QUERY_CQ), + MLX5_SGMT_STR_ASSING(PRM_QUERY_MKEY), }; struct mlx5_rsc_dump { diff --git a/include/linux/mlx5/rsc_dump.h b/include/linux/mlx5/rsc_dump.h index 87415fa754fe..d11c0b228620 100644 --- a/include/linux/mlx5/rsc_dump.h +++ b/include/linux/mlx5/rsc_dump.h @@ -23,6 +23,9 @@ enum mlx5_sgmt_type { MLX5_SGMT_TYPE_SX_SLICE_ALL, MLX5_SGMT_TYPE_RDB, MLX5_SGMT_TYPE_RX_SLICE_ALL, + MLX5_SGMT_TYPE_PRM_QUERY_QP, + MLX5_SGMT_TYPE_PRM_QUERY_CQ, + MLX5_SGMT_TYPE_PRM_QUERY_MKEY, MLX5_SGMT_TYPE_MENU, MLX5_SGMT_TYPE_TERMINATE, From 9205d7b1c1cffa827c23bdbf35e04c7cbe1e1f10 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 25 Jun 2020 22:59:41 -0700 Subject: [PATCH 03/20] net/mlx5: Avoid RDMA file inclusion in core driver mlx5 cq.h does not depend on RDMA verbs. Remove RDMA verbs file inclusion. Signed-off-by: Parav Pandit Signed-off-by: Saeed Mahameed --- include/linux/mlx5/cq.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/mlx5/cq.h b/include/linux/mlx5/cq.h index b5a9399e07ee..7bfb67363434 100644 --- a/include/linux/mlx5/cq.h +++ b/include/linux/mlx5/cq.h @@ -33,7 +33,6 @@ #ifndef MLX5_CORE_CQ_H #define MLX5_CORE_CQ_H -#include #include #include From 188f0f988bdf27a94208568314e3b0c012d1629c Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 25 Jun 2020 22:59:42 -0700 Subject: [PATCH 04/20] net/mlx5: Avoid eswitch header inclusion in fs core layer Flow steering core layer is independent of the eswitch layer. Hence avoid fs_core dependency on eswitch. Fixes: 328edb499f99 ("net/mlx5: Split FDB fast path prio to multiple namespaces") Signed-off-by: Parav Pandit Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/eswitch.h | 10 ---------- drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/fs_core.h | 10 ++++++++++ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index a5175e98c0b3..bb309b2f77f2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -44,16 +44,6 @@ #include "lib/mpfs.h" #include "en/tc_ct.h" -#define FDB_TC_MAX_CHAIN 3 -#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) -#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) - -/* The index of the last real chain (FT) + 1 as chain zero is valid as well */ -#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) - -#define FDB_TC_MAX_PRIO 16 -#define FDB_TC_LEVELS_PER_PRIO 2 - #ifdef CONFIG_MLX5_ESWITCH #define ESW_OFFLOADS_DEFAULT_NUM_GROUPS 15 diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 13e2fb79c21a..e47a66983935 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -41,7 +41,6 @@ #include "diag/fs_tracepoint.h" #include "accel/ipsec.h" #include "fpga/ipsec.h" -#include "eswitch.h" #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ sizeof(struct init_tree_node)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index 825b662f809b..afe7f0bffb93 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -39,6 +39,16 @@ #include #include +#define FDB_TC_MAX_CHAIN 3 +#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) +#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) + +/* The index of the last real chain (FT) + 1 as chain zero is valid as well */ +#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) + +#define FDB_TC_MAX_PRIO 16 +#define FDB_TC_LEVELS_PER_PRIO 2 + struct mlx5_modify_hdr { enum mlx5_flow_namespace_type ns_type; union { From 2d1b69ed65ee033aa541518cc9f6a815296ac493 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Thu, 25 Jun 2020 22:59:43 -0700 Subject: [PATCH 05/20] net/mlx5: kTLS, Improve TLS params layout structures Add explicit WQE segment structures for the TLS static and progress params. According to the HW spec, TISN is not part of the progress params context, take it out of it. Rename the control segment tisn field as it could hold either a TIS or a TIR number. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h | 2 +- .../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 2 +- .../ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c | 14 +++++++++----- .../mellanox/mlx5/core/en_accel/tls_rxtx.c | 2 +- include/linux/mlx5/device.h | 9 +++++++++ include/linux/mlx5/mlx5_ifc.h | 5 +---- include/linux/mlx5/qp.h | 2 +- 7 files changed, 23 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index bfd3e1161bc6..31cac239563d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -182,7 +182,7 @@ mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map, static inline bool mlx5e_transport_inline_tx_wqe(struct mlx5_wqe_ctrl_seg *cseg) { - return cseg && !!cseg->tisn; + return cseg && !!cseg->tis_tir_num; } static inline u8 diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index c6180892cfcb..806ed185dd4c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -19,7 +19,7 @@ #define MLX5E_KTLS_PROGRESS_WQE_SZ \ (offsetof(struct mlx5e_tx_wqe, tls_progress_params_ctx) + \ - MLX5_ST_SZ_BYTES(tls_progress_params)) + sizeof(struct mlx5_wqe_tls_progress_params_seg)) #define MLX5E_KTLS_PROGRESS_WQEBBS \ (DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_BB)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 3cd78d9503c1..ad7300f19815 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -64,7 +64,7 @@ build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn, cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | STATIC_PARAMS_DS_CNT); cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; - cseg->tisn = cpu_to_be32(priv_tx->tisn << 8); + cseg->tis_tir_num = cpu_to_be32(priv_tx->tisn << 8); ucseg->flags = MLX5_UMR_INLINE; ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); @@ -75,10 +75,14 @@ build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn, static void fill_progress_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx) { - MLX5_SET(tls_progress_params, ctx, tisn, priv_tx->tisn); - MLX5_SET(tls_progress_params, ctx, record_tracker_state, + struct mlx5_wqe_tls_progress_params_seg *params; + + params = ctx; + + params->tis_tir_num = cpu_to_be32(priv_tx->tisn); + MLX5_SET(tls_progress_params, params->ctx, record_tracker_state, MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); - MLX5_SET(tls_progress_params, ctx, auth_state, + MLX5_SET(tls_progress_params, params->ctx, auth_state, MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD); } @@ -284,7 +288,7 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); - cseg->tisn = cpu_to_be32(tisn << 8); + cseg->tis_tir_num = cpu_to_be32(tisn << 8); cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; fsz = skb_frag_size(frag); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index 05454a843b28..72d26fbc8d5b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -305,7 +305,7 @@ err_out: void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, struct mlx5e_accel_tx_tls_state *state) { - cseg->tisn = cpu_to_be32(state->tls_tisn << 8); + cseg->tis_tir_num = cpu_to_be32(state->tls_tisn << 8); } static int tls_update_resync_sn(struct net_device *netdev, diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 1bc27aca648b..57db125e5802 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -458,6 +458,15 @@ enum { MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS = 0x2, }; +struct mlx5_wqe_tls_static_params_seg { + u8 ctx[MLX5_ST_SZ_BYTES(tls_static_params)]; +}; + +struct mlx5_wqe_tls_progress_params_seg { + __be32 tis_tir_num; + u8 ctx[MLX5_ST_SZ_BYTES(tls_progress_params)]; +}; + enum { MLX5_SET_PORT_RESET_QKEY = 0, MLX5_SET_PORT_GUID0 = 16, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 116bd9bb347f..a227518c70cf 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -10638,16 +10638,13 @@ struct mlx5_ifc_tls_static_params_bits { }; struct mlx5_ifc_tls_progress_params_bits { - u8 reserved_at_0[0x8]; - u8 tisn[0x18]; - u8 next_record_tcp_sn[0x20]; u8 hw_resync_tcp_sn[0x20]; u8 record_tracker_state[0x2]; u8 auth_state[0x2]; - u8 reserved_at_64[0x4]; + u8 reserved_at_44[0x4]; u8 hw_offset_record_number[0x18]; }; diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index b8992b861ae6..36492a1342cf 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -209,7 +209,7 @@ struct mlx5_wqe_ctrl_seg { __be32 general_id; __be32 imm; __be32 umr_mkey; - __be32 tisn; + __be32 tis_tir_num; }; }; From 8d94b590f1e4d60e0a1f51a43aded1b0e6fd06f7 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Tue, 26 Nov 2019 16:23:23 +0200 Subject: [PATCH 06/20] net/mlx5e: Turn XSK ICOSQ into a general asynchronous one There is an upcoming demand (in downstream patches) for an ICOSQ to be populated out of the NAPI context, asynchronously. There is already an existing one serving XSK-related use case. In this patch, promote this ICOSQ to serve as general async ICOSQ, to be used for XSK and non-XSK flows. As part of this, the reg_umr bit of the SQ context is now set (if capable), as the general async ICOSQ should support possible posts of UMR WQEs. Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 8 ++-- .../mellanox/mlx5/core/en/xsk/setup.c | 47 ++----------------- .../ethernet/mellanox/mlx5/core/en/xsk/tx.c | 12 ++--- .../net/ethernet/mellanox/mlx5/core/en_main.c | 24 +++++++++- .../net/ethernet/mellanox/mlx5/core/en_txrx.c | 12 ++--- 5 files changed, 42 insertions(+), 61 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 842db20493df..29265ca6050c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -651,9 +651,11 @@ struct mlx5e_channel { /* AF_XDP zero-copy */ struct mlx5e_rq xskrq; struct mlx5e_xdpsq xsksq; - struct mlx5e_icosq xskicosq; - /* xskicosq can be accessed from any CPU - the spinlock protects it. */ - spinlock_t xskicosq_lock; + + /* Async ICOSQ */ + struct mlx5e_icosq async_icosq; + /* async_icosq can be accessed from any CPU - the spinlock protects it. */ + spinlock_t async_icosq_lock; /* data path - accessed per napi poll */ struct irq_desc *irq_desc; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 2c80205dc939..1eb817e62830 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -34,31 +34,15 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, } } -static void mlx5e_build_xskicosq_param(struct mlx5e_priv *priv, - u8 log_wq_size, - struct mlx5e_sq_param *param) -{ - void *sqc = param->sqc; - void *wq = MLX5_ADDR_OF(sqc, sqc, wq); - - mlx5e_build_sq_param_common(priv, param); - - MLX5_SET(wq, wq, log_wq_sz, log_wq_size); -} - static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, struct mlx5e_channel_param *cparam) { - const u8 xskicosq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; - mlx5e_build_rq_param(priv, params, xsk, &cparam->rq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); - mlx5e_build_xskicosq_param(priv, xskicosq_size, &cparam->icosq); mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq); mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); - mlx5e_build_ico_cq_param(priv, xskicosq_size, &cparam->icosq_cq); } int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, @@ -66,7 +50,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_channel *c) { struct mlx5e_channel_param *cparam; - struct dim_cq_moder icocq_moder = {}; int err; if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) @@ -100,31 +83,12 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_tx_cq; - err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->xskicosq.cq); - if (unlikely(err)) - goto err_close_sq; - - /* Create a dedicated SQ for posting NOPs whenever we need an IRQ to be - * triggered and NAPI to be called on the correct CPU. - */ - err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->xskicosq); - if (unlikely(err)) - goto err_close_icocq; - kvfree(cparam); - spin_lock_init(&c->xskicosq_lock); - set_bit(MLX5E_CHANNEL_STATE_XSK, c->state); return 0; -err_close_icocq: - mlx5e_close_cq(&c->xskicosq.cq); - -err_close_sq: - mlx5e_close_xdpsq(&c->xsksq); - err_close_tx_cq: mlx5e_close_cq(&c->xsksq.cq); @@ -148,32 +112,27 @@ void mlx5e_close_xsk(struct mlx5e_channel *c) mlx5e_close_rq(&c->xskrq); mlx5e_close_cq(&c->xskrq.cq); - mlx5e_close_icosq(&c->xskicosq); - mlx5e_close_cq(&c->xskicosq.cq); mlx5e_close_xdpsq(&c->xsksq); mlx5e_close_cq(&c->xsksq.cq); memset(&c->xskrq, 0, sizeof(c->xskrq)); memset(&c->xsksq, 0, sizeof(c->xsksq)); - memset(&c->xskicosq, 0, sizeof(c->xskicosq)); } void mlx5e_activate_xsk(struct mlx5e_channel *c) { - mlx5e_activate_icosq(&c->xskicosq); set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); /* TX queue is created active. */ - spin_lock(&c->xskicosq_lock); - mlx5e_trigger_irq(&c->xskicosq); - spin_unlock(&c->xskicosq_lock); + spin_lock(&c->async_icosq_lock); + mlx5e_trigger_irq(&c->async_icosq); + spin_unlock(&c->async_icosq_lock); } void mlx5e_deactivate_xsk(struct mlx5e_channel *c) { mlx5e_deactivate_rq(&c->xskrq); /* TX queue is disabled on close. */ - mlx5e_deactivate_icosq(&c->xskicosq); } static int mlx5e_redirect_xsk_rqt(struct mlx5e_priv *priv, u16 ix, u32 rqn) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index 83dce9cdb8c2..e0b3c61af93e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -26,19 +26,19 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) return -ENXIO; if (!napi_if_scheduled_mark_missed(&c->napi)) { - /* To avoid WQE overrun, don't post a NOP if XSKICOSQ is not + /* To avoid WQE overrun, don't post a NOP if async_icosq is not * active and not polled by NAPI. Return 0, because the upcoming * activate will trigger the IRQ for us. */ - if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->xskicosq.state))) + if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) return 0; - if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->xskicosq.state)) + if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) return 0; - spin_lock(&c->xskicosq_lock); - mlx5e_trigger_irq(&c->xskicosq); - spin_unlock(&c->xskicosq_lock); + spin_lock(&c->async_icosq_lock); + mlx5e_trigger_irq(&c->async_icosq); + spin_unlock(&c->async_icosq_lock); } return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index a836a02a2116..72ce5808d583 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1817,10 +1817,14 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, struct dim_cq_moder icocq_moder = {0, 0}; int err; - err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); + err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->async_icosq.cq); if (err) return err; + err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); + if (err) + goto err_close_async_icosq_cq; + err = mlx5e_open_tx_cqs(c, params, cparam); if (err) goto err_close_icosq_cq; @@ -1841,10 +1845,16 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, napi_enable(&c->napi); - err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); + spin_lock_init(&c->async_icosq_lock); + + err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->async_icosq); if (err) goto err_disable_napi; + err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); + if (err) + goto err_close_async_icosq; + err = mlx5e_open_sqs(c, params, cparam); if (err) goto err_close_icosq; @@ -1879,6 +1889,9 @@ err_close_sqs: err_close_icosq: mlx5e_close_icosq(&c->icosq); +err_close_async_icosq: + mlx5e_close_icosq(&c->async_icosq); + err_disable_napi: napi_disable(&c->napi); @@ -1897,6 +1910,9 @@ err_close_tx_cqs: err_close_icosq_cq: mlx5e_close_cq(&c->icosq.cq); +err_close_async_icosq_cq: + mlx5e_close_cq(&c->async_icosq.cq); + return err; } @@ -1908,6 +1924,7 @@ static void mlx5e_close_queues(struct mlx5e_channel *c) mlx5e_close_xdpsq(&c->rq_xdpsq); mlx5e_close_sqs(c); mlx5e_close_icosq(&c->icosq); + mlx5e_close_icosq(&c->async_icosq); napi_disable(&c->napi); if (c->xdp) mlx5e_close_cq(&c->rq_xdpsq.cq); @@ -1915,6 +1932,7 @@ static void mlx5e_close_queues(struct mlx5e_channel *c) mlx5e_close_cq(&c->xdpsq.cq); mlx5e_close_tx_cqs(c); mlx5e_close_cq(&c->icosq.cq); + mlx5e_close_cq(&c->async_icosq.cq); } static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) @@ -1995,6 +2013,7 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c) for (tc = 0; tc < c->num_tc; tc++) mlx5e_activate_txqsq(&c->sq[tc]); mlx5e_activate_icosq(&c->icosq); + mlx5e_activate_icosq(&c->async_icosq); mlx5e_activate_rq(&c->rq); if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) @@ -2009,6 +2028,7 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) mlx5e_deactivate_xsk(c); mlx5e_deactivate_rq(&c->rq); + mlx5e_deactivate_icosq(&c->async_icosq); mlx5e_deactivate_icosq(&c->icosq); for (tc = 0; tc < c->num_tc; tc++) mlx5e_deactivate_txqsq(&c->sq[tc]); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index 8480278f2ee2..e3dbab2a294c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -149,17 +149,17 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) } mlx5e_poll_ico_cq(&c->icosq.cq); + if (mlx5e_poll_ico_cq(&c->async_icosq.cq)) + /* Don't clear the flag if nothing was polled to prevent + * queueing more WQEs and overflowing the async ICOSQ. + */ + clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state); busy |= INDIRECT_CALL_2(rq->post_wqes, mlx5e_post_rx_mpwqes, mlx5e_post_rx_wqes, rq); if (xsk_open) { - if (mlx5e_poll_ico_cq(&c->xskicosq.cq)) - /* Don't clear the flag if nothing was polled to prevent - * queueing more WQEs and overflowing XSKICOSQ. - */ - clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->xskicosq.state); busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq); busy_xsk |= mlx5e_napi_xsk_post(xsksq, xskrq); } @@ -189,11 +189,11 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) mlx5e_cq_arm(&rq->cq); mlx5e_cq_arm(&c->icosq.cq); + mlx5e_cq_arm(&c->async_icosq.cq); mlx5e_cq_arm(&c->xdpsq.cq); if (xsk_open) { mlx5e_handle_rx_dim(xskrq); - mlx5e_cq_arm(&c->xskicosq.cq); mlx5e_cq_arm(&xsksq->cq); mlx5e_cq_arm(&xskrq->cq); } From c293ac927fbb0c804b91118535f64d5d9f7a0d8f Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Sat, 13 Jun 2020 22:53:32 +0300 Subject: [PATCH 07/20] net/mlx5e: Refactor build channel params Take the CQ params into their respective RQ/SQ params. Split the params build of the different ICOSQs (sync and async), as they require different init values. Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/en/params.h | 32 +++++++++---------- .../mellanox/mlx5/core/en/xsk/setup.c | 6 ++-- .../net/ethernet/mellanox/mlx5/core/en_main.c | 30 +++++++++-------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 989d8f429438..a87273e801b2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -11,18 +11,6 @@ struct mlx5e_xsk_param { u16 chunk_size; }; -struct mlx5e_rq_param { - u32 rqc[MLX5_ST_SZ_DW(rqc)]; - struct mlx5_wq_param wq; - struct mlx5e_rq_frags_info frags_info; -}; - -struct mlx5e_sq_param { - u32 sqc[MLX5_ST_SZ_DW(sqc)]; - struct mlx5_wq_param wq; - bool is_mpw; -}; - struct mlx5e_cq_param { u32 cqc[MLX5_ST_SZ_DW(cqc)]; struct mlx5_wq_param wq; @@ -30,14 +18,26 @@ struct mlx5e_cq_param { u8 cq_period_mode; }; +struct mlx5e_rq_param { + struct mlx5e_cq_param cqp; + u32 rqc[MLX5_ST_SZ_DW(rqc)]; + struct mlx5_wq_param wq; + struct mlx5e_rq_frags_info frags_info; +}; + +struct mlx5e_sq_param { + struct mlx5e_cq_param cqp; + u32 sqc[MLX5_ST_SZ_DW(sqc)]; + struct mlx5_wq_param wq; + bool is_mpw; +}; + struct mlx5e_channel_param { struct mlx5e_rq_param rq; - struct mlx5e_sq_param sq; + struct mlx5e_sq_param txq_sq; struct mlx5e_sq_param xdp_sq; struct mlx5e_sq_param icosq; - struct mlx5e_cq_param rx_cq; - struct mlx5e_cq_param tx_cq; - struct mlx5e_cq_param icosq_cq; + struct mlx5e_sq_param async_icosq; }; static inline bool mlx5e_qid_get_ch_if_in_group(struct mlx5e_params *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 1eb817e62830..cc46414773b5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -41,8 +41,6 @@ static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv, { mlx5e_build_rq_param(priv, params, xsk, &cparam->rq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); - mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq); - mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); } int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, @@ -61,7 +59,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, mlx5e_build_xsk_cparam(priv, params, xsk, cparam); - err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->xskrq.cq); + err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rq.cqp, &c->xskrq.cq); if (unlikely(err)) goto err_free_cparam; @@ -69,7 +67,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_rx_cq; - err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xsksq.cq); + err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &c->xsksq.cq); if (unlikely(err)) goto err_close_rq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 72ce5808d583..11997c23dfb5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1675,7 +1675,7 @@ static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, for (tc = 0; tc < c->num_tc; tc++) { err = mlx5e_open_cq(c, params->tx_cq_moderation, - &cparam->tx_cq, &c->sq[tc].cq); + &cparam->txq_sq.cqp, &c->sq[tc].cq); if (err) goto err_close_tx_cqs; } @@ -1707,7 +1707,7 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c, int txq_ix = c->ix + tc * params->num_channels; err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, - params, &cparam->sq, &c->sq[tc], tc); + params, &cparam->txq_sq, &c->sq[tc], tc); if (err) goto err_close_sqs; } @@ -1817,11 +1817,11 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, struct dim_cq_moder icocq_moder = {0, 0}; int err; - err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->async_icosq.cq); + err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq.cqp, &c->async_icosq.cq); if (err) return err; - err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); + err = mlx5e_open_cq(c, icocq_moder, &cparam->async_icosq.cqp, &c->icosq.cq); if (err) goto err_close_async_icosq_cq; @@ -1829,17 +1829,16 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, if (err) goto err_close_icosq_cq; - err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq); + err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &c->xdpsq.cq); if (err) goto err_close_tx_cqs; - err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq); + err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rq.cqp, &c->rq.cq); if (err) goto err_close_xdp_tx_cqs; - /* XDP SQ CQ params are same as normal TXQ sq CQ params */ err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation, - &cparam->tx_cq, &c->rq_xdpsq.cq) : 0; + &cparam->xdp_sq.cqp, &c->rq_xdpsq.cq) : 0; if (err) goto err_close_rx_cq; @@ -1847,7 +1846,7 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, spin_lock_init(&c->async_icosq_lock); - err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->async_icosq); + err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq); if (err) goto err_disable_napi; @@ -2158,6 +2157,7 @@ void mlx5e_build_rq_param(struct mlx5e_priv *priv, MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); param->wq.buf_numa_node = dev_to_node(mdev->device); + mlx5e_build_rx_cq_param(priv, params, xsk, ¶m->cqp); } static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, @@ -2200,6 +2200,7 @@ static void mlx5e_build_sq_param(struct mlx5e_priv *priv, mlx5e_build_sq_param_common(priv, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); MLX5_SET(sqc, sqc, allow_swp, allow_swp); + mlx5e_build_tx_cq_param(priv, params, ¶m->cqp); } static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, @@ -2276,6 +2277,7 @@ void mlx5e_build_icosq_param(struct mlx5e_priv *priv, MLX5_SET(wq, wq, log_wq_sz, log_wq_size); MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq)); + mlx5e_build_ico_cq_param(priv, log_wq_size, ¶m->cqp); } void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, @@ -2288,6 +2290,7 @@ void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, mlx5e_build_sq_param_common(priv, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); + mlx5e_build_tx_cq_param(priv, params, ¶m->cqp); } static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, @@ -2306,18 +2309,17 @@ static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_channel_param *cparam) { - u8 icosq_log_wq_sz; + u8 icosq_log_wq_sz, async_icosq_log_wq_sz; mlx5e_build_rq_param(priv, params, NULL, &cparam->rq); icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); + async_icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; - mlx5e_build_sq_param(priv, params, &cparam->sq); + mlx5e_build_sq_param(priv, params, &cparam->txq_sq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq); - mlx5e_build_rx_cq_param(priv, params, NULL, &cparam->rx_cq); - mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); - mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq); + mlx5e_build_icosq_param(priv, async_icosq_log_wq_sz, &cparam->async_icosq); } int mlx5e_open_channels(struct mlx5e_priv *priv, From b8922a73ec3e37db8836dbcce2af19d85a8bc9ef Mon Sep 17 00:00:00 2001 From: Saeed Mahameed Date: Thu, 2 Apr 2020 02:02:33 -0700 Subject: [PATCH 08/20] net/mlx5e: API to manipulate TTC rules destinations Store the default destinations of the on-load generated TTC (Traffic Type Classifier) rules in the ttc rules table. Introduce TTC API functions to manipulate/restore and get the TTC rule destination and use these API functions in arfs implementation. This will allow a better decoupling between TTC implementation and its users. Signed-off-by: Saeed Mahameed Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy --- .../net/ethernet/mellanox/mlx5/core/en/fs.h | 16 +++- .../net/ethernet/mellanox/mlx5/core/en_arfs.c | 34 +++----- .../net/ethernet/mellanox/mlx5/core/en_fs.c | 84 ++++++++++++++----- 3 files changed, 86 insertions(+), 48 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index 0416f7712109..c633579474c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -105,11 +105,16 @@ enum mlx5e_tunnel_types { bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev); +struct mlx5e_ttc_rule { + struct mlx5_flow_handle *rule; + struct mlx5_flow_destination default_dest; +}; + /* L3/L4 traffic type classifier */ struct mlx5e_ttc_table { - struct mlx5e_flow_table ft; - struct mlx5_flow_handle *rules[MLX5E_NUM_TT]; - struct mlx5_flow_handle *tunnel_rules[MLX5E_NUM_TUNNEL_TT]; + struct mlx5e_flow_table ft; + struct mlx5e_ttc_rule rules[MLX5E_NUM_TT]; + struct mlx5_flow_handle *tunnel_rules[MLX5E_NUM_TUNNEL_TT]; }; /* NIC prio FTS */ @@ -248,6 +253,11 @@ void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv, struct mlx5e_ttc_table *ttc); void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft); +int mlx5e_ttc_fwd_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type, + struct mlx5_flow_destination *new_dest); +struct mlx5_flow_destination +mlx5e_ttc_get_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type); +int mlx5e_ttc_fwd_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type); void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv); void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index c4c9d6cda7e6..39475f6565c7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -90,23 +90,15 @@ static enum mlx5e_traffic_types arfs_get_tt(enum arfs_type type) static int arfs_disable(struct mlx5e_priv *priv) { - struct mlx5_flow_destination dest = {}; - struct mlx5e_tir *tir = priv->indir_tir; - int err = 0; - int tt; - int i; + int err, i; - dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; for (i = 0; i < ARFS_NUM_TYPES; i++) { - dest.tir_num = tir[i].tirn; - tt = arfs_get_tt(i); - /* Modify ttc rules destination to bypass the aRFS tables*/ - err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], - &dest, NULL); + /* Modify ttc rules destination back to their default */ + err = mlx5e_ttc_fwd_default_dest(priv, arfs_get_tt(i)); if (err) { netdev_err(priv->netdev, - "%s: modify ttc destination failed\n", - __func__); + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, arfs_get_tt(i), err); return err; } } @@ -125,21 +117,17 @@ int mlx5e_arfs_disable(struct mlx5e_priv *priv) int mlx5e_arfs_enable(struct mlx5e_priv *priv) { struct mlx5_flow_destination dest = {}; - int err = 0; - int tt; - int i; + int err, i; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < ARFS_NUM_TYPES; i++) { dest.ft = priv->fs.arfs.arfs_tables[i].ft.t; - tt = arfs_get_tt(i); /* Modify ttc rules destination to point on the aRFS FTs */ - err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], - &dest, NULL); + err = mlx5e_ttc_fwd_dest(priv, arfs_get_tt(i), &dest); if (err) { netdev_err(priv->netdev, - "%s: modify ttc destination failed err=%d\n", - __func__, err); + "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", + __func__, arfs_get_tt(i), err); arfs_disable(priv); return err; } @@ -186,8 +174,10 @@ static int arfs_add_default_rule(struct mlx5e_priv *priv, return -EINVAL; } + /* FIXME: Must use mlx5e_ttc_get_default_dest(), + * but can't since TTC default is not setup yet ! + */ dest.tir_num = tir[tt].tirn; - arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL, &flow_act, &dest, 1); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 73d3dc07331f..64d002d92250 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -672,9 +672,9 @@ static void mlx5e_cleanup_ttc_rules(struct mlx5e_ttc_table *ttc) int i; for (i = 0; i < MLX5E_NUM_TT; i++) { - if (!IS_ERR_OR_NULL(ttc->rules[i])) { - mlx5_del_flow_rules(ttc->rules[i]); - ttc->rules[i] = NULL; + if (!IS_ERR_OR_NULL(ttc->rules[i].rule)) { + mlx5_del_flow_rules(ttc->rules[i].rule); + ttc->rules[i].rule = NULL; } } @@ -857,7 +857,8 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv, struct mlx5e_ttc_table *ttc) { struct mlx5_flow_destination dest = {}; - struct mlx5_flow_handle **rules; + struct mlx5_flow_handle **trules; + struct mlx5e_ttc_rule *rules; struct mlx5_flow_table *ft; int tt; int err; @@ -867,39 +868,47 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv, dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; for (tt = 0; tt < MLX5E_NUM_TT; tt++) { + struct mlx5e_ttc_rule *rule = &rules[tt]; + if (tt == MLX5E_TT_ANY) dest.tir_num = params->any_tt_tirn; else dest.tir_num = params->indir_tirn[tt]; - rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, - ttc_rules[tt].etype, - ttc_rules[tt].proto); - if (IS_ERR(rules[tt])) + + rule->rule = mlx5e_generate_ttc_rule(priv, ft, &dest, + ttc_rules[tt].etype, + ttc_rules[tt].proto); + if (IS_ERR(rule->rule)) { + err = PTR_ERR(rule->rule); + rule->rule = NULL; goto del_rules; + } + rule->default_dest = dest; } if (!params->inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) return 0; - rules = ttc->tunnel_rules; + trules = ttc->tunnel_rules; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = params->inner_ttc->ft.t; for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) { if (!mlx5e_tunnel_proto_supported(priv->mdev, ttc_tunnel_rules[tt].proto)) continue; - rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, - ttc_tunnel_rules[tt].etype, - ttc_tunnel_rules[tt].proto); - if (IS_ERR(rules[tt])) + trules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, + ttc_tunnel_rules[tt].etype, + ttc_tunnel_rules[tt].proto); + if (IS_ERR(trules[tt])) { + err = PTR_ERR(trules[tt]); + trules[tt] = NULL; goto del_rules; + } } return 0; del_rules: - err = PTR_ERR(rules[tt]); - rules[tt] = NULL; mlx5e_cleanup_ttc_rules(ttc); return err; } @@ -1015,33 +1024,38 @@ static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv, struct mlx5e_ttc_table *ttc) { struct mlx5_flow_destination dest = {}; - struct mlx5_flow_handle **rules; + struct mlx5e_ttc_rule *rules; struct mlx5_flow_table *ft; int err; int tt; ft = ttc->ft.t; rules = ttc->rules; - dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; + for (tt = 0; tt < MLX5E_NUM_TT; tt++) { + struct mlx5e_ttc_rule *rule = &rules[tt]; + if (tt == MLX5E_TT_ANY) dest.tir_num = params->any_tt_tirn; else dest.tir_num = params->indir_tirn[tt]; - rules[tt] = mlx5e_generate_inner_ttc_rule(priv, ft, &dest, - ttc_rules[tt].etype, - ttc_rules[tt].proto); - if (IS_ERR(rules[tt])) + rule->rule = mlx5e_generate_inner_ttc_rule(priv, ft, &dest, + ttc_rules[tt].etype, + ttc_rules[tt].proto); + if (IS_ERR(rule->rule)) { + err = PTR_ERR(rule->rule); + rule->rule = NULL; goto del_rules; + } + rule->default_dest = dest; } return 0; del_rules: - err = PTR_ERR(rules[tt]); - rules[tt] = NULL; + mlx5e_cleanup_ttc_rules(ttc); return err; } @@ -1210,6 +1224,30 @@ err: return err; } +int mlx5e_ttc_fwd_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type, + struct mlx5_flow_destination *new_dest) +{ + return mlx5_modify_rule_destination(priv->fs.ttc.rules[type].rule, new_dest, NULL); +} + +struct mlx5_flow_destination +mlx5e_ttc_get_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type) +{ + struct mlx5_flow_destination *dest = &priv->fs.ttc.rules[type].default_dest; + + WARN_ONCE(dest->type != MLX5_FLOW_DESTINATION_TYPE_TIR, + "TTC[%d] default dest is not setup yet", type); + + return *dest; +} + +int mlx5e_ttc_fwd_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type) +{ + struct mlx5_flow_destination dest = mlx5e_ttc_get_default_dest(priv, type); + + return mlx5e_ttc_fwd_dest(priv, type, &dest); +} + static void mlx5e_del_l2_flow_rule(struct mlx5e_priv *priv, struct mlx5e_l2_rule *ai) { From c062d52ac24ceea0b4a5fbdc70c5cff1e47a0e49 Mon Sep 17 00:00:00 2001 From: Boris Pismenny Date: Sun, 14 Apr 2019 16:35:24 +0300 Subject: [PATCH 09/20] net/mlx5e: Receive flow steering framework for accelerated TCP flows The framework allows creating flow tables to steer incoming traffic of TCP sockets to the acceleration TIRs. This is used in downstream patches for TLS, and will be used in the future for other offloads. Signed-off-by: Boris Pismenny Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- .../net/ethernet/mellanox/mlx5/core/en/fs.h | 10 + .../mellanox/mlx5/core/en_accel/fs_tcp.c | 280 ++++++++++++++++++ .../mellanox/mlx5/core/en_accel/fs_tcp.h | 18 ++ .../net/ethernet/mellanox/mlx5/core/fs_core.c | 4 +- 5 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index b61e47bc16e8..8ffa1325a18f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -74,7 +74,7 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \ en_accel/ipsec_stats.o mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o \ - en_accel/ktls.o en_accel/ktls_tx.o + en_accel/ktls.o en_accel/ktls_tx.o en_accel/fs_tcp.o mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \ steering/dr_matcher.o steering/dr_rule.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index c633579474c3..385cbff1caf1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -123,6 +123,9 @@ enum { MLX5E_L2_FT_LEVEL, MLX5E_TTC_FT_LEVEL, MLX5E_INNER_TTC_FT_LEVEL, +#ifdef CONFIG_MLX5_EN_TLS + MLX5E_ACCEL_FS_TCP_FT_LEVEL, +#endif #ifdef CONFIG_MLX5_EN_ARFS MLX5E_ARFS_FT_LEVEL #endif @@ -216,6 +219,10 @@ static inline int mlx5e_arfs_enable(struct mlx5e_priv *priv) { return -EOPNOTSUP static inline int mlx5e_arfs_disable(struct mlx5e_priv *priv) { return -EOPNOTSUPP; } #endif +#ifdef CONFIG_MLX5_EN_TLS +struct mlx5e_accel_fs_tcp; +#endif + struct mlx5e_flow_steering { struct mlx5_flow_namespace *ns; #ifdef CONFIG_MLX5_EN_RXNFC @@ -229,6 +236,9 @@ struct mlx5e_flow_steering { #ifdef CONFIG_MLX5_EN_ARFS struct mlx5e_arfs_tables arfs; #endif +#ifdef CONFIG_MLX5_EN_TLS + struct mlx5e_accel_fs_tcp *accel_tcp; +#endif }; struct ttc_params { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c new file mode 100644 index 000000000000..a0e9082e15b0 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include +#include "en_accel/fs_tcp.h" +#include "fs_core.h" + +enum accel_fs_tcp_type { + ACCEL_FS_IPV4_TCP, + ACCEL_FS_IPV6_TCP, + ACCEL_FS_TCP_NUM_TYPES, +}; + +struct mlx5e_accel_fs_tcp { + struct mlx5e_flow_table tables[ACCEL_FS_TCP_NUM_TYPES]; + struct mlx5_flow_handle *default_rules[ACCEL_FS_TCP_NUM_TYPES]; +}; + +static enum mlx5e_traffic_types fs_accel2tt(enum accel_fs_tcp_type i) +{ + switch (i) { + case ACCEL_FS_IPV4_TCP: + return MLX5E_TT_IPV4_TCP; + default: /* ACCEL_FS_IPV6_TCP */ + return MLX5E_TT_IPV6_TCP; + } +} + +static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, + enum accel_fs_tcp_type type) +{ + struct mlx5e_flow_table *accel_fs_t; + struct mlx5_flow_destination dest; + struct mlx5e_accel_fs_tcp *fs_tcp; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_handle *rule; + int err = 0; + + fs_tcp = priv->fs.accel_tcp; + accel_fs_t = &fs_tcp->tables[type]; + + dest = mlx5e_ttc_get_default_dest(priv, fs_accel2tt(type)); + rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(priv->netdev, + "%s: add default rule failed, accel_fs type=%d, err %d\n", + __func__, type, err); + return err; + } + + fs_tcp->default_rules[type] = rule; + return 0; +} + +#define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2) +#define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1) +#define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0)) +#define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\ + MLX5E_ACCEL_FS_TCP_GROUP2_SIZE) +static int accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, + enum accel_fs_tcp_type type) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + void *outer_headers_c; + int ix = 0; + u32 *in; + int err; + u8 *mc; + + ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in || !ft->g) { + kvfree(ft->g); + kvfree(in); + return -ENOMEM; + } + + mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); + + switch (type) { + case ACCEL_FS_IPV4_TCP: + case ACCEL_FS_IPV6_TCP: + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); + break; + default: + err = -EINVAL; + goto out; + } + + switch (type) { + case ACCEL_FS_IPV4_TCP: + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, + src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); + break; + case ACCEL_FS_IPV6_TCP: + memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, + src_ipv4_src_ipv6.ipv6_layout.ipv6), + 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + 0xff, 16); + break; + default: + err = -EINVAL; + goto out; + } + + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + /* Default Flow Group */ + memset(in, 0, inlen); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + kvfree(in); + return 0; + +err: + err = PTR_ERR(ft->g[ft->num_groups]); + ft->g[ft->num_groups] = NULL; +out: + kvfree(in); + + return err; +} + +static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type) +{ + struct mlx5e_flow_table *ft = &priv->fs.accel_tcp->tables[type]; + struct mlx5_flow_table_attr ft_attr = {}; + int err; + + ft->num_groups = 0; + + ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE; + ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + + ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + if (IS_ERR(ft->t)) { + err = PTR_ERR(ft->t); + ft->t = NULL; + return err; + } + + netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n", + ft->t->id, ft->t->level); + + err = accel_fs_tcp_create_groups(ft, type); + if (err) + goto err; + + err = accel_fs_tcp_add_default_rule(priv, type); + if (err) + goto err; + + return 0; +err: + mlx5e_destroy_flow_table(ft); + return err; +} + +static int accel_fs_tcp_disable(struct mlx5e_priv *priv) +{ + int err, i; + + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { + /* Modify ttc rules destination to point back to the indir TIRs */ + err = mlx5e_ttc_fwd_default_dest(priv, fs_accel2tt(i)); + if (err) { + netdev_err(priv->netdev, + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, fs_accel2tt(i), err); + return err; + } + } + + return 0; +} + +static int accel_fs_tcp_enable(struct mlx5e_priv *priv) +{ + struct mlx5_flow_destination dest = {}; + int err, i; + + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { + dest.ft = priv->fs.accel_tcp->tables[i].t; + + /* Modify ttc rules destination to point on the accel_fs FTs */ + err = mlx5e_ttc_fwd_dest(priv, fs_accel2tt(i), &dest); + if (err) { + netdev_err(priv->netdev, + "%s: modify ttc[%d] destination to accel failed, err(%d)\n", + __func__, fs_accel2tt(i), err); + return err; + } + } + return 0; +} + +static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) +{ + struct mlx5e_accel_fs_tcp *fs_tcp; + + fs_tcp = priv->fs.accel_tcp; + if (IS_ERR_OR_NULL(fs_tcp->tables[i].t)) + return; + + mlx5_del_flow_rules(fs_tcp->default_rules[i]); + mlx5e_destroy_flow_table(&fs_tcp->tables[i]); + fs_tcp->tables[i].t = NULL; +} + +void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) +{ + int i; + + if (!priv->fs.accel_tcp) + return; + + accel_fs_tcp_disable(priv); + + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) + accel_fs_tcp_destroy_table(priv, i); + + kfree(priv->fs.accel_tcp); + priv->fs.accel_tcp = NULL; +} + +int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) +{ + int i, err; + + if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) + return -EOPNOTSUPP; + + priv->fs.accel_tcp = kzalloc(sizeof(*priv->fs.accel_tcp), GFP_KERNEL); + if (!priv->fs.accel_tcp) + return -ENOMEM; + + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { + err = accel_fs_tcp_create_table(priv, i); + if (err) + goto err_destroy_tables; + } + + err = accel_fs_tcp_enable(priv); + if (err) + goto err_destroy_tables; + + return 0; + +err_destroy_tables: + while (--i >= 0) + accel_fs_tcp_destroy_table(priv, i); + + kfree(priv->fs.accel_tcp); + priv->fs.accel_tcp = NULL; + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h new file mode 100644 index 000000000000..0df53473550a --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_ACCEL_FS_TCP_H__ +#define __MLX5E_ACCEL_FS_TCP_H__ + +#include "en.h" + +#ifdef CONFIG_MLX5_EN_TLS +int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv); +void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv); +#else +static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) { return 0; } +static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) {} +#endif + +#endif /* __MLX5E_ACCEL_FS_TCP_H__ */ + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index e47a66983935..785b2960d6b5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -105,8 +105,8 @@ #define ETHTOOL_PRIO_NUM_LEVELS 1 #define ETHTOOL_NUM_PRIOS 11 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS) -/* Vlan, mac, ttc, inner ttc, aRFS */ -#define KERNEL_NIC_PRIO_NUM_LEVELS 5 +/* Vlan, mac, ttc, inner ttc, {aRFS/accel} */ +#define KERNEL_NIC_PRIO_NUM_LEVELS 6 #define KERNEL_NIC_NUM_PRIOS 1 /* One more level for tc */ #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1) From 5229a96e59ec32466add5e87b537cc3f244afb06 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Tue, 16 Jun 2020 13:29:07 +0300 Subject: [PATCH 10/20] net/mlx5e: Accel, Expose flow steering API for rules add/del Given a socket, the function extracts the TCP/IP{4,6} ntuple and adds rule to steering. Another function gets the rule and deletes it. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed Reviewed-by: Maxim Mikityanskiy --- .../mellanox/mlx5/core/en_accel/fs_tcp.c | 120 ++++++++++++++++++ .../mellanox/mlx5/core/en_accel/fs_tcp.h | 9 ++ 2 files changed, 129 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c index a0e9082e15b0..4cdd9eac647d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c @@ -26,6 +26,126 @@ static enum mlx5e_traffic_types fs_accel2tt(enum accel_fs_tcp_type i) } } +static void accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct sock *sk) +{ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), + &inet_sk(sk)->inet_daddr, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), + &inet_sk(sk)->inet_rcv_saddr, 4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); +} + +static void accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct sock *sk) +{ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), + &sk->sk_v6_daddr, 16); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + &inet6_sk(sk)->saddr, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), + 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + 0xff, 16); +} + +void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule) +{ + mlx5_del_flow_rules(rule); +} + +struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, + struct sock *sk, u32 tirn, + uint32_t flow_tag) +{ + struct mlx5_flow_destination dest = {}; + struct mlx5e_flow_table *ft = NULL; + struct mlx5e_accel_fs_tcp *fs_tcp; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_handle *flow; + struct mlx5_flow_spec *spec; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + fs_tcp = priv->fs.accel_tcp; + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + + switch (sk->sk_family) { + case AF_INET: + accel_fs_tcp_set_ipv4_flow(spec, sk); + ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; + mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__, + &inet_sk(sk)->inet_rcv_saddr, + inet_sk(sk)->inet_sport, + &inet_sk(sk)->inet_daddr, + inet_sk(sk)->inet_dport); + break; +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: + if (!sk->sk_ipv6only && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { + accel_fs_tcp_set_ipv4_flow(spec, sk); + ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; + } else { + accel_fs_tcp_set_ipv6_flow(spec, sk); + ft = &fs_tcp->tables[ACCEL_FS_IPV6_TCP]; + } + break; +#endif + default: + break; + } + + if (!ft) { + flow = ERR_PTR(-EINVAL); + goto out; + } + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.tcp_dport); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.tcp_sport); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, + ntohs(inet_sk(sk)->inet_sport)); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, + ntohs(inet_sk(sk)->inet_dport)); + + dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; + dest.tir_num = tirn; + if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG) { + spec->flow_context.flow_tag = flow_tag; + spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG; + } + + flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); + + if (IS_ERR(flow)) + netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n", + PTR_ERR(flow)); + +out: + kvfree(spec); + return flow; +} + static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, enum accel_fs_tcp_type type) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h index 0df53473550a..589235824543 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h @@ -9,9 +9,18 @@ #ifdef CONFIG_MLX5_EN_TLS int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv); void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv); +struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, + struct sock *sk, u32 tirn, + uint32_t flow_tag); +void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule); #else static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) { return 0; } static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) {} +static inline struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, + struct sock *sk, u32 tirn, + uint32_t flow_tag) +{ return ERR_PTR(-EOPNOTSUPP); } +static inline void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule) {} #endif #endif /* __MLX5E_ACCEL_FS_TCP_H__ */ From 7d0d0d86ec6c67d3eb4f49d3bbccc2d8c02799cc Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Tue, 26 May 2020 13:58:09 +0300 Subject: [PATCH 11/20] net/mlx5e: kTLS, Improve TLS feature modularity Better separate the code into c/h files, so that kTLS internals are exposed to the corresponding non-accel flow as follows: - Necessary datapath functions are exposed via ktls_txrx.h. - Necessary caps and configuration functions are exposed via ktls.h, which became very small. In addition, kTLS internal code sharing is done via ktls_utils.h, which is not exposed to any non-accel file. Add explicit WQE structures for the TLS static and progress params, breaking the union of the static with UMR, and the progress with PSV. Generalize the API as a preparation for TLS RX offload support. Move kTLS TX-specific code to the proper file. Remove the inline tag for function in C files, let the compiler decide. Use kzalloc/kfree for the priv_tx context. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed Reviewed-by: Maxim Mikityanskiy --- .../net/ethernet/mellanox/mlx5/core/Makefile | 3 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 14 +- .../mellanox/mlx5/core/en_accel/ktls.c | 62 +---- .../mellanox/mlx5/core/en_accel/ktls.h | 107 --------- .../mellanox/mlx5/core/en_accel/ktls_tx.c | 218 +++++++++++------- .../mellanox/mlx5/core/en_accel/ktls_txrx.c | 110 +++++++++ .../mellanox/mlx5/core/en_accel/ktls_txrx.h | 36 +++ .../mellanox/mlx5/core/en_accel/ktls_utils.h | 69 ++++++ .../mellanox/mlx5/core/en_accel/tls.c | 14 -- .../mellanox/mlx5/core/en_accel/tls.h | 7 - .../mellanox/mlx5/core/en_accel/tls_rxtx.c | 14 ++ .../mellanox/mlx5/core/en_accel/tls_rxtx.h | 13 +- .../net/ethernet/mellanox/mlx5/core/en_tx.c | 1 - 13 files changed, 378 insertions(+), 290 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 8ffa1325a18f..70ad24fff2e2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -74,7 +74,8 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \ en_accel/ipsec_stats.o mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o \ - en_accel/ktls.o en_accel/ktls_tx.o en_accel/fs_tcp.o + en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \ + en_accel/ktls_tx.o mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \ steering/dr_matcher.o steering/dr_rule.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 29265ca6050c..ec5bf73f9b07 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -191,13 +191,8 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) struct mlx5e_tx_wqe { struct mlx5_wqe_ctrl_seg ctrl; - union { - struct { - struct mlx5_wqe_eth_seg eth; - struct mlx5_wqe_data_seg data[0]; - }; - u8 tls_progress_params_ctx[0]; - }; + struct mlx5_wqe_eth_seg eth; + struct mlx5_wqe_data_seg data[0]; }; struct mlx5e_rx_wqe_ll { @@ -213,10 +208,7 @@ struct mlx5e_umr_wqe { struct mlx5_wqe_ctrl_seg ctrl; struct mlx5_wqe_umr_ctrl_seg uctrl; struct mlx5_mkey_seg mkc; - union { - struct mlx5_mtt inline_mtts[0]; - u8 tls_static_params_ctx[0]; - }; + struct mlx5_mtt inline_mtts[0]; }; extern const char mlx5e_self_tests[][ETH_GSTRING_LEN]; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 452fcf59c36b..8970ea68d005 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -3,31 +3,7 @@ #include "en.h" #include "en_accel/ktls.h" - -u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq) -{ - u16 num_dumps, stop_room = 0; - - num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); - - stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_STATIC_WQEBBS); - stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_PROGRESS_WQEBBS); - stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS); - - return stop_room; -} - -static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) -{ - u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; - void *tisc; - - tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - - MLX5_SET(tisc, tisc, tls_en, 1); - - return mlx5e_create_tis(mdev, in, tisn); -} +#include "en_accel/ktls_utils.h" static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, enum tls_offload_ctx_dir direction, @@ -35,8 +11,6 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, u32 start_offload_tcp_sn) { struct mlx5e_priv *priv = netdev_priv(netdev); - struct mlx5e_ktls_offload_context_tx *tx_priv; - struct tls_context *tls_ctx = tls_get_ctx(sk); struct mlx5_core_dev *mdev = priv->mdev; int err; @@ -46,31 +20,8 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, if (WARN_ON(!mlx5e_ktls_type_check(mdev, crypto_info))) return -EOPNOTSUPP; - tx_priv = kvzalloc(sizeof(*tx_priv), GFP_KERNEL); - if (!tx_priv) - return -ENOMEM; + err = mlx5e_ktls_add_tx(netdev, sk, crypto_info, start_offload_tcp_sn); - tx_priv->expected_seq = start_offload_tcp_sn; - tx_priv->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; - mlx5e_set_ktls_tx_priv_ctx(tls_ctx, tx_priv); - - /* tc and underlay_qpn values are not in use for tls tis */ - err = mlx5e_ktls_create_tis(mdev, &tx_priv->tisn); - if (err) - goto create_tis_fail; - - err = mlx5_ktls_create_key(mdev, crypto_info, &tx_priv->key_id); - if (err) - goto encryption_key_create_fail; - - mlx5e_ktls_tx_offload_set_pending(tx_priv); - - return 0; - -encryption_key_create_fail: - mlx5e_destroy_tis(priv->mdev, tx_priv->tisn); -create_tis_fail: - kvfree(tx_priv); return err; } @@ -78,13 +29,10 @@ static void mlx5e_ktls_del(struct net_device *netdev, struct tls_context *tls_ctx, enum tls_offload_ctx_dir direction) { - struct mlx5e_priv *priv = netdev_priv(netdev); - struct mlx5e_ktls_offload_context_tx *tx_priv = - mlx5e_get_ktls_tx_priv_ctx(tls_ctx); + if (direction != TLS_OFFLOAD_CTX_DIR_TX) + return; - mlx5e_destroy_tis(priv->mdev, tx_priv->tisn); - mlx5_ktls_destroy_key(priv->mdev, tx_priv->key_id); - kvfree(tx_priv); + mlx5e_ktls_del_tx(netdev, tls_ctx); } static const struct tlsdev_ops mlx5e_ktls_ops = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index 806ed185dd4c..69d736954977 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -7,122 +7,15 @@ #include "en.h" #ifdef CONFIG_MLX5_EN_TLS -#include -#include "accel/tls.h" -#include "en_accel/tls_rxtx.h" - -#define MLX5E_KTLS_STATIC_UMR_WQE_SZ \ - (offsetof(struct mlx5e_umr_wqe, tls_static_params_ctx) + \ - MLX5_ST_SZ_BYTES(tls_static_params)) -#define MLX5E_KTLS_STATIC_WQEBBS \ - (DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_BB)) - -#define MLX5E_KTLS_PROGRESS_WQE_SZ \ - (offsetof(struct mlx5e_tx_wqe, tls_progress_params_ctx) + \ - sizeof(struct mlx5_wqe_tls_progress_params_seg)) -#define MLX5E_KTLS_PROGRESS_WQEBBS \ - (DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_BB)) - -struct mlx5e_dump_wqe { - struct mlx5_wqe_ctrl_seg ctrl; - struct mlx5_wqe_data_seg data; -}; - -#define MLX5E_TLS_FETCH_UMR_WQE(sq, pi) \ - ((struct mlx5e_umr_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_STATIC_UMR_WQE_SZ)) -#define MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi) \ - ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_PROGRESS_WQE_SZ)) -#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \ - ((struct mlx5e_dump_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, \ - sizeof(struct mlx5e_dump_wqe))) - -#define MLX5E_KTLS_DUMP_WQEBBS \ - (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) - -enum { - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0, - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1, - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2, -}; - -enum { - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0, - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1, - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2, -}; - -struct mlx5e_ktls_offload_context_tx { - struct tls_offload_context_tx *tx_ctx; - struct tls12_crypto_info_aes_gcm_128 crypto_info; - u32 expected_seq; - u32 tisn; - u32 key_id; - bool ctx_post_pending; -}; - -struct mlx5e_ktls_offload_context_tx_shadow { - struct tls_offload_context_tx tx_ctx; - struct mlx5e_ktls_offload_context_tx *priv_tx; -}; - -static inline void -mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, - struct mlx5e_ktls_offload_context_tx *priv_tx) -{ - struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); - struct mlx5e_ktls_offload_context_tx_shadow *shadow; - - BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - - shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - - shadow->priv_tx = priv_tx; - priv_tx->tx_ctx = tx_ctx; -} - -static inline struct mlx5e_ktls_offload_context_tx * -mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) -{ - struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); - struct mlx5e_ktls_offload_context_tx_shadow *shadow; - - BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - - shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - - return shadow->priv_tx; -} void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); -void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx); -bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, - struct sk_buff *skb, int datalen, - struct mlx5e_accel_tx_tls_state *state); -void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, - struct mlx5e_tx_wqe_info *wi, - u32 *dma_fifo_cc); -u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq); - -static inline u8 -mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags, - unsigned int sync_len) -{ - /* Given the MTU and sync_len, calculates an upper bound for the - * number of DUMP WQEs needed for the TX resync of a record. - */ - return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu); -} #else static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { } -static inline void -mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, - struct mlx5e_tx_wqe_info *wi, - u32 *dma_fifo_cc) {} #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index ad7300f19815..349e29214b92 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -1,109 +1,149 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2019 Mellanox Technologies. -#include -#include "en.h" -#include "en/txrx.h" -#include "en_accel/ktls.h" +#include "en_accel/ktls_txrx.h" +#include "en_accel/ktls_utils.h" -enum { - MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2, +struct mlx5e_dump_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_data_seg data; }; -enum { - MLX5E_ENCRYPTION_STANDARD_TLS = 0x1, +#define MLX5E_KTLS_DUMP_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) + +static u8 +mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags, + unsigned int sync_len) +{ + /* Given the MTU and sync_len, calculates an upper bound for the + * number of DUMP WQEs needed for the TX resync of a record. + */ + return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu); +} + +u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq) +{ + u16 num_dumps, stop_room = 0; + + num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); + + stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); + stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS); + stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS); + + return stop_room; +} + +static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) +{ + u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; + void *tisc; + + tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + + MLX5_SET(tisc, tisc, tls_en, 1); + + return mlx5e_create_tis(mdev, in, tisn); +} + +struct mlx5e_ktls_offload_context_tx { + struct tls_offload_context_tx *tx_ctx; + struct tls12_crypto_info_aes_gcm_128 crypto_info; + u32 expected_seq; + u32 tisn; + u32 key_id; + bool ctx_post_pending; }; -#define EXTRACT_INFO_FIELDS do { \ - salt = info->salt; \ - rec_seq = info->rec_seq; \ - salt_sz = sizeof(info->salt); \ - rec_seq_sz = sizeof(info->rec_seq); \ -} while (0) +struct mlx5e_ktls_offload_context_tx_shadow { + struct tls_offload_context_tx tx_ctx; + struct mlx5e_ktls_offload_context_tx *priv_tx; +}; static void -fill_static_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx) +mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, + struct mlx5e_ktls_offload_context_tx *priv_tx) { - struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info; - char *initial_rn, *gcm_iv; - u16 salt_sz, rec_seq_sz; - char *salt, *rec_seq; - u8 tls_version; + struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); + struct mlx5e_ktls_offload_context_tx_shadow *shadow; - EXTRACT_INFO_FIELDS; + BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv); - initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number); + shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - memcpy(gcm_iv, salt, salt_sz); - memcpy(initial_rn, rec_seq, rec_seq_sz); - - tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2; - - MLX5_SET(tls_static_params, ctx, tls_version, tls_version); - MLX5_SET(tls_static_params, ctx, const_1, 1); - MLX5_SET(tls_static_params, ctx, const_2, 2); - MLX5_SET(tls_static_params, ctx, encryption_standard, - MLX5E_ENCRYPTION_STANDARD_TLS); - MLX5_SET(tls_static_params, ctx, dek_index, priv_tx->key_id); + shadow->priv_tx = priv_tx; + priv_tx->tx_ctx = tx_ctx; } -static void -build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn, - struct mlx5e_ktls_offload_context_tx *priv_tx, - bool fence) +static struct mlx5e_ktls_offload_context_tx * +mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) { - struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; - struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; + struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); + struct mlx5e_ktls_offload_context_tx_shadow *shadow; -#define STATIC_PARAMS_DS_CNT \ - DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_DS) + BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR | - (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24)); - cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | - STATIC_PARAMS_DS_CNT); - cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; - cseg->tis_tir_num = cpu_to_be32(priv_tx->tisn << 8); + shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - ucseg->flags = MLX5_UMR_INLINE; - ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); - - fill_static_params_ctx(wqe->tls_static_params_ctx, priv_tx); + return shadow->priv_tx; } -static void -fill_progress_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx) +int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn) { - struct mlx5_wqe_tls_progress_params_seg *params; + struct mlx5e_ktls_offload_context_tx *priv_tx; + struct tls_context *tls_ctx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + int err; - params = ctx; + tls_ctx = tls_get_ctx(sk); + priv = netdev_priv(netdev); + mdev = priv->mdev; - params->tis_tir_num = cpu_to_be32(priv_tx->tisn); - MLX5_SET(tls_progress_params, params->ctx, record_tracker_state, - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); - MLX5_SET(tls_progress_params, params->ctx, auth_state, - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD); + priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); + if (!priv_tx) + return -ENOMEM; + + err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id); + if (err) + goto err_create_key; + + priv_tx->expected_seq = start_offload_tcp_sn; + priv_tx->crypto_info = + *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + + mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); + + err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); + if (err) + goto err_create_tis; + + priv_tx->ctx_post_pending = true; + + return 0; + +err_create_tis: + mlx5_ktls_destroy_key(mdev, priv_tx->key_id); +err_create_key: + kfree(priv_tx); + return err; } -static void -build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn, - struct mlx5e_ktls_offload_context_tx *priv_tx, - bool fence) +void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) { - struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + struct mlx5e_ktls_offload_context_tx *priv_tx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; -#define PROGRESS_PARAMS_DS_CNT \ - DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_DS) + priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); + priv = netdev_priv(netdev); + mdev = priv->mdev; - cseg->opmod_idx_opcode = - cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV | - (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24)); - cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | - PROGRESS_PARAMS_DS_CNT); - cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; - - fill_progress_params_ctx(wqe->tls_progress_params_ctx, priv_tx); + mlx5e_destroy_tis(mdev, priv_tx->tisn); + mlx5_ktls_destroy_key(mdev, priv_tx->key_id); + kfree(priv_tx); } static void tx_fill_wi(struct mlx5e_txqsq *sq, @@ -119,11 +159,6 @@ static void tx_fill_wi(struct mlx5e_txqsq *sq, }; } -void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) -{ - priv_tx->ctx_post_pending = true; -} - static bool mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) { @@ -139,12 +174,15 @@ post_static_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, bool fence) { - u16 pi, num_wqebbs = MLX5E_KTLS_STATIC_WQEBBS; - struct mlx5e_umr_wqe *umr_wqe; + struct mlx5e_set_tls_static_params_wqe *wqe; + u16 pi, num_wqebbs; + num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); - umr_wqe = MLX5E_TLS_FETCH_UMR_WQE(sq, pi); - build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence); + wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, + priv_tx->tisn, priv_tx->key_id, fence, + TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; } @@ -154,12 +192,14 @@ post_progress_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, bool fence) { - u16 pi, num_wqebbs = MLX5E_KTLS_PROGRESS_WQEBBS; - struct mlx5e_tx_wqe *wqe; + struct mlx5e_set_tls_progress_params_wqe *wqe; + u16 pi, num_wqebbs; + num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); - wqe = MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi); - build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence); + wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, + TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c new file mode 100644 index 000000000000..edf404eaa275 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include "en_accel/ktls_txrx.h" +#include "en_accel/ktls_utils.h" + +enum { + MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2, +}; + +enum { + MLX5E_ENCRYPTION_STANDARD_TLS = 0x1, +}; + +#define EXTRACT_INFO_FIELDS do { \ + salt = info->salt; \ + rec_seq = info->rec_seq; \ + salt_sz = sizeof(info->salt); \ + rec_seq_sz = sizeof(info->rec_seq); \ +} while (0) + +static void +fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, + struct tls12_crypto_info_aes_gcm_128 *info, + u32 key_id) +{ + char *initial_rn, *gcm_iv; + u16 salt_sz, rec_seq_sz; + char *salt, *rec_seq; + u8 tls_version; + u8 *ctx; + + ctx = params->ctx; + + EXTRACT_INFO_FIELDS; + + gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv); + initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number); + + memcpy(gcm_iv, salt, salt_sz); + memcpy(initial_rn, rec_seq, rec_seq_sz); + + tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2; + + MLX5_SET(tls_static_params, ctx, tls_version, tls_version); + MLX5_SET(tls_static_params, ctx, const_1, 1); + MLX5_SET(tls_static_params, ctx, const_2, 2); + MLX5_SET(tls_static_params, ctx, encryption_standard, + MLX5E_ENCRYPTION_STANDARD_TLS); + MLX5_SET(tls_static_params, ctx, dek_index, key_id); +} + +void +mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, + u16 pc, u32 sqn, + struct tls12_crypto_info_aes_gcm_128 *info, + u32 tis_tir_num, u32 key_id, + bool fence, enum tls_offload_ctx_dir direction) +{ + struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; + struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + +#define STATIC_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS) + + cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR | + (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24)); + cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | + STATIC_PARAMS_DS_CNT); + cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; + cseg->tis_tir_num = cpu_to_be32(tis_tir_num << 8); + + ucseg->flags = MLX5_UMR_INLINE; + ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); + + fill_static_params(&wqe->params, info, key_id); +} + +static void +fill_progress_params(struct mlx5_wqe_tls_progress_params_seg *params, u32 tis_tir_num) +{ + u8 *ctx = params->ctx; + + params->tis_tir_num = cpu_to_be32(tis_tir_num); + + MLX5_SET(tls_progress_params, ctx, record_tracker_state, + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); + MLX5_SET(tls_progress_params, ctx, auth_state, + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD); +} + +void +mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, + u16 pc, u32 sqn, + u32 tis_tir_num, bool fence, + enum tls_offload_ctx_dir direction) +{ + struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + +#define PROGRESS_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS) + + cseg->opmod_idx_opcode = + cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV | + (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24)); + cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | + PROGRESS_PARAMS_DS_CNT); + cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; + + fill_progress_params(&wqe->params, tis_tir_num); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h new file mode 100644 index 000000000000..7ce6b1c41d9b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_KTLS_TXRX_H__ +#define __MLX5E_KTLS_TXRX_H__ + +#ifdef CONFIG_MLX5_EN_TLS + +#include +#include "en.h" +#include "en/txrx.h" + +struct mlx5e_accel_tx_tls_state { + u32 tls_tisn; +}; + +u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq); + +bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, + struct sk_buff *skb, int datalen, + struct mlx5e_accel_tx_tls_state *state); + +void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, + struct mlx5e_tx_wqe_info *wi, + u32 *dma_fifo_cc); +#else +static inline void +mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, + struct mlx5e_tx_wqe_info *wi, + u32 *dma_fifo_cc) +{ +} + +#endif /* CONFIG_MLX5_EN_TLS */ + +#endif /* __MLX5E_TLS_TXRX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h new file mode 100644 index 000000000000..d1d747cb2dcb --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_KTLS_UTILS_H__ +#define __MLX5E_KTLS_UTILS_H__ + +#include +#include "en.h" +#include "accel/tls.h" + +enum { + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0, + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1, + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2, +}; + +enum { + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0, + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1, + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2, +}; + +int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn); +void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx); + +struct mlx5e_set_tls_static_params_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_umr_ctrl_seg uctrl; + struct mlx5_mkey_seg mkc; + struct mlx5_wqe_tls_static_params_seg params; +}; + +struct mlx5e_set_tls_progress_params_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_tls_progress_params_seg params; +}; + +#define MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_static_params_wqe), MLX5_SEND_WQE_BB)) + +#define MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_progress_params_wqe), MLX5_SEND_WQE_BB)) + +#define MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi) \ + ((struct mlx5e_set_tls_static_params_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_static_params_wqe))) + +#define MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi) \ + ((struct mlx5e_set_tls_progress_params_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_progress_params_wqe))) + +#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \ + ((struct mlx5e_dump_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_dump_wqe))) + +void +mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, + u16 pc, u32 sqn, + struct tls12_crypto_info_aes_gcm_128 *info, + u32 tis_tir_num, u32 key_id, + bool fence, enum tls_offload_ctx_dir direction); +void +mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, + u16 pc, u32 sqn, + u32 tis_tir_num, bool fence, + enum tls_offload_ctx_dir direction); + +#endif /* __MLX5E_TLS_UTILS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c index 1fbb5a90cb38..c01c17a5c6de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c @@ -240,17 +240,3 @@ void mlx5e_tls_cleanup(struct mlx5e_priv *priv) kfree(tls); priv->tls = NULL; } - -u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) -{ - struct mlx5_core_dev *mdev = sq->channel->mdev; - - if (!mlx5_accel_is_tls_device(mdev)) - return 0; - - if (MLX5_CAP_GEN(mdev, tls_tx)) - return mlx5e_ktls_get_stop_room(sq); - - /* Resync SKB. */ - return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); -} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h index 9219bdb2786e..9015f3f7792d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h @@ -94,8 +94,6 @@ int mlx5e_tls_get_count(struct mlx5e_priv *priv); int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data); int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data); -u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq); - #else static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) @@ -110,11 +108,6 @@ static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; } static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; } static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; } -static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) -{ - return 0; -} - #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index 72d26fbc8d5b..4d796fea906d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -387,3 +387,17 @@ void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, remove_metadata_hdr(skb); *cqe_bcnt -= MLX5E_METADATA_ETHER_LEN; } + +u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) +{ + struct mlx5_core_dev *mdev = sq->channel->mdev; + + if (!mlx5_accel_is_tls_device(mdev)) + return 0; + + if (MLX5_CAP_GEN(mdev, tls_tx)) + return mlx5e_ktls_get_stop_room(sq); + + /* Resync SKB. */ + return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h index a50d0394df0a..2a7b98531539 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h @@ -34,15 +34,15 @@ #ifndef __MLX5E_TLS_RXTX_H__ #define __MLX5E_TLS_RXTX_H__ +#include "en_accel/ktls_txrx.h" + #ifdef CONFIG_MLX5_EN_TLS #include #include "en.h" #include "en/txrx.h" -struct mlx5e_accel_tx_tls_state { - u32 tls_tisn; -}; +u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq); bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_tls_state *state); @@ -52,6 +52,13 @@ void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *c void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, u32 *cqe_bcnt); +#else + +static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) +{ + return 0; +} + #endif /* CONFIG_MLX5_EN_TLS */ #endif /* __MLX5E_TLS_RXTX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 6d406063aca4..da596de3abba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -38,7 +38,6 @@ #include "en/txrx.h" #include "ipoib/ipoib.h" #include "en_accel/en_accel.h" -#include "en_accel/ktls.h" #include "lib/clock.h" static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma) From df8d866770f9877dedc864af4346c73694931cab Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Thu, 28 May 2020 10:04:03 +0300 Subject: [PATCH 12/20] net/mlx5e: kTLS, Use kernel API to extract private offload context Modify the implementation of the private kTLS TX HW offload context getter and setter, so it uses the kernel API functions, instead of a local shadow structure. A single BUILD_BUG_ON check is sufficient, remove the duplicate. Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/en_accel/ktls_tx.c | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 349e29214b92..5a980f93c326 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -56,37 +56,26 @@ struct mlx5e_ktls_offload_context_tx { bool ctx_post_pending; }; -struct mlx5e_ktls_offload_context_tx_shadow { - struct tls_offload_context_tx tx_ctx; - struct mlx5e_ktls_offload_context_tx *priv_tx; -}; - static void mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, struct mlx5e_ktls_offload_context_tx *priv_tx) { - struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); - struct mlx5e_ktls_offload_context_tx_shadow *shadow; + struct mlx5e_ktls_offload_context_tx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); - BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); + BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_tx *) > + TLS_OFFLOAD_CONTEXT_SIZE_TX); - shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - - shadow->priv_tx = priv_tx; - priv_tx->tx_ctx = tx_ctx; + *ctx = priv_tx; } static struct mlx5e_ktls_offload_context_tx * mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) { - struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); - struct mlx5e_ktls_offload_context_tx_shadow *shadow; + struct mlx5e_ktls_offload_context_tx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); - BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - - shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - - return shadow->priv_tx; + return *ctx; } int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, @@ -113,6 +102,7 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, priv_tx->expected_seq = start_offload_tcp_sn; priv_tx->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx); mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); From 1182f36593570e8e9ca53f6fabfc40ccf93c21d7 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Thu, 28 May 2020 10:13:00 +0300 Subject: [PATCH 13/20] net/mlx5e: kTLS, Add kTLS RX HW offload support Implement driver support for the kTLS RX HW offload feature. Resync support is added in a downstream patch. New offload contexts post their static/progress params WQEs over the per-channel async ICOSQ, protected under a spin-lock. The Channel/RQ is selected according to the socket's rxq index. Feature is OFF by default. Can be turned on by: $ ethtool -K tls-hw-rx-offload on A new TLS-RX workqueue is used to allow asynchronous addition of steering rules, out of the NAPI context. It will be also used in a downstream patch in the resync procedure. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/Kconfig | 1 + .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- .../ethernet/mellanox/mlx5/core/accel/tls.h | 19 +- .../net/ethernet/mellanox/mlx5/core/en/txrx.h | 11 + .../mellanox/mlx5/core/en_accel/en_accel.h | 20 ++ .../mellanox/mlx5/core/en_accel/ktls.c | 66 +++- .../mellanox/mlx5/core/en_accel/ktls.h | 19 +- .../mellanox/mlx5/core/en_accel/ktls_rx.c | 311 ++++++++++++++++++ .../mellanox/mlx5/core/en_accel/ktls_tx.c | 2 +- .../mellanox/mlx5/core/en_accel/ktls_txrx.c | 20 +- .../mellanox/mlx5/core/en_accel/ktls_txrx.h | 4 + .../mellanox/mlx5/core/en_accel/ktls_utils.h | 4 + .../mellanox/mlx5/core/en_accel/tls.c | 12 +- .../mellanox/mlx5/core/en_accel/tls.h | 1 + .../mellanox/mlx5/core/en_accel/tls_rxtx.c | 20 +- .../mellanox/mlx5/core/en_accel/tls_rxtx.h | 4 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 9 + .../net/ethernet/mellanox/mlx5/core/en_rx.c | 33 +- drivers/net/ethernet/mellanox/mlx5/core/fw.c | 3 +- 19 files changed, 529 insertions(+), 32 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 4dfdbb82ea9d..76b39659c39b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -173,6 +173,7 @@ config MLX5_TLS config MLX5_EN_TLS bool "TLS cryptography-offload accelaration" depends on MLX5_CORE_EN + depends on XPS depends on MLX5_FPGA_TLS || MLX5_TLS default y help diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 70ad24fff2e2..1e7c7f10db6e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -75,7 +75,7 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \ mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o \ en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \ - en_accel/ktls_tx.o + en_accel/ktls_tx.o en_accel/ktls_rx.o mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \ steering/dr_matcher.o steering/dr_rule.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h index aefea467f7b3..fd874f0c380a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h @@ -43,9 +43,20 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, u32 *p_key_id); void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id); +static inline bool mlx5_accel_is_ktls_tx(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, tls_tx); +} + +static inline bool mlx5_accel_is_ktls_rx(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, tls_rx); +} + static inline bool mlx5_accel_is_ktls_device(struct mlx5_core_dev *mdev) { - if (!MLX5_CAP_GEN(mdev, tls_tx)) + if (!mlx5_accel_is_ktls_tx(mdev) && + !mlx5_accel_is_ktls_rx(mdev)) return false; if (!MLX5_CAP_GEN(mdev, log_max_dek)) @@ -67,6 +78,12 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, return false; } #else +static inline bool mlx5_accel_is_ktls_tx(struct mlx5_core_dev *mdev) +{ return false; } + +static inline bool mlx5_accel_is_ktls_rx(struct mlx5_core_dev *mdev) +{ return false; } + static inline int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, struct tls_crypto_info *crypto_info, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index 31cac239563d..7f55bd3229f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -11,6 +11,10 @@ enum mlx5e_icosq_wqe_type { MLX5E_ICOSQ_WQE_NOP, MLX5E_ICOSQ_WQE_UMR_RX, +#ifdef CONFIG_MLX5_EN_TLS + MLX5E_ICOSQ_WQE_UMR_TLS, + MLX5E_ICOSQ_WQE_SET_PSV_TLS, +#endif }; static inline bool @@ -114,9 +118,16 @@ struct mlx5e_icosq_wqe_info { struct { struct mlx5e_rq *rq; } umr; +#ifdef CONFIG_MLX5_EN_TLS + struct { + struct mlx5e_ktls_offload_context_rx *priv_rx; + } tls_set_params; +#endif }; }; +void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq); + static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size) { struct mlx5_wq_cyc *wq = &sq->wq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index fac145dcf2ce..7b6abea850d4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -37,6 +37,7 @@ #include #include #include "en_accel/ipsec_rxtx.h" +#include "en_accel/tls.h" #include "en_accel/tls_rxtx.h" #include "en.h" #include "en/txrx.h" @@ -147,4 +148,23 @@ static inline bool mlx5e_accel_tx_finish(struct mlx5e_priv *priv, return true; } +static inline int mlx5e_accel_sk_get_rxq(struct sock *sk) +{ + int rxq = sk_rx_queue_get(sk); + + if (unlikely(rxq == -1)) + rxq = 0; + + return rxq; +} + +static inline int mlx5e_accel_init_rx(struct mlx5e_priv *priv) +{ + return mlx5e_ktls_init_rx(priv); +} + +static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) +{ + mlx5e_ktls_cleanup_rx(priv); +} #endif /* __MLX5E_EN_ACCEL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 8970ea68d005..d4ef016ab444 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -4,6 +4,7 @@ #include "en.h" #include "en_accel/ktls.h" #include "en_accel/ktls_utils.h" +#include "en_accel/fs_tcp.h" static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, enum tls_offload_ctx_dir direction, @@ -14,13 +15,13 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, struct mlx5_core_dev *mdev = priv->mdev; int err; - if (WARN_ON(direction != TLS_OFFLOAD_CTX_DIR_TX)) - return -EINVAL; - if (WARN_ON(!mlx5e_ktls_type_check(mdev, crypto_info))) return -EOPNOTSUPP; - err = mlx5e_ktls_add_tx(netdev, sk, crypto_info, start_offload_tcp_sn); + if (direction == TLS_OFFLOAD_CTX_DIR_TX) + err = mlx5e_ktls_add_tx(netdev, sk, crypto_info, start_offload_tcp_sn); + else + err = mlx5e_ktls_add_rx(netdev, sk, crypto_info, start_offload_tcp_sn); return err; } @@ -29,26 +30,71 @@ static void mlx5e_ktls_del(struct net_device *netdev, struct tls_context *tls_ctx, enum tls_offload_ctx_dir direction) { - if (direction != TLS_OFFLOAD_CTX_DIR_TX) - return; + if (direction == TLS_OFFLOAD_CTX_DIR_TX) + mlx5e_ktls_del_tx(netdev, tls_ctx); + else + mlx5e_ktls_del_rx(netdev, tls_ctx); +} - mlx5e_ktls_del_tx(netdev, tls_ctx); +static int mlx5e_ktls_resync(struct net_device *netdev, + struct sock *sk, u32 seq, u8 *rcd_sn, + enum tls_offload_ctx_dir direction) +{ + return -EOPNOTSUPP; } static const struct tlsdev_ops mlx5e_ktls_ops = { .tls_dev_add = mlx5e_ktls_add, .tls_dev_del = mlx5e_ktls_del, + .tls_dev_resync = mlx5e_ktls_resync, }; void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; + struct mlx5_core_dev *mdev = priv->mdev; - if (!mlx5_accel_is_ktls_device(priv->mdev)) + if (!mlx5_accel_is_ktls_device(mdev)) return; - netdev->hw_features |= NETIF_F_HW_TLS_TX; - netdev->features |= NETIF_F_HW_TLS_TX; + if (mlx5_accel_is_ktls_tx(mdev)) { + netdev->hw_features |= NETIF_F_HW_TLS_TX; + netdev->features |= NETIF_F_HW_TLS_TX; + } + + if (mlx5_accel_is_ktls_rx(mdev)) + netdev->hw_features |= NETIF_F_HW_TLS_RX; netdev->tlsdev_ops = &mlx5e_ktls_ops; } + +int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable) +{ + struct mlx5e_priv *priv = netdev_priv(netdev); + int err = 0; + + mutex_lock(&priv->state_lock); + if (enable) + err = mlx5e_accel_fs_tcp_create(priv); + else + mlx5e_accel_fs_tcp_destroy(priv); + mutex_unlock(&priv->state_lock); + + return err; +} + +int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) +{ + int err = 0; + + if (priv->netdev->features & NETIF_F_HW_TLS_RX) + err = mlx5e_accel_fs_tcp_create(priv); + + return err; +} + +void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv) +{ + if (priv->netdev->features & NETIF_F_HW_TLS_RX) + mlx5e_accel_fs_tcp_destroy(priv); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index 69d736954977..baa58b62e8df 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -9,13 +9,30 @@ #ifdef CONFIG_MLX5_EN_TLS void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); - +int mlx5e_ktls_init_rx(struct mlx5e_priv *priv); +void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv); +int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable); #else static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { } +static inline int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) +{ + return 0; +} + +static inline void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv) +{ +} + +static inline int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable) +{ + netdev_warn(netdev, "kTLS is not supported\n"); + return -EOPNOTSUPP; +} + #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c new file mode 100644 index 000000000000..aae4245d0c91 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2019 Mellanox Technologies. + +#include "en_accel/en_accel.h" +#include "en_accel/ktls_txrx.h" +#include "en_accel/ktls_utils.h" +#include "en_accel/fs_tcp.h" + +struct accel_rule { + struct work_struct work; + struct mlx5e_priv *priv; + struct mlx5_flow_handle *rule; +}; + +enum { + MLX5E_PRIV_RX_FLAG_DELETING, + MLX5E_NUM_PRIV_RX_FLAGS, +}; + +struct mlx5e_ktls_offload_context_rx { + struct tls12_crypto_info_aes_gcm_128 crypto_info; + struct accel_rule rule; + struct sock *sk; + struct completion add_ctx; + u32 tirn; + u32 key_id; + u32 rxq; + DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); +}; + +static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, u32 *tirn, u32 rqtn) +{ + int err, inlen; + void *tirc; + u32 *in; + + inlen = MLX5_ST_SZ_BYTES(create_tir_in); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); + + MLX5_SET(tirc, tirc, transport_domain, mdev->mlx5e_res.td.tdn); + MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); + MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); + MLX5_SET(tirc, tirc, indirect_table, rqtn); + MLX5_SET(tirc, tirc, tls_en, 1); + MLX5_SET(tirc, tirc, self_lb_block, + MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST | + MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST); + + err = mlx5_core_create_tir(mdev, in, tirn); + + kvfree(in); + return err; +} + +static void accel_rule_handle_work(struct work_struct *work) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct accel_rule *accel_rule; + struct mlx5_flow_handle *rule; + + accel_rule = container_of(work, struct accel_rule, work); + priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule); + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) + goto out; + + rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk, + priv_rx->tirn, MLX5_FS_DEFAULT_FLOW_TAG); + if (!IS_ERR_OR_NULL(rule)) + accel_rule->rule = rule; +out: + complete(&priv_rx->add_ctx); +} + +static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv, + struct sock *sk) +{ + INIT_WORK(&rule->work, accel_rule_handle_work); + rule->priv = priv; +} + +static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi, + struct mlx5e_icosq_wqe_info *wi) +{ + sq->db.wqe_info[pi] = *wi; +} + +static struct mlx5_wqe_ctrl_seg * +post_static_params(struct mlx5e_icosq *sq, + struct mlx5e_ktls_offload_context_rx *priv_rx) +{ + struct mlx5e_set_tls_static_params_wqe *wqe; + struct mlx5e_icosq_wqe_info wi; + u16 pi, num_wqebbs, room; + + num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; + room = mlx5e_stop_room_for_wqe(num_wqebbs); + if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room))) + return ERR_PTR(-ENOSPC); + + pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); + wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, + priv_rx->tirn, priv_rx->key_id, false, + TLS_OFFLOAD_CTX_DIR_RX); + wi = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, + .num_wqebbs = num_wqebbs, + .tls_set_params.priv_rx = priv_rx, + }; + icosq_fill_wi(sq, pi, &wi); + sq->pc += num_wqebbs; + + return &wqe->ctrl; +} + +static struct mlx5_wqe_ctrl_seg * +post_progress_params(struct mlx5e_icosq *sq, + struct mlx5e_ktls_offload_context_rx *priv_rx, + u32 next_record_tcp_sn) +{ + struct mlx5e_set_tls_progress_params_wqe *wqe; + struct mlx5e_icosq_wqe_info wi; + u16 pi, num_wqebbs, room; + + num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; + room = mlx5e_stop_room_for_wqe(num_wqebbs); + if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room))) + return ERR_PTR(-ENOSPC); + + pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); + wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_rx->tirn, false, + next_record_tcp_sn, + TLS_OFFLOAD_CTX_DIR_RX); + wi = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS, + .num_wqebbs = num_wqebbs, + .tls_set_params.priv_rx = priv_rx, + }; + + icosq_fill_wi(sq, pi, &wi); + sq->pc += num_wqebbs; + + return &wqe->ctrl; +} + +static int post_rx_param_wqes(struct mlx5e_channel *c, + struct mlx5e_ktls_offload_context_rx *priv_rx, + u32 next_record_tcp_sn) +{ + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5e_icosq *sq; + int err; + + err = 0; + sq = &c->async_icosq; + spin_lock(&c->async_icosq_lock); + + cseg = post_static_params(sq, priv_rx); + if (IS_ERR(cseg)) + goto err_out; + cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn); + if (IS_ERR(cseg)) + goto err_out; + + mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); +unlock: + spin_unlock(&c->async_icosq_lock); + + return err; + +err_out: + err = PTR_ERR(cseg); + complete(&priv_rx->add_ctx); + goto unlock; +} + +static void +mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx, + struct mlx5e_ktls_offload_context_rx *priv_rx) +{ + struct mlx5e_ktls_offload_context_rx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); + + BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_rx *) > + TLS_OFFLOAD_CONTEXT_SIZE_RX); + + *ctx = priv_rx; +} + +static struct mlx5e_ktls_offload_context_rx * +mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx) +{ + struct mlx5e_ktls_offload_context_rx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); + + return *ctx; +} + +void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) +{ + u8 tls_offload = get_cqe_tls_offload(cqe); + + if (likely(tls_offload == CQE_TLS_OFFLOAD_NOT_DECRYPTED)) + return; + + switch (tls_offload) { + case CQE_TLS_OFFLOAD_DECRYPTED: + skb->decrypted = 1; + break; + case CQE_TLS_OFFLOAD_RESYNC: + break; + default: /* CQE_TLS_OFFLOAD_ERROR: */ + break; + } +} + +void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx; + struct accel_rule *rule = &priv_rx->rule; + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { + complete(&priv_rx->add_ctx); + return; + } + queue_work(rule->priv->tls->rx_wq, &rule->work); +} + +int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, + u32 start_offload_tcp_sn) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct tls_context *tls_ctx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + int rxq, err; + u32 rqtn; + + tls_ctx = tls_get_ctx(sk); + priv = netdev_priv(netdev); + mdev = priv->mdev; + priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL); + if (unlikely(!priv_rx)) + return -ENOMEM; + + err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id); + if (err) + goto err_create_key; + + priv_rx->crypto_info = + *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + priv_rx->sk = sk; + priv_rx->rxq = mlx5e_accel_sk_get_rxq(sk); + + mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx); + + rxq = priv_rx->rxq; + rqtn = priv->direct_tir[rxq].rqt.rqtn; + + err = mlx5e_ktls_create_tir(mdev, &priv_rx->tirn, rqtn); + if (err) + goto err_create_tir; + + init_completion(&priv_rx->add_ctx); + accel_rule_init(&priv_rx->rule, priv, sk); + err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn); + if (err) + goto err_post_wqes; + + return 0; + +err_post_wqes: + mlx5_core_destroy_tir(mdev, priv_rx->tirn); +err_create_tir: + mlx5_ktls_destroy_key(mdev, priv_rx->key_id); +err_create_key: + kfree(priv_rx); + return err; +} + +void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + + priv = netdev_priv(netdev); + mdev = priv->mdev; + + priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); + set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); + if (!cancel_work_sync(&priv_rx->rule.work)) + /* completion is needed, as the priv_rx in the add flow + * is maintained on the wqe info (wi), not on the socket. + */ + wait_for_completion(&priv_rx->add_ctx); + + if (priv_rx->rule.rule) + mlx5e_accel_fs_del_sk(priv_rx->rule.rule); + + mlx5_core_destroy_tir(mdev, priv_rx->tirn); + mlx5_ktls_destroy_key(mdev, priv_rx->key_id); + kfree(priv_rx); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 5a980f93c326..9c34ffa55b32 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -188,7 +188,7 @@ post_progress_params(struct mlx5e_txqsq *sq, num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); - mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, + mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0, TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c index edf404eaa275..c1f1ad32ca4c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c @@ -59,11 +59,13 @@ mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, { struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + u8 opmod = direction == TLS_OFFLOAD_CTX_DIR_TX ? + MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS : + MLX5_OPC_MOD_TLS_TIR_STATIC_PARAMS; #define STATIC_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS) - cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR | - (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24)); + cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR | (opmod << 24)); cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | STATIC_PARAMS_DS_CNT); cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; @@ -76,12 +78,15 @@ mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, } static void -fill_progress_params(struct mlx5_wqe_tls_progress_params_seg *params, u32 tis_tir_num) +fill_progress_params(struct mlx5_wqe_tls_progress_params_seg *params, u32 tis_tir_num, + u32 next_record_tcp_sn) { u8 *ctx = params->ctx; params->tis_tir_num = cpu_to_be32(tis_tir_num); + MLX5_SET(tls_progress_params, ctx, next_record_tcp_sn, + next_record_tcp_sn); MLX5_SET(tls_progress_params, ctx, record_tracker_state, MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); MLX5_SET(tls_progress_params, ctx, auth_state, @@ -92,19 +97,22 @@ void mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, u16 pc, u32 sqn, u32 tis_tir_num, bool fence, + u32 next_record_tcp_sn, enum tls_offload_ctx_dir direction) { struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + u8 opmod = direction == TLS_OFFLOAD_CTX_DIR_TX ? + MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS : + MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS; #define PROGRESS_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS) cseg->opmod_idx_opcode = - cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV | - (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24)); + cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV | (opmod << 24)); cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | PROGRESS_PARAMS_DS_CNT); cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; - fill_progress_params(&wqe->params, tis_tir_num); + fill_progress_params(&wqe->params, tis_tir_num, next_record_tcp_sn); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h index 7ce6b1c41d9b..7bdd6ec6c981 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h @@ -19,6 +19,10 @@ u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq); bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, struct sk_buff *skb, int datalen, struct mlx5e_accel_tx_tls_state *state); +void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt); + +void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi); void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h index d1d747cb2dcb..566cf24eb0fe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h @@ -23,6 +23,9 @@ enum { int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn); void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx); +int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn); +void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx); struct mlx5e_set_tls_static_params_wqe { struct mlx5_wqe_ctrl_seg ctrl; @@ -64,6 +67,7 @@ void mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, u16 pc, u32 sqn, u32 tis_tir_num, bool fence, + u32 next_record_tcp_sn, enum tls_offload_ctx_dir direction); #endif /* __MLX5E_TLS_UTILS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c index c01c17a5c6de..99beb928feff 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c @@ -197,6 +197,7 @@ void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) return; } + /* FPGA */ if (!mlx5_accel_is_tls_device(priv->mdev)) return; @@ -221,11 +222,19 @@ void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) int mlx5e_tls_init(struct mlx5e_priv *priv) { - struct mlx5e_tls *tls = kzalloc(sizeof(*tls), GFP_KERNEL); + struct mlx5e_tls *tls; + if (!mlx5_accel_is_tls_device(priv->mdev)) + return 0; + + tls = kzalloc(sizeof(*tls), GFP_KERNEL); if (!tls) return -ENOMEM; + tls->rx_wq = create_singlethread_workqueue("mlx5e_tls_rx"); + if (!tls->rx_wq) + return -ENOMEM; + priv->tls = tls; return 0; } @@ -237,6 +246,7 @@ void mlx5e_tls_cleanup(struct mlx5e_priv *priv) if (!tls) return; + destroy_workqueue(tls->rx_wq); kfree(tls); priv->tls = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h index 9015f3f7792d..ca0c2ebb41a1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h @@ -53,6 +53,7 @@ struct mlx5e_tls_sw_stats { struct mlx5e_tls { struct mlx5e_tls_sw_stats sw_stats; + struct workqueue_struct *rx_wq; }; struct mlx5e_tls_offload_context_tx { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index 4d796fea906d..182841322ce4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -278,9 +278,10 @@ bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, if (WARN_ON_ONCE(tls_ctx->netdev != netdev)) goto err_out; - if (MLX5_CAP_GEN(sq->channel->mdev, tls_tx)) + if (mlx5_accel_is_ktls_tx(sq->channel->mdev)) return mlx5e_ktls_handle_tx_skb(tls_ctx, sq, skb, datalen, state); + /* FPGA */ skb_seq = ntohl(tcp_hdr(skb)->seq); context = mlx5e_get_tls_tx_context(tls_ctx); expected_seq = context->expected_seq; @@ -354,12 +355,16 @@ out: return 0; } -void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - u32 *cqe_bcnt) +void mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) { struct mlx5e_tls_metadata *mdata; struct mlx5e_priv *priv; + if (likely(mlx5_accel_is_ktls_rx(rq->mdev))) + return mlx5e_ktls_handle_rx_skb(rq, skb, cqe, cqe_bcnt); + + /* FPGA */ if (!is_metadata_hdr_valid(skb)) return; @@ -370,13 +375,13 @@ void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, skb->decrypted = 1; break; case SYNDROM_RESYNC_REQUEST: - tls_update_resync_sn(netdev, skb, mdata); - priv = netdev_priv(netdev); + tls_update_resync_sn(rq->netdev, skb, mdata); + priv = netdev_priv(rq->netdev); atomic64_inc(&priv->tls->sw_stats.rx_tls_resync_request); break; case SYNDROM_AUTH_FAILED: /* Authentication failure will be observed and verified by kTLS */ - priv = netdev_priv(netdev); + priv = netdev_priv(rq->netdev); atomic64_inc(&priv->tls->sw_stats.rx_tls_auth_fail); break; default: @@ -395,9 +400,10 @@ u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) if (!mlx5_accel_is_tls_device(mdev)) return 0; - if (MLX5_CAP_GEN(mdev, tls_tx)) + if (mlx5_accel_is_ktls_device(mdev)) return mlx5e_ktls_get_stop_room(sq); + /* FPGA */ /* Resync SKB. */ return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h index 2a7b98531539..8bb790674042 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h @@ -49,8 +49,8 @@ bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, struct mlx5e_accel_tx_tls_state *state); -void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - u32 *cqe_bcnt); +void mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt); #else diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 11997c23dfb5..0f1578a5e538 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1441,6 +1441,7 @@ void mlx5e_close_icosq(struct mlx5e_icosq *sq) struct mlx5e_channel *c = sq->channel; mlx5e_destroy_sq(c->mdev, sq->sqn); + mlx5e_free_icosq_descs(sq); mlx5e_free_icosq(sq); } @@ -3853,6 +3854,7 @@ int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) #ifdef CONFIG_MLX5_EN_ARFS err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); #endif + err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); if (err) { netdev->features = oper_features; @@ -5143,8 +5145,14 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) if (err) goto err_destroy_flow_steering; + err = mlx5e_accel_init_rx(priv); + if (err) + goto err_tc_nic_cleanup; + return 0; +err_tc_nic_cleanup: + mlx5e_tc_nic_cleanup(priv); err_destroy_flow_steering: mlx5e_destroy_flow_steering(priv); err_destroy_xsk_tirs: @@ -5168,6 +5176,7 @@ err_destroy_q_counters: static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) { + mlx5e_accel_cleanup_rx(priv); mlx5e_tc_nic_cleanup(priv); mlx5e_destroy_flow_steering(priv); mlx5e_destroy_direct_tirs(priv, priv->xsk_tir); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index dbb1c6323967..9a6958acf87d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -578,6 +578,30 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) return !!err; } +void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq) +{ + u16 sqcc; + + sqcc = sq->cc; + + while (sqcc != sq->pc) { + struct mlx5e_icosq_wqe_info *wi; + u16 ci; + + ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); + wi = &sq->db.wqe_info[ci]; + sqcc += wi->num_wqebbs; +#ifdef CONFIG_MLX5_EN_TLS + switch (wi->wqe_type) { + case MLX5E_ICOSQ_WQE_SET_PSV_TLS: + mlx5e_ktls_handle_ctx_completion(wi); + break; + } +#endif + } + sq->cc = sqcc; +} + int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) { struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq); @@ -633,6 +657,13 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) break; case MLX5E_ICOSQ_WQE_NOP: break; +#ifdef CONFIG_MLX5_EN_TLS + case MLX5E_ICOSQ_WQE_UMR_TLS: + break; + case MLX5E_ICOSQ_WQE_SET_PSV_TLS: + mlx5e_ktls_handle_ctx_completion(wi); + break; +#endif default: netdev_WARN_ONCE(cq->channel->netdev, "Bad WQE type in ICOSQ WQE info: 0x%x\n", @@ -983,7 +1014,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, skb->mac_len = ETH_HLEN; #ifdef CONFIG_MLX5_EN_TLS - mlx5e_tls_handle_rx_skb(netdev, skb, &cqe_bcnt); + mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt); #endif if (lro_num_seg > 1) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index a5fbe7343508..c3095863372c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -35,6 +35,7 @@ #include #include "mlx5_core.h" #include "../../mlxfw/mlxfw.h" +#include "accel/tls.h" enum { MCQS_IDENTIFIER_BOOT_IMG = 0x1, @@ -236,7 +237,7 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } - if (MLX5_CAP_GEN(dev, tls_tx)) { + if (mlx5_accel_is_ktls_tx(dev) || mlx5_accel_is_ktls_rx(dev)) { err = mlx5_core_get_caps(dev, MLX5_CAP_TLS); if (err) return err; From acb5a07aaf2723cd273a4089e62611a414fb1c35 Mon Sep 17 00:00:00 2001 From: Boris Pismenny Date: Mon, 8 Jun 2020 12:42:52 +0300 Subject: [PATCH 14/20] Revert "net/tls: Add force_resync for driver resync" This reverts commit b3ae2459f89773adcbf16fef4b68deaaa3be1929. Revert the force resync API. Not in use. To be replaced by a better async resync API downstream. Signed-off-by: Boris Pismenny Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- include/net/tls.h | 12 +----------- net/tls/tls_device.c | 9 +++------ 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/include/net/tls.h b/include/net/tls.h index 3212d3c214a9..ca5f7f437289 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -607,22 +607,12 @@ tls_driver_ctx(const struct sock *sk, enum tls_offload_ctx_dir direction) #endif /* The TLS context is valid until sk_destruct is called */ -#define RESYNC_REQ (1 << 0) -#define RESYNC_REQ_FORCE (1 << 1) static inline void tls_offload_rx_resync_request(struct sock *sk, __be32 seq) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); - atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | RESYNC_REQ); -} - -static inline void tls_offload_rx_force_resync_request(struct sock *sk) -{ - struct tls_context *tls_ctx = tls_get_ctx(sk); - struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); - - atomic64_set(&rx_ctx->resync_req, RESYNC_REQ | RESYNC_REQ_FORCE); + atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | 1); } static inline void diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 0e55f8365ce2..a562ebaaa33c 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -694,11 +694,10 @@ void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_offload_context_rx *rx_ctx; - bool is_req_pending, is_force_resync; u8 rcd_sn[TLS_MAX_REC_SEQ_SIZE]; + u32 sock_data, is_req_pending; struct tls_prot_info *prot; s64 resync_req; - u32 sock_data; u32 req_seq; if (tls_ctx->rx_conf != TLS_HW) @@ -713,11 +712,9 @@ void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) resync_req = atomic64_read(&rx_ctx->resync_req); req_seq = resync_req >> 32; seq += TLS_HEADER_SIZE - 1; - is_req_pending = resync_req & RESYNC_REQ; - is_force_resync = resync_req & RESYNC_REQ_FORCE; + is_req_pending = resync_req; - if (likely(!is_req_pending) || - (!is_force_resync && req_seq != seq) || + if (likely(!is_req_pending) || req_seq != seq || !atomic64_try_cmpxchg(&rx_ctx->resync_req, &resync_req, 0)) return; break; From ed9b7646b06a2ed2450dd9437fc7d1ad2783140c Mon Sep 17 00:00:00 2001 From: Boris Pismenny Date: Mon, 8 Jun 2020 19:11:38 +0300 Subject: [PATCH 15/20] net/tls: Add asynchronous resync This patch adds support for asynchronous resynchronization in tls_device. Async resync follows two distinct stages: 1. The NIC driver indicates that it would like to resync on some TLS record within the received packet (P), but the driver does not know (yet) which of the TLS records within the packet. At this stage, the NIC driver will query the device to find the exact TCP sequence for resync (tcpsn), however, the driver does not wait for the device to provide the response. 2. Eventually, the device responds, and the driver provides the tcpsn within the resync packet to KTLS. Now, KTLS can check the tcpsn against any processed TLS records within packet P, and also against any record that is processed in the future within packet P. The asynchronous resync path simplifies the device driver, as it can save bits on the packet completion (32-bit TCP sequence), and pass this information on an asynchronous command instead. Signed-off-by: Boris Pismenny Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- include/net/tls.h | 38 ++++++++++++++++++++++++++++++++- net/tls/tls_device.c | 51 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/include/net/tls.h b/include/net/tls.h index ca5f7f437289..c875c0a445a6 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -291,11 +291,19 @@ struct tlsdev_ops { enum tls_offload_sync_type { TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ = 0, TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT = 1, + TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC = 2, }; #define TLS_DEVICE_RESYNC_NH_START_IVAL 2 #define TLS_DEVICE_RESYNC_NH_MAX_IVAL 128 +#define TLS_DEVICE_RESYNC_ASYNC_LOGMAX 13 +struct tls_offload_resync_async { + atomic64_t req; + u32 loglen; + u32 log[TLS_DEVICE_RESYNC_ASYNC_LOGMAX]; +}; + struct tls_offload_context_rx { /* sw must be the first member of tls_offload_context_rx */ struct tls_sw_context_rx sw; @@ -314,6 +322,10 @@ struct tls_offload_context_rx { u32 decrypted_failed; u32 decrypted_tgt; } resync_nh; + /* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC */ + struct { + struct tls_offload_resync_async *resync_async; + }; }; u8 driver_state[] __aligned(8); /* The TLS layer reserves room for driver specific state @@ -606,13 +618,37 @@ tls_driver_ctx(const struct sock *sk, enum tls_offload_ctx_dir direction) } #endif +#define RESYNC_REQ BIT(0) +#define RESYNC_REQ_ASYNC BIT(1) /* The TLS context is valid until sk_destruct is called */ static inline void tls_offload_rx_resync_request(struct sock *sk, __be32 seq) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); - atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | 1); + atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | RESYNC_REQ); +} + +/* Log all TLS record header TCP sequences in [seq, seq+len] */ +static inline void +tls_offload_rx_resync_async_request_start(struct sock *sk, __be32 seq, u16 len) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); + + atomic64_set(&rx_ctx->resync_async->req, ((u64)ntohl(seq) << 32) | + (len << 16) | RESYNC_REQ | RESYNC_REQ_ASYNC); + rx_ctx->resync_async->loglen = 0; +} + +static inline void +tls_offload_rx_resync_async_request_end(struct sock *sk, __be32 seq) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); + + atomic64_set(&rx_ctx->resync_async->req, + ((u64)ntohl(seq) << 32) | RESYNC_REQ); } static inline void diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index a562ebaaa33c..18fa6067bb7f 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -690,6 +690,47 @@ static void tls_device_resync_rx(struct tls_context *tls_ctx, TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICERESYNC); } +static bool +tls_device_rx_resync_async(struct tls_offload_resync_async *resync_async, + s64 resync_req, u32 *seq) +{ + u32 is_async = resync_req & RESYNC_REQ_ASYNC; + u32 req_seq = resync_req >> 32; + u32 req_end = req_seq + ((resync_req >> 16) & 0xffff); + + if (is_async) { + /* asynchronous stage: log all headers seq such that + * req_seq <= seq <= end_seq, and wait for real resync request + */ + if (between(*seq, req_seq, req_end) && + resync_async->loglen < TLS_DEVICE_RESYNC_ASYNC_LOGMAX) + resync_async->log[resync_async->loglen++] = *seq; + + return false; + } + + /* synchronous stage: check against the logged entries and + * proceed to check the next entries if no match was found + */ + while (resync_async->loglen) { + if (req_seq == resync_async->log[resync_async->loglen - 1] && + atomic64_try_cmpxchg(&resync_async->req, + &resync_req, 0)) { + resync_async->loglen = 0; + *seq = req_seq; + return true; + } + resync_async->loglen--; + } + + if (req_seq == *seq && + atomic64_try_cmpxchg(&resync_async->req, + &resync_req, 0)) + return true; + + return false; +} + void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) { struct tls_context *tls_ctx = tls_get_ctx(sk); @@ -736,6 +777,16 @@ void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) seq += rcd_len; tls_bigint_increment(rcd_sn, prot->rec_seq_size); break; + case TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC: + resync_req = atomic64_read(&rx_ctx->resync_async->req); + is_req_pending = resync_req; + if (likely(!is_req_pending)) + return; + + if (!tls_device_rx_resync_async(rx_ctx->resync_async, + resync_req, &seq)) + return; + break; } tls_device_resync_rx(tls_ctx, sk, seq, rcd_sn); From 0419d8c9d8f8d825576a41b2bb1e6043f34d1ae0 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Tue, 16 Jun 2020 15:15:06 +0300 Subject: [PATCH 16/20] net/mlx5e: kTLS, Add kTLS RX resync support Implement the RX resync procedure, using the TLS async resync API. The HW offload of TLS decryption in RX side might get out-of-sync due to out-of-order reception of packets. This requires SW intervention to update the HW context and get it back in-sync. Performance: CPU: Intel(R) Xeon(R) CPU E5-2687W v4 @ 3.00GHz, 24 cores, HT off NIC: ConnectX-6 Dx 100GbE dual port Goodput (app-layer throughput) comparison: +---------------+-------+-------+---------+ | # connections | 1 | 4 | 8 | +---------------+-------+-------+---------+ | SW (Gbps) | 7.26 | 24.70 | 50.30 | +---------------+-------+-------+---------+ | HW (Gbps) | 18.50 | 64.30 | 92.90 | +---------------+-------+-------+---------+ | Speedup | 2.55x | 2.56x | 1.85x * | +---------------+-------+-------+---------+ * After linerate is reached, diff is observed in CPU util. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/en/txrx.h | 4 + .../mellanox/mlx5/core/en_accel/ktls.c | 6 +- .../mellanox/mlx5/core/en_accel/ktls_rx.c | 346 +++++++++++++++++- .../mellanox/mlx5/core/en_accel/ktls_tx.c | 2 +- .../mellanox/mlx5/core/en_accel/ktls_txrx.c | 7 +- .../mellanox/mlx5/core/en_accel/ktls_txrx.h | 2 + .../mellanox/mlx5/core/en_accel/ktls_utils.h | 15 +- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 6 + 8 files changed, 381 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index 7f55bd3229f1..e9d4a61b6bbb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -14,6 +14,7 @@ enum mlx5e_icosq_wqe_type { #ifdef CONFIG_MLX5_EN_TLS MLX5E_ICOSQ_WQE_UMR_TLS, MLX5E_ICOSQ_WQE_SET_PSV_TLS, + MLX5E_ICOSQ_WQE_GET_PSV_TLS, #endif }; @@ -122,6 +123,9 @@ struct mlx5e_icosq_wqe_info { struct { struct mlx5e_ktls_offload_context_rx *priv_rx; } tls_set_params; + struct { + struct mlx5e_ktls_rx_resync_buf *buf; + } tls_get_params; #endif }; }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index d4ef016ab444..deec17af5a69 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -40,7 +40,11 @@ static int mlx5e_ktls_resync(struct net_device *netdev, struct sock *sk, u32 seq, u8 *rcd_sn, enum tls_offload_ctx_dir direction) { - return -EOPNOTSUPP; + if (unlikely(direction != TLS_OFFLOAD_CTX_DIR_RX)) + return -EOPNOTSUPP; + + mlx5e_ktls_rx_resync(netdev, sk, seq, rcd_sn); + return 0; } static const struct tlsdev_ops mlx5e_ktls_ops = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index aae4245d0c91..b26dad909ef5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2019 Mellanox Technologies. +#include #include "en_accel/en_accel.h" +#include "en_accel/tls.h" #include "en_accel/ktls_txrx.h" #include "en_accel/ktls_utils.h" #include "en_accel/fs_tcp.h" @@ -12,11 +14,34 @@ struct accel_rule { struct mlx5_flow_handle *rule; }; +#define PROGRESS_PARAMS_WRITE_UNIT 64 +#define PROGRESS_PARAMS_PADDED_SIZE \ + (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \ + PROGRESS_PARAMS_WRITE_UNIT)) + +struct mlx5e_ktls_rx_resync_buf { + union { + struct mlx5_wqe_tls_progress_params_seg progress; + u8 pad[PROGRESS_PARAMS_PADDED_SIZE]; + } ____cacheline_aligned_in_smp; + dma_addr_t dma_addr; + struct mlx5e_ktls_offload_context_rx *priv_rx; +}; + enum { MLX5E_PRIV_RX_FLAG_DELETING, MLX5E_NUM_PRIV_RX_FLAGS, }; +struct mlx5e_ktls_rx_resync_ctx { + struct tls_offload_resync_async core; + struct work_struct work; + struct mlx5e_priv *priv; + refcount_t refcnt; + __be64 sw_rcd_sn_be; + u32 seq; +}; + struct mlx5e_ktls_offload_context_rx { struct tls12_crypto_info_aes_gcm_128 crypto_info; struct accel_rule rule; @@ -26,6 +51,9 @@ struct mlx5e_ktls_offload_context_rx { u32 key_id; u32 rxq; DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); + + /* resync */ + struct mlx5e_ktls_rx_resync_ctx resync; }; static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, u32 *tirn, u32 rqtn) @@ -104,7 +132,8 @@ post_static_params(struct mlx5e_icosq *sq, pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, - priv_rx->tirn, priv_rx->key_id, false, + priv_rx->tirn, priv_rx->key_id, + priv_rx->resync.seq, false, TLS_OFFLOAD_CTX_DIR_RX); wi = (struct mlx5e_icosq_wqe_info) { .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, @@ -201,6 +230,281 @@ mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx) return *ctx; } +/* Re-sync */ +/* Runs in work context */ +static struct mlx5_wqe_ctrl_seg * +resync_post_get_progress_params(struct mlx5e_icosq *sq, + struct mlx5e_ktls_offload_context_rx *priv_rx) +{ + struct mlx5e_get_tls_progress_params_wqe *wqe; + struct mlx5e_ktls_rx_resync_buf *buf; + struct mlx5e_icosq_wqe_info wi; + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5_seg_get_psv *psv; + struct device *pdev; + int err; + u16 pi; + + buf = kzalloc(sizeof(*buf), GFP_KERNEL); + if (unlikely(!buf)) { + err = -ENOMEM; + goto err_out; + } + + pdev = sq->channel->priv->mdev->device; + buf->dma_addr = dma_map_single(pdev, &buf->progress, + PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { + err = -ENOMEM; + goto err_out; + } + + buf->priv_rx = priv_rx; + + BUILD_BUG_ON(MLX5E_KTLS_GET_PROGRESS_WQEBBS != 1); + if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) { + err = -ENOSPC; + goto err_out; + } + + pi = mlx5e_icosq_get_next_pi(sq, 1); + wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi); + +#define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)) + + cseg = &wqe->ctrl; + cseg->opmod_idx_opcode = + cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV | + (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24)); + cseg->qpn_ds = + cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT); + + psv = &wqe->psv; + psv->num_psv = 1 << 4; + psv->l_key = sq->channel->mkey_be; + psv->psv_index[0] = cpu_to_be32(priv_rx->tirn); + psv->va = cpu_to_be64(buf->dma_addr); + + wi = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS, + .num_wqebbs = 1, + .tls_get_params.buf = buf, + }; + icosq_fill_wi(sq, pi, &wi); + sq->pc++; + + return cseg; + +err_out: + return ERR_PTR(err); +} + +/* Function is called with elevated refcount. + * It decreases it only if no WQE is posted. + */ +static void resync_handle_work(struct work_struct *work) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5e_channel *c; + struct mlx5e_icosq *sq; + struct mlx5_wq_cyc *wq; + + resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work); + priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync); + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { + refcount_dec(&resync->refcnt); + return; + } + + c = resync->priv->channels.c[priv_rx->rxq]; + sq = &c->async_icosq; + wq = &sq->wq; + + spin_lock(&c->async_icosq_lock); + + cseg = resync_post_get_progress_params(sq, priv_rx); + if (IS_ERR(cseg)) { + refcount_dec(&resync->refcnt); + goto unlock; + } + mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg); +unlock: + spin_unlock(&c->async_icosq_lock); +} + +static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync, + struct mlx5e_priv *priv) +{ + INIT_WORK(&resync->work, resync_handle_work); + resync->priv = priv; + refcount_set(&resync->refcnt, 1); +} + +/* Function can be called with the refcount being either elevated or not. + * It does not affect the refcount. + */ +static int resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx, + struct mlx5e_channel *c) +{ + struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info; + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5e_icosq *sq; + int err; + + memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq)); + err = 0; + + sq = &c->async_icosq; + spin_lock(&c->async_icosq_lock); + + cseg = post_static_params(sq, priv_rx); + if (IS_ERR(cseg)) { + err = PTR_ERR(cseg); + goto unlock; + } + /* Do not increment priv_rx refcnt, CQE handling is empty */ + mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); +unlock: + spin_unlock(&c->async_icosq_lock); + + return err; +} + +/* Function is called with elevated refcount, it decreases it. */ +void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, + struct mlx5e_icosq *sq) +{ + struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf; + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + u8 tracker_state, auth_state, *ctx; + u32 hw_seq; + + priv_rx = buf->priv_rx; + resync = &priv_rx->resync; + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) + goto out; + + dma_sync_single_for_cpu(resync->priv->mdev->device, buf->dma_addr, + PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); + + ctx = buf->progress.ctx; + tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state); + auth_state = MLX5_GET(tls_progress_params, ctx, auth_state); + if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING || + auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) + goto out; + + hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn); + tls_offload_rx_resync_async_request_end(priv_rx->sk, cpu_to_be32(hw_seq)); +out: + refcount_dec(&resync->refcnt); + kfree(buf); +} + +/* Runs in NAPI. + * Function elevates the refcount, unless no work is queued. + */ +static bool resync_queue_get_psv(struct sock *sk) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + + priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); + if (unlikely(!priv_rx)) + return false; + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) + return false; + + resync = &priv_rx->resync; + refcount_inc(&resync->refcnt); + if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work))) + refcount_dec(&resync->refcnt); + + return true; +} + +/* Runs in NAPI */ +static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) +{ + struct ethhdr *eth = (struct ethhdr *)(skb->data); + struct net_device *netdev = rq->netdev; + struct sock *sk = NULL; + unsigned int datalen; + struct iphdr *iph; + struct tcphdr *th; + __be32 seq; + int depth = 0; + + __vlan_get_protocol(skb, eth->h_proto, &depth); + iph = (struct iphdr *)(skb->data + depth); + + if (iph->version == 4) { + depth += sizeof(struct iphdr); + th = (void *)iph + sizeof(struct iphdr); + + sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, + iph->saddr, th->source, iph->daddr, + th->dest, netdev->ifindex); +#if IS_ENABLED(CONFIG_IPV6) + } else { + struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph; + + depth += sizeof(struct ipv6hdr); + th = (void *)ipv6h + sizeof(struct ipv6hdr); + + sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, + &ipv6h->saddr, th->source, + &ipv6h->daddr, ntohs(th->dest), + netdev->ifindex, 0); +#endif + } + + depth += sizeof(struct tcphdr); + + if (unlikely(!sk || sk->sk_state == TCP_TIME_WAIT)) + return; + + if (unlikely(!resync_queue_get_psv(sk))) + return; + + skb->sk = sk; + skb->destructor = sock_edemux; + + seq = th->seq; + datalen = skb->len - depth; + tls_offload_rx_resync_async_request_start(sk, seq, datalen); +} + +void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, + u32 seq, u8 *rcd_sn) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + struct mlx5e_priv *priv; + struct mlx5e_channel *c; + + priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); + if (unlikely(!priv_rx)) + return; + + resync = &priv_rx->resync; + resync->sw_rcd_sn_be = *(__be64 *)rcd_sn; + resync->seq = seq; + + priv = netdev_priv(netdev); + c = priv->channels.c[priv_rx->rxq]; + + resync_handle_seq_match(priv_rx, c); +} + +/* End of resync section */ + void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) { @@ -214,6 +518,7 @@ void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, skb->decrypted = 1; break; case CQE_TLS_OFFLOAD_RESYNC: + resync_update_sn(rq, skb); break; default: /* CQE_TLS_OFFLOAD_ERROR: */ break; @@ -237,6 +542,7 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, u32 start_offload_tcp_sn) { struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; struct tls_context *tls_ctx; struct mlx5_core_dev *mdev; struct mlx5e_priv *priv; @@ -269,7 +575,13 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, goto err_create_tir; init_completion(&priv_rx->add_ctx); + accel_rule_init(&priv_rx->rule, priv, sk); + resync = &priv_rx->resync; + resync_init(resync, priv); + tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core; + tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC); + err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn); if (err) goto err_post_wqes; @@ -285,9 +597,35 @@ err_create_key: return err; } +/* Elevated refcount on the resync object means there are + * outstanding operations (uncompleted GET_PSV WQEs) that + * will read the resync / priv_rx objects once completed. + * Wait for them to avoid use-after-free. + */ +static void wait_for_resync(struct net_device *netdev, + struct mlx5e_ktls_rx_resync_ctx *resync) +{ +#define MLX5E_KTLS_RX_RESYNC_TIMEOUT 20000 /* msecs */ + unsigned long exp_time = jiffies + msecs_to_jiffies(MLX5E_KTLS_RX_RESYNC_TIMEOUT); + unsigned int refcnt; + + do { + refcnt = refcount_read(&resync->refcnt); + if (refcnt == 1) + return; + + msleep(20); + } while (time_before(jiffies, exp_time)); + + netdev_warn(netdev, + "Failed waiting for kTLS RX resync refcnt to be released (%u).\n", + refcnt); +} + void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) { struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; struct mlx5_core_dev *mdev; struct mlx5e_priv *priv; @@ -296,11 +634,17 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); + mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL); + napi_synchronize(&priv->channels.c[priv_rx->rxq]->napi); if (!cancel_work_sync(&priv_rx->rule.work)) /* completion is needed, as the priv_rx in the add flow * is maintained on the wqe info (wi), not on the socket. */ wait_for_completion(&priv_rx->add_ctx); + resync = &priv_rx->resync; + if (cancel_work_sync(&resync->work)) + refcount_dec(&resync->refcnt); + wait_for_resync(netdev, resync); if (priv_rx->rule.rule) mlx5e_accel_fs_del_sk(priv_rx->rule.rule); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 9c34ffa55b32..0e6698d1b4ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -171,7 +171,7 @@ post_static_params(struct mlx5e_txqsq *sq, pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, - priv_tx->tisn, priv_tx->key_id, fence, + priv_tx->tisn, priv_tx->key_id, 0, fence, TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c index c1f1ad32ca4c..ac29aeb8af49 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c @@ -22,7 +22,7 @@ enum { static void fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, struct tls12_crypto_info_aes_gcm_128 *info, - u32 key_id) + u32 key_id, u32 resync_tcp_sn) { char *initial_rn, *gcm_iv; u16 salt_sz, rec_seq_sz; @@ -47,6 +47,7 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, MLX5_SET(tls_static_params, ctx, const_2, 2); MLX5_SET(tls_static_params, ctx, encryption_standard, MLX5E_ENCRYPTION_STANDARD_TLS); + MLX5_SET(tls_static_params, ctx, resync_tcp_sn, resync_tcp_sn); MLX5_SET(tls_static_params, ctx, dek_index, key_id); } @@ -54,7 +55,7 @@ void mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, u16 pc, u32 sqn, struct tls12_crypto_info_aes_gcm_128 *info, - u32 tis_tir_num, u32 key_id, + u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn, bool fence, enum tls_offload_ctx_dir direction) { struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; @@ -74,7 +75,7 @@ mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, ucseg->flags = MLX5_UMR_INLINE; ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); - fill_static_params(&wqe->params, info, key_id); + fill_static_params(&wqe->params, info, key_id, resync_tcp_sn); } static void diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h index 7bdd6ec6c981..ff4c740af10b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h @@ -23,6 +23,8 @@ void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, struct mlx5_cqe64 *cqe, u32 *cqe_bcnt); void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi); +void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, + struct mlx5e_icosq *sq); void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h index 566cf24eb0fe..e5c180f2403b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h @@ -26,6 +26,7 @@ void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx); int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn); void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx); +void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, u32 seq, u8 *rcd_sn); struct mlx5e_set_tls_static_params_wqe { struct mlx5_wqe_ctrl_seg ctrl; @@ -39,12 +40,20 @@ struct mlx5e_set_tls_progress_params_wqe { struct mlx5_wqe_tls_progress_params_seg params; }; +struct mlx5e_get_tls_progress_params_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_seg_get_psv psv; +}; + #define MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS \ (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_static_params_wqe), MLX5_SEND_WQE_BB)) #define MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS \ (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_progress_params_wqe), MLX5_SEND_WQE_BB)) +#define MLX5E_KTLS_GET_PROGRESS_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_get_tls_progress_params_wqe), MLX5_SEND_WQE_BB)) + #define MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi) \ ((struct mlx5e_set_tls_static_params_wqe *)\ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_static_params_wqe))) @@ -53,6 +62,10 @@ struct mlx5e_set_tls_progress_params_wqe { ((struct mlx5e_set_tls_progress_params_wqe *)\ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_progress_params_wqe))) +#define MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi) \ + ((struct mlx5e_get_tls_progress_params_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_get_tls_progress_params_wqe))) + #define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \ ((struct mlx5e_dump_wqe *)\ mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_dump_wqe))) @@ -61,7 +74,7 @@ void mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, u16 pc, u32 sqn, struct tls12_crypto_info_aes_gcm_128 *info, - u32 tis_tir_num, u32 key_id, + u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn, bool fence, enum tls_offload_ctx_dir direction); void mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 9a6958acf87d..4b7c119c8946 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -596,6 +596,9 @@ void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq) case MLX5E_ICOSQ_WQE_SET_PSV_TLS: mlx5e_ktls_handle_ctx_completion(wi); break; + case MLX5E_ICOSQ_WQE_GET_PSV_TLS: + mlx5e_ktls_handle_get_psv_completion(wi, sq); + break; } #endif } @@ -663,6 +666,9 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) case MLX5E_ICOSQ_WQE_SET_PSV_TLS: mlx5e_ktls_handle_ctx_completion(wi); break; + case MLX5E_ICOSQ_WQE_GET_PSV_TLS: + mlx5e_ktls_handle_get_psv_completion(wi, sq); + break; #endif default: netdev_WARN_ONCE(cq->channel->netdev, From 76c1e1ac2aaeddd5505e4ecfafa963885b5551ab Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Mon, 15 Jun 2020 15:25:23 +0300 Subject: [PATCH 17/20] net/mlx5e: kTLS, Add kTLS RX stats Add global and per-channel ethtool SW stats for the device offload. Document the new counters in tls-offload.rst. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- Documentation/networking/tls-offload.rst | 18 +++++++++ .../mellanox/mlx5/core/en_accel/ktls_rx.c | 29 ++++++++++++-- .../ethernet/mellanox/mlx5/core/en_stats.c | 39 +++++++++++++++++++ .../ethernet/mellanox/mlx5/core/en_stats.h | 25 ++++++++++++ 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/Documentation/networking/tls-offload.rst b/Documentation/networking/tls-offload.rst index f914e81fd3a6..37773da2bee5 100644 --- a/Documentation/networking/tls-offload.rst +++ b/Documentation/networking/tls-offload.rst @@ -428,6 +428,24 @@ by the driver: which were part of a TLS stream. * ``rx_tls_decrypted_bytes`` - number of TLS payload bytes in RX packets which were successfully decrypted. + * ``rx_tls_ctx`` - number of TLS RX HW offload contexts added to device for + decryption. + * ``rx_tls_del`` - number of TLS RX HW offload contexts deleted from device + (connection has finished). + * ``rx_tls_resync_req_pkt`` - number of received TLS packets with a resync + request. + * ``rx_tls_resync_req_start`` - number of times the TLS async resync request + was started. + * ``rx_tls_resync_req_end`` - number of times the TLS async resync request + properly ended with providing the HW tracked tcp-seq. + * ``rx_tls_resync_req_skip`` - number of times the TLS async resync request + procedure was started by not properly ended. + * ``rx_tls_resync_res_ok`` - number of times the TLS resync response call to + the driver was successfully handled. + * ``rx_tls_resync_res_skip`` - number of times the TLS resync response call to + the driver was terminated unsuccessfully. + * ``rx_tls_err`` - number of RX packets which were part of a TLS stream + but were not decrypted due to unexpected error in the state machine. * ``tx_tls_encrypted_packets`` - number of TX packets passed to the device for encryption of their TLS payload. * ``tx_tls_encrypted_bytes`` - number of TLS payload bytes in TX packets diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index b26dad909ef5..e0a8f9d63b30 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -46,6 +46,7 @@ struct mlx5e_ktls_offload_context_rx { struct tls12_crypto_info_aes_gcm_128 crypto_info; struct accel_rule rule; struct sock *sk; + struct mlx5e_rq_stats *stats; struct completion add_ctx; u32 tirn; u32 key_id; @@ -203,6 +204,7 @@ unlock: return err; err_out: + priv_rx->stats->tls_resync_req_skip++; err = PTR_ERR(cseg); complete(&priv_rx->add_ctx); goto unlock; @@ -296,6 +298,7 @@ resync_post_get_progress_params(struct mlx5e_icosq *sq, return cseg; err_out: + priv_rx->stats->tls_resync_req_skip++; return ERR_PTR(err); } @@ -362,11 +365,13 @@ static int resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx cseg = post_static_params(sq, priv_rx); if (IS_ERR(cseg)) { + priv_rx->stats->tls_resync_res_skip++; err = PTR_ERR(cseg); goto unlock; } /* Do not increment priv_rx refcnt, CQE handling is empty */ mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); + priv_rx->stats->tls_resync_res_ok++; unlock: spin_unlock(&c->async_icosq_lock); @@ -396,11 +401,14 @@ void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state); auth_state = MLX5_GET(tls_progress_params, ctx, auth_state); if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING || - auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) + auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) { + priv_rx->stats->tls_resync_req_skip++; goto out; + } hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn); tls_offload_rx_resync_async_request_end(priv_rx->sk, cpu_to_be32(hw_seq)); + priv_rx->stats->tls_resync_req_end++; out: refcount_dec(&resync->refcnt); kfree(buf); @@ -479,6 +487,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) seq = th->seq; datalen = skb->len - depth; tls_offload_rx_resync_async_request_start(sk, seq, datalen); + rq->stats->tls_resync_req_start++; } void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, @@ -509,18 +518,25 @@ void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) { u8 tls_offload = get_cqe_tls_offload(cqe); + struct mlx5e_rq_stats *stats; if (likely(tls_offload == CQE_TLS_OFFLOAD_NOT_DECRYPTED)) return; + stats = rq->stats; + switch (tls_offload) { case CQE_TLS_OFFLOAD_DECRYPTED: skb->decrypted = 1; + stats->tls_decrypted_packets++; + stats->tls_decrypted_bytes += *cqe_bcnt; break; case CQE_TLS_OFFLOAD_RESYNC: + stats->tls_resync_req_pkt++; resync_update_sn(rq, skb); break; default: /* CQE_TLS_OFFLOAD_ERROR: */ + stats->tls_err++; break; } } @@ -562,12 +578,14 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, priv_rx->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; - priv_rx->sk = sk; - priv_rx->rxq = mlx5e_accel_sk_get_rxq(sk); + rxq = mlx5e_accel_sk_get_rxq(sk); + priv_rx->rxq = rxq; + priv_rx->sk = sk; + + priv_rx->stats = &priv->channel_stats[rxq].rq; mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx); - rxq = priv_rx->rxq; rqtn = priv->direct_tir[rxq].rqt.rqtn; err = mlx5e_ktls_create_tir(mdev, &priv_rx->tirn, rqtn); @@ -586,6 +604,8 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, if (err) goto err_post_wqes; + priv_rx->stats->tls_ctx++; + return 0; err_post_wqes: @@ -646,6 +666,7 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) refcount_dec(&resync->refcnt); wait_for_resync(netdev, resync); + priv_rx->stats->tls_del++; if (priv_rx->rule.rule) mlx5e_accel_fs_del_sk(priv_rx->rule.rule); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index f009fe09e99b..e3b2f59408e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -163,6 +163,19 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_congst_umr) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_arfs_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_recover) }, +#ifdef CONFIG_MLX5_EN_TLS + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_decrypted_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_decrypted_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_ctx) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_del) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_pkt) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_start) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_end) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_skip) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_res_ok) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_res_skip) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_err) }, +#endif { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_events) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_poll) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_arm) }, @@ -275,6 +288,19 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw) s->rx_congst_umr += rq_stats->congst_umr; s->rx_arfs_err += rq_stats->arfs_err; s->rx_recover += rq_stats->recover; +#ifdef CONFIG_MLX5_EN_TLS + s->rx_tls_decrypted_packets += rq_stats->tls_decrypted_packets; + s->rx_tls_decrypted_bytes += rq_stats->tls_decrypted_bytes; + s->rx_tls_ctx += rq_stats->tls_ctx; + s->rx_tls_del += rq_stats->tls_del; + s->rx_tls_resync_req_pkt += rq_stats->tls_resync_req_pkt; + s->rx_tls_resync_req_start += rq_stats->tls_resync_req_start; + s->rx_tls_resync_req_end += rq_stats->tls_resync_req_end; + s->rx_tls_resync_req_skip += rq_stats->tls_resync_req_skip; + s->rx_tls_resync_res_ok += rq_stats->tls_resync_res_ok; + s->rx_tls_resync_res_skip += rq_stats->tls_resync_res_skip; + s->rx_tls_err += rq_stats->tls_err; +#endif s->ch_events += ch_stats->events; s->ch_poll += ch_stats->poll; s->ch_arm += ch_stats->arm; @@ -1475,6 +1501,19 @@ static const struct counter_desc rq_stats_desc[] = { { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, congst_umr) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, arfs_err) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, recover) }, +#ifdef CONFIG_MLX5_EN_TLS + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_decrypted_packets) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_decrypted_bytes) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_ctx) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_del) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_pkt) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_start) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_end) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_skip) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_res_ok) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_res_skip) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_err) }, +#endif }; static const struct counter_desc sq_stats_desc[] = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 2b83ba990714..2e1cca1923b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -186,6 +186,18 @@ struct mlx5e_sw_stats { u64 tx_tls_skip_no_sync_data; u64 tx_tls_drop_no_sync_data; u64 tx_tls_drop_bypass_req; + + u64 rx_tls_decrypted_packets; + u64 rx_tls_decrypted_bytes; + u64 rx_tls_ctx; + u64 rx_tls_del; + u64 rx_tls_resync_req_pkt; + u64 rx_tls_resync_req_start; + u64 rx_tls_resync_req_end; + u64 rx_tls_resync_req_skip; + u64 rx_tls_resync_res_ok; + u64 rx_tls_resync_res_skip; + u64 rx_tls_err; #endif u64 rx_xsk_packets; @@ -305,6 +317,19 @@ struct mlx5e_rq_stats { u64 congst_umr; u64 arfs_err; u64 recover; +#ifdef CONFIG_MLX5_EN_TLS + u64 tls_decrypted_packets; + u64 tls_decrypted_bytes; + u64 tls_ctx; + u64 tls_del; + u64 tls_resync_req_pkt; + u64 tls_resync_req_start; + u64 tls_resync_req_end; + u64 tls_resync_req_skip; + u64 tls_resync_res_ok; + u64 tls_resync_res_skip; + u64 tls_err; +#endif }; struct mlx5e_sq_stats { From c5607360ec4ea5d0c955de501b9eeb04fa2465d0 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Thu, 18 Jun 2020 12:45:59 +0300 Subject: [PATCH 18/20] net/mlx5e: Increase Async ICO SQ size Resync communication with HW for kTLS RX is done via the async ICOSQs. kTLS RX resync requests might come in bursts. To improve the success chances for such bursts, use a larger ICOSQ. Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 0f1578a5e538..3e6fcd545d2c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -2306,6 +2306,14 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, } } +static u8 mlx5e_build_async_icosq_log_wq_sz(struct net_device *netdev) +{ + if (netdev->hw_features & NETIF_F_HW_TLS_RX) + return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; + + return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; +} + static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_channel_param *cparam) @@ -2315,7 +2323,7 @@ static void mlx5e_build_channel_param(struct mlx5e_priv *priv, mlx5e_build_rq_param(priv, params, NULL, &cparam->rq); icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); - async_icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(priv->netdev); mlx5e_build_sq_param(priv, params, &cparam->txq_sq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); From ed9a7c53b8781f851bd8406551d6abfb2d826683 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Mon, 22 Jun 2020 18:32:36 +0300 Subject: [PATCH 19/20] net/mlx5e: kTLS, Cleanup redundant capability check All callers of mlx5e_ktls_build_netdev() check capability before the call. Remove the repeated check in the function. Signed-off-by: Tariq Toukan Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index deec17af5a69..1b392696280d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -58,9 +58,6 @@ void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; - if (!mlx5_accel_is_ktls_device(mdev)) - return; - if (mlx5_accel_is_ktls_tx(mdev)) { netdev->hw_features |= NETIF_F_HW_TLS_TX; netdev->features |= NETIF_F_HW_TLS_TX; From a29074367b347af9e19d36522f7ad9a7db4b9c28 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Mon, 15 Jun 2020 13:02:49 +0300 Subject: [PATCH 20/20] net/mlx5e: kTLS, Improve rx handler function call Prior to this patch mlx5e tls rx handler was called unconditionally on all rx frames and the decision whether a frame is a valid tls record is done inside that function. A function call can be expensive especially for regular rx packet rate. To avoid this, check the tls validity before jumping into the tls rx handler. While at it, split between kTLS device offload rx handler and FPGA tls rx handler using a similar method. Signed-off-by: Saeed Mahameed Signed-off-by: Tariq Toukan --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 1 + .../mellanox/mlx5/core/en_accel/ktls_rx.c | 10 ++------- .../mellanox/mlx5/core/en_accel/tls.h | 6 ++++++ .../mellanox/mlx5/core/en_accel/tls_rxtx.c | 12 +++-------- .../mellanox/mlx5/core/en_accel/tls_rxtx.h | 21 +++++++++++++++++-- .../net/ethernet/mellanox/mlx5/core/en_main.c | 3 +++ .../net/ethernet/mellanox/mlx5/core/en_rx.c | 2 -- 7 files changed, 34 insertions(+), 21 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index ec5bf73f9b07..2957edb7e0b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -263,6 +263,7 @@ enum { MLX5E_RQ_STATE_AM, MLX5E_RQ_STATE_NO_CSUM_COMPLETE, MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */ + MLX5E_RQ_STATE_FPGA_TLS, /* FPGA TLS enabled */ }; struct mlx5e_cq { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index e0a8f9d63b30..d7215defd403 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -517,15 +517,9 @@ void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) { - u8 tls_offload = get_cqe_tls_offload(cqe); - struct mlx5e_rq_stats *stats; + struct mlx5e_rq_stats *stats = rq->stats; - if (likely(tls_offload == CQE_TLS_OFFLOAD_NOT_DECRYPTED)) - return; - - stats = rq->stats; - - switch (tls_offload) { + switch (get_cqe_tls_offload(cqe)) { case CQE_TLS_OFFLOAD_DECRYPTED: skb->decrypted = 1; stats->tls_decrypted_packets++; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h index ca0c2ebb41a1..bd270a85c804 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h @@ -87,6 +87,11 @@ mlx5e_get_tls_rx_context(struct tls_context *tls_ctx) base); } +static inline bool mlx5e_is_tls_on(struct mlx5e_priv *priv) +{ + return priv->tls; +} + void mlx5e_tls_build_netdev(struct mlx5e_priv *priv); int mlx5e_tls_init(struct mlx5e_priv *priv); void mlx5e_tls_cleanup(struct mlx5e_priv *priv); @@ -103,6 +108,7 @@ static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) mlx5e_ktls_build_netdev(priv); } +static inline bool mlx5e_is_tls_on(struct mlx5e_priv *priv) { return false; } static inline int mlx5e_tls_init(struct mlx5e_priv *priv) { return 0; } static inline void mlx5e_tls_cleanup(struct mlx5e_priv *priv) { } static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index 182841322ce4..b0c31d49ff8d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -355,19 +355,13 @@ out: return 0; } -void mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, - struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) +/* FPGA tls rx handler */ +void mlx5e_tls_handle_rx_skb_metadata(struct mlx5e_rq *rq, struct sk_buff *skb, + u32 *cqe_bcnt) { struct mlx5e_tls_metadata *mdata; struct mlx5e_priv *priv; - if (likely(mlx5_accel_is_ktls_rx(rq->mdev))) - return mlx5e_ktls_handle_rx_skb(rq, skb, cqe, cqe_bcnt); - - /* FPGA */ - if (!is_metadata_hdr_valid(skb)) - return; - /* Use the metadata */ mdata = (struct mlx5e_tls_metadata *)(skb->data + ETH_HLEN); switch (mdata->content.recv.syndrome) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h index 8bb790674042..5f162ad2ee8f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h @@ -34,6 +34,7 @@ #ifndef __MLX5E_TLS_RXTX_H__ #define __MLX5E_TLS_RXTX_H__ +#include "accel/accel.h" #include "en_accel/ktls_txrx.h" #ifdef CONFIG_MLX5_EN_TLS @@ -49,11 +50,27 @@ bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, struct mlx5e_accel_tx_tls_state *state); -void mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, - struct mlx5_cqe64 *cqe, u32 *cqe_bcnt); +void mlx5e_tls_handle_rx_skb_metadata(struct mlx5e_rq *rq, struct sk_buff *skb, + u32 *cqe_bcnt); + +static inline void +mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) +{ + if (unlikely(get_cqe_tls_offload(cqe))) /* cqe bit indicates a TLS device */ + return mlx5e_ktls_handle_rx_skb(rq, skb, cqe, cqe_bcnt); + + if (unlikely(test_bit(MLX5E_RQ_STATE_FPGA_TLS, &rq->state) && is_metadata_hdr_valid(skb))) + return mlx5e_tls_handle_rx_skb_metadata(rq, skb, cqe_bcnt); +} #else +static inline bool +mlx5e_accel_is_tls(struct mlx5_cqe64 *cqe, struct sk_buff *skb) { return false; } +static inline void +mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) {} static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) { return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 3e6fcd545d2c..046cfb0ea180 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -873,6 +873,9 @@ int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params, if (err) goto err_destroy_rq; + if (mlx5e_is_tls_on(c->priv) && !mlx5_accel_is_ktls_device(c->mdev)) + __set_bit(MLX5E_RQ_STATE_FPGA_TLS, &c->rq.state); /* must be FPGA */ + if (MLX5_CAP_ETH(c->mdev, cqe_checksum_full)) __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &c->rq.state); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 4b7c119c8946..8b42f729a4f7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1019,9 +1019,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, skb->mac_len = ETH_HLEN; -#ifdef CONFIG_MLX5_EN_TLS mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt); -#endif if (lro_num_seg > 1) { mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);