From d14f28b8c1de668bab863bf5892a49c824cb110d Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:27 +0200 Subject: [PATCH 01/38] xfrm: add new packet offload flag In the next patches, the xfrm core code will be extended to support new type of offload - packet offload. In that mode, both policy and state should be specially configured in order to perform whole offloaded data path. Full offload takes care of encryption, decryption, encapsulation and other operations with headers. As this mode is new for XFRM policy flow, we can "start fresh" with flag bits and release first and second bit for future use. Reviewed-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- include/net/xfrm.h | 7 +++++++ include/uapi/linux/xfrm.h | 6 ++++++ net/xfrm/xfrm_device.c | 3 +++ net/xfrm/xfrm_user.c | 2 ++ 4 files changed, 18 insertions(+) diff --git a/include/net/xfrm.h b/include/net/xfrm.h index e0cc6791c001..b39d24fa2ef0 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -131,12 +131,19 @@ enum { XFRM_DEV_OFFLOAD_OUT, }; +enum { + XFRM_DEV_OFFLOAD_UNSPECIFIED, + XFRM_DEV_OFFLOAD_CRYPTO, + XFRM_DEV_OFFLOAD_PACKET, +}; + struct xfrm_dev_offload { struct net_device *dev; netdevice_tracker dev_tracker; struct net_device *real_dev; unsigned long offload_handle; u8 dir : 2; + u8 type : 2; }; struct xfrm_mode { diff --git a/include/uapi/linux/xfrm.h b/include/uapi/linux/xfrm.h index 4f84ea7ee14c..23543c33fee8 100644 --- a/include/uapi/linux/xfrm.h +++ b/include/uapi/linux/xfrm.h @@ -519,6 +519,12 @@ struct xfrm_user_offload { */ #define XFRM_OFFLOAD_IPV6 1 #define XFRM_OFFLOAD_INBOUND 2 +/* Two bits above are relevant for state path only, while + * offload is used for both policy and state flows. + * + * In policy offload mode, they are free and can be safely reused. + */ +#define XFRM_OFFLOAD_PACKET 4 struct xfrm_userpolicy_default { #define XFRM_USERPOLICY_UNSPEC 0 diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 21269e8f2db4..3b0c1ca8d4bb 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -291,12 +291,15 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, else xso->dir = XFRM_DEV_OFFLOAD_OUT; + xso->type = XFRM_DEV_OFFLOAD_CRYPTO; + err = dev->xfrmdev_ops->xdo_dev_state_add(x); if (err) { xso->dev = NULL; xso->dir = 0; xso->real_dev = NULL; netdev_put(dev, &xso->dev_tracker); + xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; if (err != -EOPNOTSUPP) { NL_SET_ERR_MSG(extack, "Device failed to offload this state"); diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 0eb4696661c8..c3b8c1532718 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -956,6 +956,8 @@ static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) xuo->ifindex = xso->dev->ifindex; if (xso->dir == XFRM_DEV_OFFLOAD_IN) xuo->flags = XFRM_OFFLOAD_INBOUND; + if (xso->type == XFRM_DEV_OFFLOAD_PACKET) + xuo->flags |= XFRM_OFFLOAD_PACKET; return 0; } From 62f6eca5de103c6823f6ca2abbf2ee242e132207 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:28 +0200 Subject: [PATCH 02/38] xfrm: allow state packet offload mode Allow users to configure xfrm states with packet offload mode. The packet mode must be requested both for policy and state, and such requires us to do not implement fallback. We explicitly return an error if requested packet mode can't be configured. Reviewed-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../inline_crypto/ch_ipsec/chcr_ipsec.c | 4 ++++ .../net/ethernet/intel/ixgbe/ixgbe_ipsec.c | 5 ++++ drivers/net/ethernet/intel/ixgbevf/ipsec.c | 5 ++++ .../mellanox/mlx5/core/en_accel/ipsec.c | 4 ++++ .../net/ethernet/netronome/nfp/crypto/ipsec.c | 5 ++++ drivers/net/netdevsim/ipsec.c | 5 ++++ net/xfrm/xfrm_device.c | 24 +++++++++++++++---- 7 files changed, 47 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 585590520076..ca21794281d6 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -283,6 +283,10 @@ static int ch_ipsec_xfrm_add_state(struct xfrm_state *x) pr_debug("Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } + if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + pr_debug("Unsupported xfrm offload\n"); + return -EINVAL; + } sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); if (!sa_entry) { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c index 774de63dd93a..53a969e34883 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c @@ -585,6 +585,11 @@ static int ixgbe_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { struct rx_sa rsa; diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index 9984ebc62d78..c1cf540d162a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -280,6 +280,11 @@ static int ixgbevf_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { struct rx_sa rsa; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 1b03ab03fc5a..e6411533f911 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -253,6 +253,10 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } + if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_info(netdev, "Unsupported xfrm offload type\n"); + return -EINVAL; + } return 0; } diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c index 3728870d8e9c..4632268695cb 100644 --- a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c +++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c @@ -302,6 +302,11 @@ static int nfp_net_xfrm_add_state(struct xfrm_state *x) return -EINVAL; } + if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + nn_err(nn, "Unsupported xfrm offload tyoe\n"); + return -EINVAL; + } + cfg->spi = ntohl(x->id.spi); /* Hash/Authentication */ diff --git a/drivers/net/netdevsim/ipsec.c b/drivers/net/netdevsim/ipsec.c index 386336a38f34..b93baf5c8bee 100644 --- a/drivers/net/netdevsim/ipsec.c +++ b/drivers/net/netdevsim/ipsec.c @@ -149,6 +149,11 @@ static int nsim_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + /* find the first unused index */ ret = nsim_ipsec_find_empty_idx(ipsec); if (ret < 0) { diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 3b0c1ca8d4bb..3184b2c394b6 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -229,6 +229,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_dev_offload *xso = &x->xso; xfrm_address_t *saddr; xfrm_address_t *daddr; + bool is_packet_offload; if (!x->type_offload) { NL_SET_ERR_MSG(extack, "Type doesn't support offload"); @@ -241,11 +242,13 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, return -EINVAL; } - if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) { + if (xuo->flags & + ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) { NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); return -EINVAL; } + is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET; dev = dev_get_by_index(net, xuo->ifindex); if (!dev) { if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { @@ -260,7 +263,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, x->props.family, xfrm_smark_get(0, x)); if (IS_ERR(dst)) - return 0; + return (is_packet_offload) ? -EINVAL : 0; dev = dst->dev; @@ -271,7 +274,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) { xso->dev = NULL; dev_put(dev); - return 0; + return (is_packet_offload) ? -EINVAL : 0; } if (x->props.flags & XFRM_STATE_ESN && @@ -291,7 +294,10 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, else xso->dir = XFRM_DEV_OFFLOAD_OUT; - xso->type = XFRM_DEV_OFFLOAD_CRYPTO; + if (is_packet_offload) + xso->type = XFRM_DEV_OFFLOAD_PACKET; + else + xso->type = XFRM_DEV_OFFLOAD_CRYPTO; err = dev->xfrmdev_ops->xdo_dev_state_add(x); if (err) { @@ -301,7 +307,15 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, netdev_put(dev, &xso->dev_tracker); xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; - if (err != -EOPNOTSUPP) { + /* User explicitly requested packet offload mode and configured + * policy in addition to the XFRM state. So be civil to users, + * and return an error instead of taking fallback path. + * + * This WARN_ON() can be seen as a documentation for driver + * authors to do not return -EOPNOTSUPP in packet offload mode. + */ + WARN_ON(err == -EOPNOTSUPP && is_packet_offload); + if (err != -EOPNOTSUPP || is_packet_offload) { NL_SET_ERR_MSG(extack, "Device failed to offload this state"); return err; } From 919e43fad5163a8ceb39826ecdee897a9f799351 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:29 +0200 Subject: [PATCH 03/38] xfrm: add an interface to offload policy Extend netlink interface to add and delete XFRM policy from the device. This functionality is a first step to implement packet IPsec offload solution. Signed-off-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- include/linux/netdevice.h | 3 ++ include/net/xfrm.h | 45 +++++++++++++++++++++++++ net/xfrm/xfrm_device.c | 67 ++++++++++++++++++++++++++++++++++++- net/xfrm/xfrm_policy.c | 69 +++++++++++++++++++++++++++++++++++++++ net/xfrm/xfrm_user.c | 18 ++++++++++ 5 files changed, 201 insertions(+), 1 deletion(-) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5aa35c58c342..4096e3fe8e4a 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1040,6 +1040,9 @@ struct xfrmdev_ops { bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + int (*xdo_dev_policy_add) (struct xfrm_policy *x); + void (*xdo_dev_policy_delete) (struct xfrm_policy *x); + void (*xdo_dev_policy_free) (struct xfrm_policy *x); }; #endif diff --git a/include/net/xfrm.h b/include/net/xfrm.h index b39d24fa2ef0..6fea34cbdf48 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -129,6 +129,7 @@ struct xfrm_state_walk { enum { XFRM_DEV_OFFLOAD_IN = 1, XFRM_DEV_OFFLOAD_OUT, + XFRM_DEV_OFFLOAD_FWD, }; enum { @@ -541,6 +542,8 @@ struct xfrm_policy { struct xfrm_tmpl xfrm_vec[XFRM_MAX_DEPTH]; struct hlist_node bydst_inexact_list; struct rcu_head rcu; + + struct xfrm_dev_offload xdo; }; static inline struct net *xp_net(const struct xfrm_policy *xp) @@ -1585,6 +1588,8 @@ struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq); int xfrm_state_delete(struct xfrm_state *x); int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync); int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid); +int xfrm_dev_policy_flush(struct net *net, struct net_device *dev, + bool task_valid); void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si); void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si); u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq); @@ -1899,6 +1904,9 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack); +int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, + struct xfrm_user_offload *xuo, u8 dir, + struct netlink_ext_ack *extack); bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x); static inline void xfrm_dev_state_advance_esn(struct xfrm_state *x) @@ -1947,6 +1955,28 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) netdev_put(dev, &xso->dev_tracker); } } + +static inline void xfrm_dev_policy_delete(struct xfrm_policy *x) +{ + struct xfrm_dev_offload *xdo = &x->xdo; + struct net_device *dev = xdo->dev; + + if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_policy_delete) + dev->xfrmdev_ops->xdo_dev_policy_delete(x); +} + +static inline void xfrm_dev_policy_free(struct xfrm_policy *x) +{ + struct xfrm_dev_offload *xdo = &x->xdo; + struct net_device *dev = xdo->dev; + + if (dev && dev->xfrmdev_ops) { + if (dev->xfrmdev_ops->xdo_dev_policy_free) + dev->xfrmdev_ops->xdo_dev_policy_free(x); + xdo->dev = NULL; + netdev_put(dev, &xdo->dev_tracker); + } +} #else static inline void xfrm_dev_resume(struct sk_buff *skb) { @@ -1974,6 +2004,21 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) { } +static inline int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, + struct xfrm_user_offload *xuo, u8 dir, + struct netlink_ext_ack *extack) +{ + return 0; +} + +static inline void xfrm_dev_policy_delete(struct xfrm_policy *x) +{ +} + +static inline void xfrm_dev_policy_free(struct xfrm_policy *x) +{ +} + static inline bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) { return false; diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 3184b2c394b6..04ae510dcc66 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -325,6 +325,69 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, } EXPORT_SYMBOL_GPL(xfrm_dev_state_add); +int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, + struct xfrm_user_offload *xuo, u8 dir, + struct netlink_ext_ack *extack) +{ + struct xfrm_dev_offload *xdo = &xp->xdo; + struct net_device *dev; + int err; + + if (!xuo->flags || xuo->flags & ~XFRM_OFFLOAD_PACKET) { + /* We support only packet offload mode and it means + * that user must set XFRM_OFFLOAD_PACKET bit. + */ + NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); + return -EINVAL; + } + + dev = dev_get_by_index(net, xuo->ifindex); + if (!dev) + return -EINVAL; + + if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_policy_add) { + xdo->dev = NULL; + dev_put(dev); + NL_SET_ERR_MSG(extack, "Policy offload is not supported"); + return -EINVAL; + } + + xdo->dev = dev; + netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC); + xdo->real_dev = dev; + xdo->type = XFRM_DEV_OFFLOAD_PACKET; + switch (dir) { + case XFRM_POLICY_IN: + xdo->dir = XFRM_DEV_OFFLOAD_IN; + break; + case XFRM_POLICY_OUT: + xdo->dir = XFRM_DEV_OFFLOAD_OUT; + break; + case XFRM_POLICY_FWD: + xdo->dir = XFRM_DEV_OFFLOAD_FWD; + break; + default: + xdo->dev = NULL; + dev_put(dev); + NL_SET_ERR_MSG(extack, "Unrecognized oflload direction"); + return -EINVAL; + } + + err = dev->xfrmdev_ops->xdo_dev_policy_add(xp); + if (err) { + xdo->dev = NULL; + xdo->real_dev = NULL; + xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + xdo->dir = 0; + netdev_put(dev, &xdo->dev_tracker); + NL_SET_ERR_MSG(extack, "Device failed to offload this policy"); + return err; + } + + return 0; +} +EXPORT_SYMBOL_GPL(xfrm_dev_policy_add); + bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) { int mtu; @@ -427,8 +490,10 @@ static int xfrm_api_check(struct net_device *dev) static int xfrm_dev_down(struct net_device *dev) { - if (dev->features & NETIF_F_HW_ESP) + if (dev->features & NETIF_F_HW_ESP) { xfrm_dev_state_flush(dev_net(dev), dev, true); + xfrm_dev_policy_flush(dev_net(dev), dev, true); + } return NOTIFY_DONE; } diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 9b9e2765363d..8b8760907563 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -425,6 +425,7 @@ void xfrm_policy_destroy(struct xfrm_policy *policy) if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) BUG(); + xfrm_dev_policy_free(policy); call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); } EXPORT_SYMBOL(xfrm_policy_destroy); @@ -1769,12 +1770,41 @@ xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) } return err; } + +static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, + struct net_device *dev, + bool task_valid) +{ + struct xfrm_policy *pol; + int err = 0; + + list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { + if (pol->walk.dead || + xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || + pol->xdo.dev != dev) + continue; + + err = security_xfrm_policy_delete(pol->security); + if (err) { + xfrm_audit_policy_delete(pol, 0, task_valid); + return err; + } + } + return err; +} #else static inline int xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) { return 0; } + +static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, + struct net_device *dev, + bool task_valid) +{ + return 0; +} #endif int xfrm_policy_flush(struct net *net, u8 type, bool task_valid) @@ -1814,6 +1844,44 @@ out: } EXPORT_SYMBOL(xfrm_policy_flush); +int xfrm_dev_policy_flush(struct net *net, struct net_device *dev, + bool task_valid) +{ + int dir, err = 0, cnt = 0; + struct xfrm_policy *pol; + + spin_lock_bh(&net->xfrm.xfrm_policy_lock); + + err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid); + if (err) + goto out; + +again: + list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { + dir = xfrm_policy_id2dir(pol->index); + if (pol->walk.dead || + dir >= XFRM_POLICY_MAX || + pol->xdo.dev != dev) + continue; + + __xfrm_policy_unlink(pol, dir); + spin_unlock_bh(&net->xfrm.xfrm_policy_lock); + cnt++; + xfrm_audit_policy_delete(pol, 1, task_valid); + xfrm_policy_kill(pol); + spin_lock_bh(&net->xfrm.xfrm_policy_lock); + goto again; + } + if (cnt) + __xfrm_policy_inexact_flush(net); + else + err = -ESRCH; +out: + spin_unlock_bh(&net->xfrm.xfrm_policy_lock); + return err; +} +EXPORT_SYMBOL(xfrm_dev_policy_flush); + int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk, int (*func)(struct xfrm_policy *, int, int, void*), void *data) @@ -2245,6 +2313,7 @@ int xfrm_policy_delete(struct xfrm_policy *pol, int dir) pol = __xfrm_policy_unlink(pol, dir); spin_unlock_bh(&net->xfrm.xfrm_policy_lock); if (pol) { + xfrm_dev_policy_delete(pol); xfrm_policy_kill(pol); return 0; } diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index c3b8c1532718..cf5172d4ce68 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -1892,6 +1892,15 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, if (attrs[XFRMA_IF_ID]) xp->if_id = nla_get_u32(attrs[XFRMA_IF_ID]); + /* configure the hardware if offload is requested */ + if (attrs[XFRMA_OFFLOAD_DEV]) { + err = xfrm_dev_policy_add(net, xp, + nla_data(attrs[XFRMA_OFFLOAD_DEV]), + p->dir, extack); + if (err) + goto error; + } + return xp; error: *errp = err; @@ -1931,6 +1940,7 @@ static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh, xfrm_audit_policy_add(xp, err ? 0 : 1, true); if (err) { + xfrm_dev_policy_delete(xp); security_xfrm_policy_free(xp->security); kfree(xp); return err; @@ -2043,6 +2053,8 @@ static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) { nlmsg_cancel(skb, nlh); return err; @@ -3381,6 +3393,8 @@ static int build_acquire(struct sk_buff *skb, struct xfrm_state *x, err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) { nlmsg_cancel(skb, nlh); return err; @@ -3499,6 +3513,8 @@ static int build_polexpire(struct sk_buff *skb, struct xfrm_policy *xp, err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) { nlmsg_cancel(skb, nlh); return err; @@ -3582,6 +3598,8 @@ static int xfrm_notify_policy(struct xfrm_policy *xp, int dir, const struct km_e err = xfrm_mark_put(skb, &xp->mark); if (!err) err = xfrm_if_id_put(skb, xp->if_id); + if (!err && xp->xdo.dev) + err = copy_user_offload(&xp->xdo, skb); if (err) goto out_free_skb; From f8a70afafc1759b1fca4baaa891625dde49c10b7 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:30 +0200 Subject: [PATCH 04/38] xfrm: add TX datapath support for IPsec packet offload mode In IPsec packet mode, the device is going to encrypt and encapsulate packets that are associated with offloaded policy. After successful policy lookup to indicate if packets should be offloaded or not, the stack forwards packets to the device to do the magic. Signed-off-by: Raed Salem Signed-off-by: Huy Nguyen Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- net/xfrm/xfrm_device.c | 15 +++++- net/xfrm/xfrm_output.c | 12 ++++- net/xfrm/xfrm_state.c | 119 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 5 deletions(-) diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 04ae510dcc66..3e9e874522a8 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -132,6 +132,16 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN) return skb; + /* The packet was sent to HW IPsec packet offload engine, + * but to wrong device. Drop the packet, so it won't skip + * XFRM stack. + */ + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) { + kfree_skb(skb); + dev_core_stats_tx_dropped_inc(dev); + return NULL; + } + /* This skb was already validated on the upper/virtual dev */ if ((x->xso.dev != dev) && (x->xso.real_dev == dev)) return skb; @@ -398,8 +408,9 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) if (!x->type_offload || x->encap) return false; - if ((!dev || (dev == xfrm_dst_path(dst)->dev)) && - (!xdst->child->xfrm)) { + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET || + ((!dev || (dev == xfrm_dst_path(dst)->dev)) && + !xdst->child->xfrm)) { mtu = xfrm_state_mtu(x, xdst->child_mtu_cached); if (skb->len <= mtu) goto ok; diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index 78cb8d0a6a18..ff114d68cc43 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -492,7 +492,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err) struct xfrm_state *x = dst->xfrm; struct net *net = xs_net(x); - if (err <= 0) + if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET) goto resume; do { @@ -717,6 +717,16 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb) break; } + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) { + if (!xfrm_dev_offload_ok(skb, x)) { + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); + kfree_skb(skb); + return -EHOSTUNREACH; + } + + return xfrm_output_resume(sk, skb, 0); + } + secpath_reset(skb); if (xfrm_dev_offload_ok(skb, x)) { diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 9ec481fbfb63..4d315e1a88fa 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -951,6 +951,49 @@ xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl, x->props.family = tmpl->encap_family; } +static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark, + const xfrm_address_t *daddr, + __be32 spi, u8 proto, + unsigned short family, + struct xfrm_dev_offload *xdo) +{ + unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); + struct xfrm_state *x; + + hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { +#ifdef CONFIG_XFRM_OFFLOAD + if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + /* HW states are in the head of list, there is + * no need to iterate further. + */ + break; + + /* Packet offload: both policy and SA should + * have same device. + */ + if (xdo->dev != x->xso.dev) + continue; + } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + /* Skip HW policy for SW lookups */ + continue; +#endif + if (x->props.family != family || + x->id.spi != spi || + x->id.proto != proto || + !xfrm_addr_equal(&x->id.daddr, daddr, family)) + continue; + + if ((mark & x->mark.m) != x->mark.v) + continue; + if (!xfrm_state_hold_rcu(x)) + continue; + return x; + } + + return NULL; +} + static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi, u8 proto, @@ -1092,6 +1135,23 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, rcu_read_lock(); h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) { +#ifdef CONFIG_XFRM_OFFLOAD + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + /* HW states are in the head of list, there is + * no need to iterate further. + */ + break; + + /* Packet offload: both policy and SA should + * have same device. + */ + if (pol->xdo.dev != x->xso.dev) + continue; + } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + /* Skip HW policy for SW lookups */ + continue; +#endif if (x->props.family == encap_family && x->props.reqid == tmpl->reqid && (mark & x->mark.m) == x->mark.v && @@ -1109,6 +1169,23 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family); hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) { +#ifdef CONFIG_XFRM_OFFLOAD + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + /* HW states are in the head of list, there is + * no need to iterate further. + */ + break; + + /* Packet offload: both policy and SA should + * have same device. + */ + if (pol->xdo.dev != x->xso.dev) + continue; + } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + /* Skip HW policy for SW lookups */ + continue; +#endif if (x->props.family == encap_family && x->props.reqid == tmpl->reqid && (mark & x->mark.m) == x->mark.v && @@ -1126,8 +1203,10 @@ found: x = best; if (!x && !error && !acquire_in_progress) { if (tmpl->id.spi && - (x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi, - tmpl->id.proto, encap_family)) != NULL) { + (x0 = __xfrm_state_lookup_all(net, mark, daddr, + tmpl->id.spi, tmpl->id.proto, + encap_family, + &pol->xdo)) != NULL) { to_put = x0; error = -EEXIST; goto out; @@ -1161,7 +1240,31 @@ found: x = NULL; goto out; } +#ifdef CONFIG_XFRM_OFFLOAD + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { + struct xfrm_dev_offload *xdo = &pol->xdo; + struct xfrm_dev_offload *xso = &x->xso; + xso->type = XFRM_DEV_OFFLOAD_PACKET; + xso->dir = xdo->dir; + xso->dev = xdo->dev; + xso->real_dev = xdo->real_dev; + netdev_tracker_alloc(xso->dev, &xso->dev_tracker, + GFP_ATOMIC); + error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x); + if (error) { + xso->dir = 0; + netdev_put(xso->dev, &xso->dev_tracker); + xso->dev = NULL; + xso->real_dev = NULL; + xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + x->km.state = XFRM_STATE_DEAD; + to_put = x; + x = NULL; + goto out; + } + } +#endif if (km_query(x, tmpl, pol) == 0) { spin_lock_bh(&net->xfrm.xfrm_state_lock); x->km.state = XFRM_STATE_ACQ; @@ -1185,6 +1288,18 @@ found: xfrm_hash_grow_check(net, x->bydst.next != NULL); spin_unlock_bh(&net->xfrm.xfrm_state_lock); } else { +#ifdef CONFIG_XFRM_OFFLOAD + struct xfrm_dev_offload *xso = &x->xso; + + if (xso->type == XFRM_DEV_OFFLOAD_PACKET) { + xso->dev->xfrmdev_ops->xdo_dev_state_delete(x); + xso->dir = 0; + netdev_put(xso->dev, &xso->dev_tracker); + xso->dev = NULL; + xso->real_dev = NULL; + xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; + } +#endif x->km.state = XFRM_STATE_DEAD; to_put = x; x = NULL; From 5958372ddf628fe6f4c3e49425734ad32fcfb13c Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:31 +0200 Subject: [PATCH 05/38] xfrm: add RX datapath protection for IPsec packet offload mode Traffic received by device with enabled IPsec packet offload should be forwarded to the stack only after decryption, packet headers and trailers removed. Such packets are expected to be seen as normal (non-XFRM) ones, while not-supported packets should be dropped by the HW. Reviewed-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- include/net/xfrm.h | 55 +++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 6fea34cbdf48..b6ee14991a32 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -1102,6 +1102,29 @@ xfrm_state_addr_cmp(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, un return !0; } +#ifdef CONFIG_XFRM +static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb) +{ + struct sec_path *sp = skb_sec_path(skb); + + return sp->xvec[sp->len - 1]; +} +#endif + +static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb) +{ +#ifdef CONFIG_XFRM + struct sec_path *sp = skb_sec_path(skb); + + if (!sp || !sp->olen || sp->len != sp->olen) + return NULL; + + return &sp->ovec[sp->olen - 1]; +#else + return NULL; +#endif +} + #ifdef CONFIG_XFRM int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb, unsigned short family); @@ -1133,10 +1156,19 @@ static inline int __xfrm_policy_check2(struct sock *sk, int dir, { struct net *net = dev_net(skb->dev); int ndir = dir | (reverse ? XFRM_POLICY_MASK + 1 : 0); + struct xfrm_offload *xo = xfrm_offload(skb); + struct xfrm_state *x; if (sk && sk->sk_policy[XFRM_POLICY_IN]) return __xfrm_policy_check(sk, ndir, skb, family); + if (xo) { + x = xfrm_input_state(skb); + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) + return (xo->flags & CRYPTO_DONE) && + (xo->status & CRYPTO_SUCCESS); + } + return __xfrm_check_nopolicy(net, skb, dir) || __xfrm_check_dev_nopolicy(skb, dir, family) || __xfrm_policy_check(sk, ndir, skb, family); @@ -1872,29 +1904,6 @@ static inline void xfrm_states_delete(struct xfrm_state **states, int n) } #endif -#ifdef CONFIG_XFRM -static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb) -{ - struct sec_path *sp = skb_sec_path(skb); - - return sp->xvec[sp->len - 1]; -} -#endif - -static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb) -{ -#ifdef CONFIG_XFRM - struct sec_path *sp = skb_sec_path(skb); - - if (!sp || !sp->olen || sp->len != sp->olen) - return NULL; - - return &sp->ovec[sp->olen - 1]; -#else - return NULL; -#endif -} - void __init xfrm_dev_init(void); #ifdef CONFIG_XFRM_OFFLOAD From 3c611d40c6923c81e6a83a67156cd30a9503c155 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:32 +0200 Subject: [PATCH 06/38] xfrm: speed-up lookup of HW policies Devices that implement IPsec packet offload mode should offload SA and policies too. In RX path, it causes to the situation that HW will always have higher priority over any SW policies. It means that we don't need to perform any search of inexact policies and/or priority checks if HW policy was discovered. In such situation, the HW will catch the packets anyway and HW can still implement inexact lookups. In case specific policy is not found, we will continue with packet lookup and check for existence of HW policies in inexact list. HW policies are added to the head of SPD to ensure fast lookup, as XFRM iterates over all policies in the loop. The same solution of adding HW SAs at the begging of the list is applied to SA database too. However, we don't need to change lookups as they are sorted by insertion order and not priority. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- net/xfrm/xfrm_policy.c | 16 ++++++---- net/xfrm/xfrm_state.c | 66 ++++++++++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 8b8760907563..e9eb82c5457d 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -536,7 +536,7 @@ redo: __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, pol->family, nhashmask, dbits, sbits); - if (!entry0) { + if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { hlist_del_rcu(&pol->bydst); hlist_add_head_rcu(&pol->bydst, ndsttable + h); h0 = h; @@ -867,7 +867,7 @@ static void xfrm_policy_inexact_list_reinsert(struct net *net, break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst, newpos); else hlist_add_head_rcu(&policy->bydst, &n->hhead); @@ -1348,7 +1348,7 @@ static void xfrm_hash_rebuild(struct work_struct *work) else break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst, newpos); else hlist_add_head_rcu(&policy->bydst, chain); @@ -1525,7 +1525,7 @@ static void xfrm_policy_insert_inexact_list(struct hlist_head *chain, break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos); else hlist_add_head_rcu(&policy->bydst_inexact_list, chain); @@ -1562,9 +1562,12 @@ static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain, break; } - if (newpos) + if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); else + /* Packet offload policies enter to the head + * to speed-up lookups. + */ hlist_add_head_rcu(&policy->bydst, chain); return delpol; @@ -2181,6 +2184,9 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, break; } } + if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) + goto skip_inexact; + bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id); if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr, daddr)) diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 4d315e1a88fa..2a190e85da80 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -84,6 +84,25 @@ static unsigned int xfrm_seq_hash(struct net *net, u32 seq) return __xfrm_seq_hash(seq, net->xfrm.state_hmask); } +#define XFRM_STATE_INSERT(by, _n, _h, _type) \ + { \ + struct xfrm_state *_x = NULL; \ + \ + if (_type != XFRM_DEV_OFFLOAD_PACKET) { \ + hlist_for_each_entry_rcu(_x, _h, by) { \ + if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ + continue; \ + break; \ + } \ + } \ + \ + if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ + /* SAD is empty or consist from HW SAs only */ \ + hlist_add_head_rcu(_n, _h); \ + else \ + hlist_add_before_rcu(_n, &_x->by); \ + } + static void xfrm_hash_transfer(struct hlist_head *list, struct hlist_head *ndsttable, struct hlist_head *nsrctable, @@ -100,23 +119,25 @@ static void xfrm_hash_transfer(struct hlist_head *list, h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr, x->props.reqid, x->props.family, nhashmask); - hlist_add_head_rcu(&x->bydst, ndsttable + h); + XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type); h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr, x->props.family, nhashmask); - hlist_add_head_rcu(&x->bysrc, nsrctable + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type); if (x->id.spi) { h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto, x->props.family, nhashmask); - hlist_add_head_rcu(&x->byspi, nspitable + h); + XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h, + x->xso.type); } if (x->km.seq) { h = __xfrm_seq_hash(x->km.seq, nhashmask); - hlist_add_head_rcu(&x->byseq, nseqtable + h); + XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h, + x->xso.type); } } } @@ -1269,16 +1290,24 @@ found: spin_lock_bh(&net->xfrm.xfrm_state_lock); x->km.state = XFRM_STATE_ACQ; list_add(&x->km.all, &net->xfrm.state_all); - hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); + XFRM_STATE_INSERT(bydst, &x->bydst, + net->xfrm.state_bydst + h, + x->xso.type); h = xfrm_src_hash(net, daddr, saddr, encap_family); - hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, + net->xfrm.state_bysrc + h, + x->xso.type); if (x->id.spi) { h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); - hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); + XFRM_STATE_INSERT(byspi, &x->byspi, + net->xfrm.state_byspi + h, + x->xso.type); } if (x->km.seq) { h = xfrm_seq_hash(net, x->km.seq); - hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h); + XFRM_STATE_INSERT(byseq, &x->byseq, + net->xfrm.state_byseq + h, + x->xso.type); } x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; hrtimer_start(&x->mtimer, @@ -1395,22 +1424,26 @@ static void __xfrm_state_insert(struct xfrm_state *x) h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, x->props.reqid, x->props.family); - hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); + XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, + x->xso.type); h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); - hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, + x->xso.type); if (x->id.spi) { h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); - hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); + XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, + x->xso.type); } if (x->km.seq) { h = xfrm_seq_hash(net, x->km.seq); - hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h); + XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h, + x->xso.type); } hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); @@ -1524,9 +1557,11 @@ static struct xfrm_state *__find_acq_core(struct net *net, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL_SOFT); list_add(&x->km.all, &net->xfrm.state_all); - hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); + XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, + x->xso.type); h = xfrm_src_hash(net, daddr, saddr, family); - hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); + XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, + x->xso.type); net->xfrm.state_num++; @@ -2209,7 +2244,8 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, spin_lock_bh(&net->xfrm.xfrm_state_lock); x->id.spi = newspi; h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); - hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); + XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, + x->xso.type); spin_unlock_bh(&net->xfrm.xfrm_state_lock); err = 0; From f3da86dc2c8c9004445cfbb15ac086773622d853 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:33 +0200 Subject: [PATCH 07/38] xfrm: add support to HW update soft and hard limits Both in RX and TX, the traffic that performs IPsec packet offload transformation is accounted by HW. It is needed to properly handle hard limits that require to drop the packet. It means that XFRM core needs to update internal counters with the one that accounted by the HW, so new callbacks are introduced in this patch. In case of soft or hard limit is occurred, the driver should call to xfrm_state_check_expire() that will perform key rekeying exactly as done by XFRM core. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- include/linux/netdevice.h | 1 + include/net/xfrm.h | 17 +++++++++++++++++ net/xfrm/xfrm_state.c | 4 ++++ 3 files changed, 22 insertions(+) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 4096e3fe8e4a..29ae964e3b89 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1040,6 +1040,7 @@ struct xfrmdev_ops { bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + void (*xdo_dev_state_update_curlft) (struct xfrm_state *x); int (*xdo_dev_policy_add) (struct xfrm_policy *x); void (*xdo_dev_policy_delete) (struct xfrm_policy *x); void (*xdo_dev_policy_free) (struct xfrm_policy *x); diff --git a/include/net/xfrm.h b/include/net/xfrm.h index b6ee14991a32..5413cdd5ad62 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -1571,6 +1571,23 @@ struct xfrm_state *xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, unsigned short family); int xfrm_state_check_expire(struct xfrm_state *x); +#ifdef CONFIG_XFRM_OFFLOAD +static inline void xfrm_dev_state_update_curlft(struct xfrm_state *x) +{ + struct xfrm_dev_offload *xdo = &x->xso; + struct net_device *dev = xdo->dev; + + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + return; + + if (dev && dev->xfrmdev_ops && + dev->xfrmdev_ops->xdo_dev_state_update_curlft) + dev->xfrmdev_ops->xdo_dev_state_update_curlft(x); + +} +#else +static inline void xfrm_dev_state_update_curlft(struct xfrm_state *x) {} +#endif void xfrm_state_insert(struct xfrm_state *x); int xfrm_state_add(struct xfrm_state *x); int xfrm_state_update(struct xfrm_state *x); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 2a190e85da80..cc1d0ea42672 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -570,6 +570,8 @@ static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) int err = 0; spin_lock(&x->lock); + xfrm_dev_state_update_curlft(x); + if (x->km.state == XFRM_STATE_DEAD) goto out; if (x->km.state == XFRM_STATE_EXPIRED) @@ -1936,6 +1938,8 @@ EXPORT_SYMBOL(xfrm_state_update); int xfrm_state_check_expire(struct xfrm_state *x) { + xfrm_dev_state_update_curlft(x); + if (!x->curlft.use_time) x->curlft.use_time = ktime_get_real_seconds(); From 2b7c72e0e5f98696f58490c7e4583412ad8be8f6 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 20:41:34 +0200 Subject: [PATCH 08/38] xfrm: document IPsec packet offload mode Extend XFRM device offload API description with newly added packet offload mode. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- Documentation/networking/xfrm_device.rst | 62 ++++++++++++++++++++---- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/Documentation/networking/xfrm_device.rst b/Documentation/networking/xfrm_device.rst index 01391dfd37d9..c43ace79e320 100644 --- a/Documentation/networking/xfrm_device.rst +++ b/Documentation/networking/xfrm_device.rst @@ -5,6 +5,7 @@ XFRM device - offloading the IPsec computations =============================================== Shannon Nelson +Leon Romanovsky Overview @@ -18,10 +19,21 @@ can radically increase throughput and decrease CPU utilization. The XFRM Device interface allows NIC drivers to offer to the stack access to the hardware offload. +Right now, there are two types of hardware offload that kernel supports. + * IPsec crypto offload: + * NIC performs encrypt/decrypt + * Kernel does everything else + * IPsec packet offload: + * NIC performs encrypt/decrypt + * NIC does encapsulation + * Kernel and NIC have SA and policy in-sync + * NIC handles the SA and policies states + * The Kernel talks to the keymanager + Userland access to the offload is typically through a system such as libreswan or KAME/raccoon, but the iproute2 'ip xfrm' command set can be handy when experimenting. An example command might look something -like this:: +like this for crypto offload: ip x s add proto esp dst 14.0.0.70 src 14.0.0.52 spi 0x07 mode transport \ reqid 0x07 replay-window 32 \ @@ -29,6 +41,17 @@ like this:: sel src 14.0.0.52/24 dst 14.0.0.70/24 proto tcp \ offload dev eth4 dir in +and for packet offload + + ip x s add proto esp dst 14.0.0.70 src 14.0.0.52 spi 0x07 mode transport \ + reqid 0x07 replay-window 32 \ + aead 'rfc4106(gcm(aes))' 0x44434241343332312423222114131211f4f3f2f1 128 \ + sel src 14.0.0.52/24 dst 14.0.0.70/24 proto tcp \ + offload packet dev eth4 dir in + + ip x p add src 14.0.0.70 dst 14.0.0.52 offload packet dev eth4 dir in + tmpl src 14.0.0.70 dst 14.0.0.52 proto esp reqid 10000 mode transport + Yes, that's ugly, but that's what shell scripts and/or libreswan are for. @@ -40,17 +63,24 @@ Callbacks to implement /* from include/linux/netdevice.h */ struct xfrmdev_ops { + /* Crypto and Packet offload callbacks */ int (*xdo_dev_state_add) (struct xfrm_state *x); void (*xdo_dev_state_delete) (struct xfrm_state *x); void (*xdo_dev_state_free) (struct xfrm_state *x); bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + + /* Solely packet offload callbacks */ + void (*xdo_dev_state_update_curlft) (struct xfrm_state *x); + int (*xdo_dev_policy_add) (struct xfrm_policy *x); + void (*xdo_dev_policy_delete) (struct xfrm_policy *x); + void (*xdo_dev_policy_free) (struct xfrm_policy *x); }; -The NIC driver offering ipsec offload will need to implement these -callbacks to make the offload available to the network stack's -XFRM subsystem. Additionally, the feature bits NETIF_F_HW_ESP and +The NIC driver offering ipsec offload will need to implement callbacks +relevant to supported offload to make the offload available to the network +stack's XFRM subsystem. Additionally, the feature bits NETIF_F_HW_ESP and NETIF_F_HW_ESP_TX_CSUM will signal the availability of the offload. @@ -79,7 +109,8 @@ and an indication of whether it is for Rx or Tx. The driver should =========== =================================== 0 success - -EOPNETSUPP offload not supported, try SW IPsec + -EOPNETSUPP offload not supported, try SW IPsec, + not applicable for packet offload mode other fail the request =========== =================================== @@ -96,6 +127,7 @@ will serviceable. This can check the packet information to be sure the offload can be supported (e.g. IPv4 or IPv6, no IPv4 options, etc) and return true of false to signify its support. +Crypto offload mode: When ready to send, the driver needs to inspect the Tx packet for the offload information, including the opaque context, and set up the packet send accordingly:: @@ -139,13 +171,25 @@ the stack in xfrm_input(). In ESN mode, xdo_dev_state_advance_esn() is called from xfrm_replay_advance_esn(). Driver will check packet seq number and update HW ESN state machine if needed. +Packet offload mode: +HW adds and deletes XFRM headers. So in RX path, XFRM stack is bypassed if HW +reported success. In TX path, the packet lefts kernel without extra header +and not encrypted, the HW is responsible to perform it. + When the SA is removed by the user, the driver's xdo_dev_state_delete() -is asked to disable the offload. Later, xdo_dev_state_free() is called -from a garbage collection routine after all reference counts to the state +and xdo_dev_policy_delete() are asked to disable the offload. Later, +xdo_dev_state_free() and xdo_dev_policy_free() are called from a garbage +collection routine after all reference counts to the state and policy have been removed and any remaining resources can be cleared for the offload state. How these are used by the driver will depend on specific hardware needs. As a netdev is set to DOWN the XFRM stack's netdev listener will call -xdo_dev_state_delete() and xdo_dev_state_free() on any remaining offloaded -states. +xdo_dev_state_delete(), xdo_dev_policy_delete(), xdo_dev_state_free() and +xdo_dev_policy_free() on any remaining offloaded states. + +Outcome of HW handling packets, the XFRM core can't count hard, soft limits. +The HW/driver are responsible to perform it and provide accurate data when +xdo_dev_state_update_curlft() is called. In case of one of these limits +occuried, the driver needs to call to xfrm_state_check_expire() to make sure +that XFRM performs rekeying sequence. From e77bbde73ed0a2d5342a98fb050287cdbe5db524 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:22 +0200 Subject: [PATCH 09/38] net/mlx5: Return ready to use ASO WQE There is no need in hiding returned ASO WQE type by providing void*, use the real type instead. Do it together with zeroing that memory, so ASO WQE will be ready to use immediately. Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c | 7 +++++-- drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index be74e1403328..25cd449e8aad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -162,7 +162,6 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); aso_ctrl = &aso_wqe->aso_ctrl; - memset(aso_ctrl, 0, sizeof(*aso_ctrl)); aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE << 4; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c index 0f9e4f01c85a..5a80fb7dbbca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -353,12 +353,15 @@ void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, cseg->general_id = cpu_to_be32(obj_id); } -void *mlx5_aso_get_wqe(struct mlx5_aso *aso) +struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso) { + struct mlx5_aso_wqe *wqe; u16 pi; pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc); - return mlx5_wq_cyc_get_wqe(&aso->wq, pi); + wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi); + memset(wqe, 0, sizeof(*wqe)); + return wqe; } void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h index 2d40dcf9d42e..4312614bf3bc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -77,7 +77,7 @@ enum { struct mlx5_aso; -void *mlx5_aso_get_wqe(struct mlx5_aso *aso); +struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso); void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, struct mlx5_aso_wqe *aso_wqe, u32 obj_id, u32 opc_mode); From 3afee4ed336ed7a6cff78d23339879ffb654e02e Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:23 +0200 Subject: [PATCH 10/38] net/mlx5: Add HW definitions for IPsec packet offload Add all needed bits to support IPsec packet offload mode. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../net/ethernet/mellanox/mlx5/core/lib/aso.h | 1 + include/linux/mlx5/mlx5_ifc.h | 53 +++++++++++++++++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h index 4312614bf3bc..c8fc3c838642 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -71,6 +71,7 @@ enum { }; enum { + MLX5_ACCESS_ASO_OPC_MOD_IPSEC = 0x0, MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2, MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5, }; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 5a4e914e2a6f..300b56ea5ff4 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -445,7 +445,10 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 max_modify_header_actions[0x8]; u8 max_ft_level[0x8]; - u8 reserved_at_40[0x6]; + u8 reformat_add_esp_trasport[0x1]; + u8 reserved_at_41[0x2]; + u8 reformat_del_esp_trasport[0x1]; + u8 reserved_at_44[0x2]; u8 execute_aso[0x1]; u8 reserved_at_47[0x19]; @@ -638,8 +641,10 @@ struct mlx5_ifc_fte_match_set_misc2_bits { u8 reserved_at_1a0[0x8]; u8 macsec_syndrome[0x8]; + u8 ipsec_syndrome[0x8]; + u8 reserved_at_1b8[0x8]; - u8 reserved_at_1b0[0x50]; + u8 reserved_at_1c0[0x40]; }; struct mlx5_ifc_fte_match_set_misc3_bits { @@ -6384,6 +6389,9 @@ enum mlx5_reformat_ctx_type { MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x2, MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x3, MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4, + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4 = 0x5, + MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT = 0x8, + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6 = 0xb, MLX5_REFORMAT_TYPE_INSERT_HDR = 0xf, MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10, MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11, @@ -11563,6 +11571,41 @@ enum { MLX5_IPSEC_OBJECT_ICV_LEN_16B, }; +enum { + MLX5_IPSEC_ASO_REG_C_0_1 = 0x0, + MLX5_IPSEC_ASO_REG_C_2_3 = 0x1, + MLX5_IPSEC_ASO_REG_C_4_5 = 0x2, + MLX5_IPSEC_ASO_REG_C_6_7 = 0x3, +}; + +enum { + MLX5_IPSEC_ASO_MODE = 0x0, + MLX5_IPSEC_ASO_REPLAY_PROTECTION = 0x1, + MLX5_IPSEC_ASO_INC_SN = 0x2, +}; + +struct mlx5_ifc_ipsec_aso_bits { + u8 valid[0x1]; + u8 reserved_at_201[0x1]; + u8 mode[0x2]; + u8 window_sz[0x2]; + u8 soft_lft_arm[0x1]; + u8 hard_lft_arm[0x1]; + u8 remove_flow_enable[0x1]; + u8 esn_event_arm[0x1]; + u8 reserved_at_20a[0x16]; + + u8 remove_flow_pkt_cnt[0x20]; + + u8 remove_flow_soft_lft[0x20]; + + u8 reserved_at_260[0x80]; + + u8 mode_parameter[0x20]; + + u8 replay_protection_window[0x100]; +}; + struct mlx5_ifc_ipsec_obj_bits { u8 modify_field_select[0x40]; u8 full_offload[0x1]; @@ -11584,7 +11627,11 @@ struct mlx5_ifc_ipsec_obj_bits { u8 implicit_iv[0x40]; - u8 reserved_at_100[0x700]; + u8 reserved_at_100[0x8]; + u8 ipsec_aso_access_pd[0x18]; + u8 reserved_at_120[0xe0]; + + struct mlx5_ifc_ipsec_aso_bits ipsec_aso; }; struct mlx5_ifc_create_ipsec_obj_in_bits { From 59592cfdf8a44fd1f25cf16f4e25edc7f1a4e709 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:24 +0200 Subject: [PATCH 11/38] net/mlx5e: Advertise IPsec packet offload support Add needed capabilities check to determine if device supports IPsec packet offload mode. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h | 1 + .../ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 4c47347d0ee2..fa052a89c4dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -88,6 +88,7 @@ struct mlx5_accel_esp_xfrm_attrs { enum mlx5_ipsec_cap { MLX5_IPSEC_CAP_CRYPTO = 1 << 0, MLX5_IPSEC_CAP_ESN = 1 << 1, + MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2, }; struct mlx5e_priv; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 792724ce7336..3d5a1f875398 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -31,6 +31,12 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) MLX5_CAP_ETH(mdev, insert_trailer) && MLX5_CAP_ETH(mdev, swp)) caps |= MLX5_IPSEC_CAP_CRYPTO; + if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload) && + MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) && + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) && + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap)) + caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD; + if (!caps) return 0; From cded6d80129babde53627efe32d164842cdd23a0 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:25 +0200 Subject: [PATCH 12/38] net/mlx5e: Store replay window in XFRM attributes As a preparation for future extension of IPsec hardware object to allow configuration of packet offload mode, extend the XFRM validator to check replay window values. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c | 12 ++++++++++++ .../net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h | 1 + 2 files changed, 13 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index e6411533f911..734b486db5d6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -166,6 +166,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->esn = sa_entry->esn_state.esn; if (sa_entry->esn_state.overlap) attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; + attrs->replay_window = x->replay_esn->replay_window; } /* action */ @@ -257,6 +258,17 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Unsupported xfrm offload type\n"); return -EINVAL; } + if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) { + if (x->replay_esn && x->replay_esn->replay_window != 32 && + x->replay_esn->replay_window != 64 && + x->replay_esn->replay_window != 128 && + x->replay_esn->replay_window != 256) { + netdev_info(netdev, + "Unsupported replay window size %u\n", + x->replay_esn->replay_window); + return -EINVAL; + } + } return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index fa052a89c4dd..6fe55675bee9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -83,6 +83,7 @@ struct mlx5_accel_esp_xfrm_attrs { } daddr; u8 is_ipv6; + u32 replay_window; }; enum mlx5_ipsec_cap { From e3840530b4c8d0964f8757f2cffbbce5741ad327 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:26 +0200 Subject: [PATCH 13/38] net/mlx5e: Remove extra layers of defines Instead of performing redefinition of XFRM core defines to same values but with MLX5_* prefix, cache the input values as is by making sure that the proper storage objects are used. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 17 ++++----------- .../mellanox/mlx5/core/en_accel/ipsec.h | 18 ++++------------ .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 21 ++++++++++--------- .../mlx5/core/en_accel/ipsec_offload.c | 10 ++++----- 4 files changed, 23 insertions(+), 43 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 734b486db5d6..14ed72b26bbe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -162,29 +162,20 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, /* esn */ if (sa_entry->esn_state.trigger) { - attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; + attrs->esn_trigger = true; attrs->esn = sa_entry->esn_state.esn; - if (sa_entry->esn_state.overlap) - attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; + attrs->esn_overlap = sa_entry->esn_state.overlap; attrs->replay_window = x->replay_esn->replay_window; } - /* action */ - attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ? - MLX5_ACCEL_ESP_ACTION_ENCRYPT : - MLX5_ACCEL_ESP_ACTION_DECRYPT; - /* flags */ - attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? - MLX5_ACCEL_ESP_FLAGS_TRANSPORT : - MLX5_ACCEL_ESP_FLAGS_TUNNEL; - + attrs->dir = x->xso.dir; /* spi */ attrs->spi = be32_to_cpu(x->id.spi); /* source , destination ips */ memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); - attrs->is_ipv6 = (x->props.family != AF_INET); + attrs->family = x->props.family; } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 6fe55675bee9..05790b7d062f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -43,18 +43,6 @@ #define MLX5E_IPSEC_SADB_RX_BITS 10 #define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L -enum mlx5_accel_esp_flags { - MLX5_ACCEL_ESP_FLAGS_TUNNEL = 0, /* Default */ - MLX5_ACCEL_ESP_FLAGS_TRANSPORT = 1UL << 0, - MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED = 1UL << 1, - MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP = 1UL << 2, -}; - -enum mlx5_accel_esp_action { - MLX5_ACCEL_ESP_ACTION_DECRYPT, - MLX5_ACCEL_ESP_ACTION_ENCRYPT, -}; - struct aes_gcm_keymat { u64 seq_iv; @@ -66,7 +54,6 @@ struct aes_gcm_keymat { }; struct mlx5_accel_esp_xfrm_attrs { - enum mlx5_accel_esp_action action; u32 esn; u32 spi; u32 flags; @@ -82,7 +69,10 @@ struct mlx5_accel_esp_xfrm_attrs { __be32 a6[4]; } daddr; - u8 is_ipv6; + u8 dir : 2; + u8 esn_overlap : 1; + u8 esn_trigger : 1; + u8 family; u32 replay_window; }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index b859e4a4c744..886263228b5d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -341,7 +341,7 @@ static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, struct mlx5_flow_spec *spec, struct mlx5_flow_act *flow_act) { - u8 ip_version = attrs->is_ipv6 ? 6 : 4; + u8 ip_version = (attrs->family == AF_INET) ? 4 : 6; spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS; @@ -411,7 +411,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, int err = 0; accel_esp = priv->ipsec->rx_fs; - type = attrs->is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4; + type = (attrs->family == AF_INET) ? ACCEL_FS_ESP4 : ACCEL_FS_ESP6; fs_prot = &accel_esp->fs_prot[type]; err = rx_ft_get(priv, type); @@ -453,8 +453,8 @@ static int rx_add_rule(struct mlx5e_priv *priv, rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", - attrs->action, err); + netdev_err(priv->netdev, "fail to add RX ipsec rule err=%d\n", + err); goto out_err; } @@ -505,8 +505,8 @@ static int tx_add_rule(struct mlx5e_priv *priv, rule = mlx5_add_flow_rules(priv->ipsec->tx_fs->ft, spec, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", - sa_entry->attrs.action, err); + netdev_err(priv->netdev, "fail to add TX ipsec rule err=%d\n", + err); goto out; } @@ -522,7 +522,7 @@ out: int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_sa_entry *sa_entry) { - if (sa_entry->attrs.action == MLX5_ACCEL_ESP_ACTION_ENCRYPT) + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) return tx_add_rule(priv, sa_entry); return rx_add_rule(priv, sa_entry); @@ -533,17 +533,18 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, { struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + enum accel_fs_esp_type type; mlx5_del_flow_rules(ipsec_rule->rule); - if (sa_entry->attrs.action == MLX5_ACCEL_ESP_ACTION_ENCRYPT) { + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { tx_ft_put(priv); return; } mlx5_modify_header_dealloc(mdev, ipsec_rule->set_modify_hdr); - rx_ft_put(priv, - sa_entry->attrs.is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4); + type = (sa_entry->attrs.family == AF_INET) ? ACCEL_FS_ESP4 : ACCEL_FS_ESP6; + rx_ft_put(priv, type); } void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 3d5a1f875398..3f2aeb07ea84 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -72,11 +72,10 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv); memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv)); /* esn */ - if (attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) { + if (attrs->esn_trigger) { MLX5_SET(ipsec_obj, obj, esn_en, 1); MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn); - if (attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) - MLX5_SET(ipsec_obj, obj, esn_overlap, 1); + MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap); } MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id); @@ -158,7 +157,7 @@ static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry, void *obj; int err; - if (!(attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED)) + if (!attrs->esn_trigger) return 0; general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); @@ -189,8 +188,7 @@ static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry, MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP | MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB); MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn); - if (attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) - MLX5_SET(ipsec_obj, obj, esn_overlap, 1); + MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap); /* general object fields set */ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); From 9e5286dcbbbc47b4666df1733590260ac0b475af Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:27 +0200 Subject: [PATCH 14/38] net/mlx5e: Create symmetric IPsec RX and TX flow steering structs Remove AF family obfuscation by creating symmetric structs for RX and TX IPsec flow steering chains. This simplifies to us low level IPsec FS creation logic without need to dig into multiple levels of structs. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.h | 7 +- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 275 ++++++++---------- 2 files changed, 129 insertions(+), 153 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 05790b7d062f..6b961ff08ed7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -94,7 +94,7 @@ struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_tx_drop_trailer; }; -struct mlx5e_accel_fs_esp; +struct mlx5e_ipsec_rx; struct mlx5e_ipsec_tx; struct mlx5e_ipsec { @@ -103,8 +103,9 @@ struct mlx5e_ipsec { spinlock_t sadb_rx_lock; /* Protects sadb_rx */ struct mlx5e_ipsec_sw_stats sw_stats; struct workqueue_struct *wq; - struct mlx5e_accel_fs_esp *rx_fs; - struct mlx5e_ipsec_tx *tx_fs; + struct mlx5e_ipsec_rx *rx_ipv4; + struct mlx5e_ipsec_rx *rx_ipv6; + struct mlx5e_ipsec_tx *tx; }; struct mlx5e_ipsec_esn_state { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 886263228b5d..a8cf3f8d0515 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,49 +9,40 @@ #define NUM_IPSEC_FTE BIT(15) -enum accel_fs_esp_type { - ACCEL_FS_ESP4, - ACCEL_FS_ESP6, - ACCEL_FS_ESP_NUM_TYPES, -}; - struct mlx5e_ipsec_rx_err { struct mlx5_flow_table *ft; struct mlx5_flow_handle *rule; struct mlx5_modify_hdr *copy_modify_hdr; }; -struct mlx5e_accel_fs_esp_prot { - struct mlx5_flow_table *ft; +struct mlx5e_ipsec_ft { + struct mutex mutex; /* Protect changes to this struct */ + struct mlx5_flow_table *sa; + u32 refcnt; +}; + +struct mlx5e_ipsec_rx { + struct mlx5e_ipsec_ft ft; struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; struct mlx5_flow_destination default_dest; struct mlx5e_ipsec_rx_err rx_err; - u32 refcnt; - struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ -}; - -struct mlx5e_accel_fs_esp { - struct mlx5e_accel_fs_esp_prot fs_prot[ACCEL_FS_ESP_NUM_TYPES]; }; struct mlx5e_ipsec_tx { + struct mlx5e_ipsec_ft ft; struct mlx5_flow_namespace *ns; - struct mlx5_flow_table *ft; - struct mutex mutex; /* Protect IPsec TX steering */ - u32 refcnt; }; /* IPsec RX flow steering */ -static enum mlx5_traffic_types fs_esp2tt(enum accel_fs_esp_type i) +static enum mlx5_traffic_types family2tt(u32 family) { - if (i == ACCEL_FS_ESP4) + if (family == AF_INET) return MLX5_TT_IPV4_IPSEC_ESP; return MLX5_TT_IPV6_IPSEC_ESP; } -static int rx_err_add_rule(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot, +static int rx_err_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, struct mlx5e_ipsec_rx_err *rx_err) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; @@ -89,7 +80,7 @@ static int rx_err_add_rule(struct mlx5e_priv *priv, MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; flow_act.modify_hdr = modify_hdr; fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, - &fs_prot->default_dest, 1); + &rx->default_dest, 1); if (IS_ERR(fte)) { err = PTR_ERR(fte); netdev_err(priv->netdev, "fail to add ipsec rx err copy rule err=%d\n", err); @@ -108,11 +99,10 @@ out_spec: return err; } -static int rx_fs_create(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot) +static int rx_fs_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - struct mlx5_flow_table *ft = fs_prot->ft; + struct mlx5_flow_table *ft = rx->ft.sa; struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; MLX5_DECLARE_FLOW_ACT(flow_act); @@ -136,56 +126,45 @@ static int rx_fs_create(struct mlx5e_priv *priv, netdev_err(priv->netdev, "fail to create ipsec rx miss_group err=%d\n", err); goto out; } - fs_prot->miss_group = miss_group; + rx->miss_group = miss_group; /* Create miss rule */ - miss_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &fs_prot->default_dest, 1); + miss_rule = + mlx5_add_flow_rules(ft, spec, &flow_act, &rx->default_dest, 1); if (IS_ERR(miss_rule)) { - mlx5_destroy_flow_group(fs_prot->miss_group); + mlx5_destroy_flow_group(rx->miss_group); err = PTR_ERR(miss_rule); netdev_err(priv->netdev, "fail to create ipsec rx miss_rule err=%d\n", err); goto out; } - fs_prot->miss_rule = miss_rule; + rx->miss_rule = miss_rule; out: kvfree(flow_group_in); kvfree(spec); return err; } -static void rx_destroy(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static void rx_destroy(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; + mlx5_del_flow_rules(rx->miss_rule); + mlx5_destroy_flow_group(rx->miss_group); + mlx5_destroy_flow_table(rx->ft.sa); - accel_esp = priv->ipsec->rx_fs; - - /* The netdev unreg already happened, so all offloaded rule are already removed */ - fs_prot = &accel_esp->fs_prot[type]; - - mlx5_del_flow_rules(fs_prot->miss_rule); - mlx5_destroy_flow_group(fs_prot->miss_group); - mlx5_destroy_flow_table(fs_prot->ft); - - mlx5_del_flow_rules(fs_prot->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, fs_prot->rx_err.copy_modify_hdr); - mlx5_destroy_flow_table(fs_prot->rx_err.ft); + mlx5_del_flow_rules(rx->rx_err.rule); + mlx5_modify_header_dealloc(priv->mdev, rx->rx_err.copy_modify_hdr); + mlx5_destroy_flow_table(rx->rx_err.ft); } -static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static int rx_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, + u32 family) { struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(priv->fs, false); struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_table *ft; int err; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - fs_prot->default_dest = - mlx5_ttc_get_default_dest(ttc, fs_esp2tt(type)); + rx->default_dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); ft_attr.max_fte = 1; ft_attr.autogroup.max_num_groups = 1; @@ -195,8 +174,8 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) if (IS_ERR(ft)) return PTR_ERR(ft); - fs_prot->rx_err.ft = ft; - err = rx_err_add_rule(priv, fs_prot, &fs_prot->rx_err); + rx->rx_err.ft = ft; + err = rx_err_add_rule(priv, rx, &rx->rx_err); if (err) goto err_add; @@ -211,76 +190,82 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) err = PTR_ERR(ft); goto err_fs_ft; } - fs_prot->ft = ft; + rx->ft.sa = ft; - err = rx_fs_create(priv, fs_prot); + err = rx_fs_create(priv, rx); if (err) goto err_fs; return 0; err_fs: - mlx5_destroy_flow_table(fs_prot->ft); + mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: - mlx5_del_flow_rules(fs_prot->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, fs_prot->rx_err.copy_modify_hdr); + mlx5_del_flow_rules(rx->rx_err.rule); + mlx5_modify_header_dealloc(priv->mdev, rx->rx_err.copy_modify_hdr); err_add: - mlx5_destroy_flow_table(fs_prot->rx_err.ft); + mlx5_destroy_flow_table(rx->rx_err.ft); return err; } -static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5e_priv *priv, u32 family) { struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5e_accel_fs_esp_prot *fs_prot; struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5e_ipsec_rx *rx; int err = 0; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - if (fs_prot->refcnt) + if (family == AF_INET) + rx = priv->ipsec->rx_ipv4; + else + rx = priv->ipsec->rx_ipv6; + + mutex_lock(&rx->ft.mutex); + if (rx->ft.refcnt) goto skip; /* create FT */ - err = rx_create(priv, type); + err = rx_create(priv, rx, family); if (err) goto out; /* connect */ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = fs_prot->ft; - mlx5_ttc_fwd_dest(ttc, fs_esp2tt(type), &dest); + dest.ft = rx->ft.sa; + mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest); skip: - fs_prot->refcnt++; + rx->ft.refcnt++; out: - mutex_unlock(&fs_prot->prot_mutex); - return err; + mutex_unlock(&rx->ft.mutex); + if (err) + return ERR_PTR(err); + return rx; } -static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static void rx_ft_put(struct mlx5e_priv *priv, u32 family) { struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5e_ipsec_rx *rx; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - fs_prot->refcnt--; - if (fs_prot->refcnt) + if (family == AF_INET) + rx = priv->ipsec->rx_ipv4; + else + rx = priv->ipsec->rx_ipv6; + + mutex_lock(&rx->ft.mutex); + rx->ft.refcnt--; + if (rx->ft.refcnt) goto out; /* disconnect */ - mlx5_ttc_fwd_default_dest(ttc, fs_esp2tt(type)); + mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); /* remove FT */ - rx_destroy(priv, type); + rx_destroy(priv, rx); out: - mutex_unlock(&fs_prot->prot_mutex); + mutex_unlock(&rx->ft.mutex); } /* IPsec TX flow steering */ @@ -293,47 +278,49 @@ static int tx_create(struct mlx5e_priv *priv) ft_attr.max_fte = NUM_IPSEC_FTE; ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ipsec->tx_fs->ns, &ft_attr); + ft = mlx5_create_auto_grouped_flow_table(ipsec->tx->ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); netdev_err(priv->netdev, "fail to create ipsec tx ft err=%d\n", err); return err; } - ipsec->tx_fs->ft = ft; + ipsec->tx->ft.sa = ft; return 0; } -static int tx_ft_get(struct mlx5e_priv *priv) +static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5e_priv *priv) { - struct mlx5e_ipsec_tx *tx_fs = priv->ipsec->tx_fs; + struct mlx5e_ipsec_tx *tx = priv->ipsec->tx; int err = 0; - mutex_lock(&tx_fs->mutex); - if (tx_fs->refcnt) + mutex_lock(&tx->ft.mutex); + if (tx->ft.refcnt) goto skip; err = tx_create(priv); if (err) goto out; skip: - tx_fs->refcnt++; + tx->ft.refcnt++; out: - mutex_unlock(&tx_fs->mutex); - return err; + mutex_unlock(&tx->ft.mutex); + if (err) + return ERR_PTR(err); + return tx; } static void tx_ft_put(struct mlx5e_priv *priv) { - struct mlx5e_ipsec_tx *tx_fs = priv->ipsec->tx_fs; + struct mlx5e_ipsec_tx *tx = priv->ipsec->tx; - mutex_lock(&tx_fs->mutex); - tx_fs->refcnt--; - if (tx_fs->refcnt) + mutex_lock(&tx->ft.mutex); + tx->ft.refcnt--; + if (tx->ft.refcnt) goto out; - mlx5_destroy_flow_table(tx_fs->ft); + mlx5_destroy_flow_table(tx->ft.sa); out: - mutex_unlock(&tx_fs->mutex); + mutex_unlock(&tx->ft.mutex); } static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, @@ -401,22 +388,16 @@ static int rx_add_rule(struct mlx5e_priv *priv, struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; u32 ipsec_obj_id = sa_entry->ipsec_obj_id; struct mlx5_modify_hdr *modify_hdr = NULL; - struct mlx5e_accel_fs_esp_prot *fs_prot; struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; - enum accel_fs_esp_type type; struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_rx *rx; int err = 0; - accel_esp = priv->ipsec->rx_fs; - type = (attrs->family == AF_INET) ? ACCEL_FS_ESP4 : ACCEL_FS_ESP6; - fs_prot = &accel_esp->fs_prot[type]; - - err = rx_ft_get(priv, type); - if (err) - return err; + rx = rx_ft_get(priv, attrs->family); + if (IS_ERR(rx)) + return PTR_ERR(rx); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { @@ -449,8 +430,8 @@ static int rx_add_rule(struct mlx5e_priv *priv, MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; flow_act.modify_hdr = modify_hdr; - dest.ft = fs_prot->rx_err.ft; - rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); + dest.ft = rx->rx_err.ft; + rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); netdev_err(priv->netdev, "fail to add RX ipsec rule err=%d\n", @@ -465,7 +446,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, out_err: if (modify_hdr) mlx5_modify_header_dealloc(priv->mdev, modify_hdr); - rx_ft_put(priv, type); + rx_ft_put(priv, attrs->family); out: kvfree(spec); @@ -478,11 +459,12 @@ static int tx_add_rule(struct mlx5e_priv *priv, struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_tx *tx; int err = 0; - err = tx_ft_get(priv); - if (err) - return err; + tx = tx_ft_get(priv); + if (IS_ERR(tx)) + return PTR_ERR(tx); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { @@ -502,7 +484,7 @@ static int tx_add_rule(struct mlx5e_priv *priv, flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; - rule = mlx5_add_flow_rules(priv->ipsec->tx_fs->ft, spec, &flow_act, NULL, 0); + rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); netdev_err(priv->netdev, "fail to add TX ipsec rule err=%d\n", @@ -533,7 +515,6 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, { struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); - enum accel_fs_esp_type type; mlx5_del_flow_rules(ipsec_rule->rule); @@ -543,38 +524,30 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, } mlx5_modify_header_dealloc(mdev, ipsec_rule->set_modify_hdr); - type = (sa_entry->attrs.family == AF_INET) ? ACCEL_FS_ESP4 : ACCEL_FS_ESP6; - rx_ft_put(priv, type); + rx_ft_put(priv, sa_entry->attrs.family); } void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; - enum accel_fs_esp_type i; - - if (!ipsec->rx_fs) + if (!ipsec->tx) return; - mutex_destroy(&ipsec->tx_fs->mutex); - WARN_ON(ipsec->tx_fs->refcnt); - kfree(ipsec->tx_fs); + mutex_destroy(&ipsec->tx->ft.mutex); + WARN_ON(ipsec->tx->ft.refcnt); + kfree(ipsec->tx); - accel_esp = ipsec->rx_fs; - for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { - fs_prot = &accel_esp->fs_prot[i]; - mutex_destroy(&fs_prot->prot_mutex); - WARN_ON(fs_prot->refcnt); - } - kfree(ipsec->rx_fs); + mutex_destroy(&ipsec->rx_ipv4->ft.mutex); + WARN_ON(ipsec->rx_ipv4->ft.refcnt); + kfree(ipsec->rx_ipv4); + + mutex_destroy(&ipsec->rx_ipv6->ft.mutex); + WARN_ON(ipsec->rx_ipv6->ft.refcnt); + kfree(ipsec->rx_ipv6); } int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_namespace *ns; - enum accel_fs_esp_type i; int err = -ENOMEM; ns = mlx5_get_flow_namespace(ipsec->mdev, @@ -582,26 +555,28 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) if (!ns) return -EOPNOTSUPP; - ipsec->tx_fs = kzalloc(sizeof(*ipsec->tx_fs), GFP_KERNEL); - if (!ipsec->tx_fs) + ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); + if (!ipsec->tx) return -ENOMEM; - ipsec->rx_fs = kzalloc(sizeof(*ipsec->rx_fs), GFP_KERNEL); - if (!ipsec->rx_fs) - goto err_rx; + ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); + if (!ipsec->rx_ipv4) + goto err_rx_ipv4; - mutex_init(&ipsec->tx_fs->mutex); - ipsec->tx_fs->ns = ns; + ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); + if (!ipsec->rx_ipv6) + goto err_rx_ipv6; - accel_esp = ipsec->rx_fs; - for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { - fs_prot = &accel_esp->fs_prot[i]; - mutex_init(&fs_prot->prot_mutex); - } + mutex_init(&ipsec->tx->ft.mutex); + mutex_init(&ipsec->rx_ipv4->ft.mutex); + mutex_init(&ipsec->rx_ipv6->ft.mutex); + ipsec->tx->ns = ns; return 0; -err_rx: - kfree(ipsec->tx_fs); +err_rx_ipv6: + kfree(ipsec->rx_ipv4); +err_rx_ipv4: + kfree(ipsec->tx); return err; } From fb2caa711f690f71cc6cd256faadeff7ab76f900 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:28 +0200 Subject: [PATCH 15/38] net/mlx5e: Use mlx5 print routines for low level IPsec code Low level mlx5 code needs to use mlx5_core print routines and not netdev ones, as the failures are relevant to the HW itself and not to its netdev. This change allows us to remove access to mlx5 priv structure, which holds high level driver data that isn't needed for mlx5 IPsec code. Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index a8cf3f8d0515..08feff765032 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -70,8 +70,8 @@ static int rx_err_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); - netdev_err(priv->netdev, - "fail to alloc ipsec copy modify_header_id err=%d\n", err); + mlx5_core_err(mdev, + "fail to alloc ipsec copy modify_header_id err=%d\n", err); goto out_spec; } @@ -83,7 +83,7 @@ static int rx_err_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, &rx->default_dest, 1); if (IS_ERR(fte)) { err = PTR_ERR(fte); - netdev_err(priv->netdev, "fail to add ipsec rx err copy rule err=%d\n", err); + mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); goto out; } @@ -103,6 +103,7 @@ static int rx_fs_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_table *ft = rx->ft.sa; + struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; MLX5_DECLARE_FLOW_ACT(flow_act); @@ -123,7 +124,7 @@ static int rx_fs_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) miss_group = mlx5_create_flow_group(ft, flow_group_in); if (IS_ERR(miss_group)) { err = PTR_ERR(miss_group); - netdev_err(priv->netdev, "fail to create ipsec rx miss_group err=%d\n", err); + mlx5_core_err(mdev, "fail to create ipsec rx miss_group err=%d\n", err); goto out; } rx->miss_group = miss_group; @@ -134,7 +135,7 @@ static int rx_fs_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) if (IS_ERR(miss_rule)) { mlx5_destroy_flow_group(rx->miss_group); err = PTR_ERR(miss_rule); - netdev_err(priv->netdev, "fail to create ipsec rx miss_rule err=%d\n", err); + mlx5_core_err(mdev, "fail to create ipsec rx miss_rule err=%d\n", err); goto out; } rx->miss_rule = miss_rule; @@ -273,6 +274,7 @@ static int tx_create(struct mlx5e_priv *priv) { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_table *ft; int err; @@ -281,7 +283,7 @@ static int tx_create(struct mlx5e_priv *priv) ft = mlx5_create_auto_grouped_flow_table(ipsec->tx->ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); - netdev_err(priv->netdev, "fail to create ipsec tx ft err=%d\n", err); + mlx5_core_err(mdev, "fail to create ipsec tx ft err=%d\n", err); return err; } ipsec->tx->ft.sa = ft; @@ -386,6 +388,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); u32 ipsec_obj_id = sa_entry->ipsec_obj_id; struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_flow_destination dest = {}; @@ -419,8 +422,8 @@ static int rx_add_rule(struct mlx5e_priv *priv, 1, action); if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); - netdev_err(priv->netdev, - "fail to alloc ipsec set modify_header_id err=%d\n", err); + mlx5_core_err(mdev, + "fail to alloc ipsec set modify_header_id err=%d\n", err); modify_hdr = NULL; goto out_err; } @@ -434,8 +437,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add RX ipsec rule err=%d\n", - err); + mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); goto out_err; } @@ -456,6 +458,7 @@ out: static int tx_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_sa_entry *sa_entry) { + struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; @@ -487,8 +490,7 @@ static int tx_add_rule(struct mlx5e_priv *priv, rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add TX ipsec rule err=%d\n", - err); + mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); goto out; } From c7049ca6213732e5d511a4b8d6fdabea4c3b4811 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:29 +0200 Subject: [PATCH 16/38] net/mlx5e: Remove accesses to priv for low level IPsec FS code mlx5 priv structure is driver main structure that holds high level data. That information is not needed for IPsec flow steering logic and the pointer to mlx5e_priv was not supposed to be passed in the first place. This change "cleans" the logic to rely on internal to IPsec structures without touching global mlx5e_priv. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 8 +- .../mellanox/mlx5/core/en_accel/ipsec.h | 7 +- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 100 +++++++++--------- 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 14ed72b26bbe..f518322c1ac1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -306,7 +306,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (err) goto err_xfrm; - err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry); + err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); if (err) goto err_hw_ctx; @@ -324,7 +324,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) goto out; err_add_rule: - mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(sa_entry); err_hw_ctx: mlx5_ipsec_free_sa_ctx(sa_entry); err_xfrm: @@ -344,10 +344,9 @@ static void mlx5e_xfrm_del_state(struct xfrm_state *x) static void mlx5e_xfrm_free_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); - struct mlx5e_priv *priv = netdev_priv(x->xso.dev); cancel_work_sync(&sa_entry->modify_work.work); - mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(sa_entry); mlx5_ipsec_free_sa_ctx(sa_entry); kfree(sa_entry); } @@ -378,6 +377,7 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) if (ret) goto err_fs_init; + ipsec->fs = priv->fs; priv->ipsec = ipsec; netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 6b961ff08ed7..db0ccf2a797a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -103,6 +103,7 @@ struct mlx5e_ipsec { spinlock_t sadb_rx_lock; /* Protects sadb_rx */ struct mlx5e_ipsec_sw_stats sw_stats; struct workqueue_struct *wq; + struct mlx5e_flow_steering *fs; struct mlx5e_ipsec_rx *rx_ipv4; struct mlx5e_ipsec_rx *rx_ipv6; struct mlx5e_ipsec_tx *tx; @@ -148,10 +149,8 @@ struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev, void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry); -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry); +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry); int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 08feff765032..8e87d8d02511 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -42,11 +42,11 @@ static enum mlx5_traffic_types family2tt(u32 family) return MLX5_TT_IPV6_IPSEC_ESP; } -static int rx_err_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, +static int rx_err_add_rule(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec_rx *rx, struct mlx5e_ipsec_rx_err *rx_err) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_act flow_act = {}; struct mlx5_modify_hdr *modify_hdr; struct mlx5_flow_handle *fte; @@ -99,11 +99,10 @@ out_spec: return err; } -static int rx_fs_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) +static int rx_fs_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_table *ft = rx->ft.sa; - struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; MLX5_DECLARE_FLOW_ACT(flow_act); @@ -145,22 +144,22 @@ out: return err; } -static void rx_destroy(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx) +static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) { mlx5_del_flow_rules(rx->miss_rule); mlx5_destroy_flow_group(rx->miss_group); mlx5_destroy_flow_table(rx->ft.sa); mlx5_del_flow_rules(rx->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, rx->rx_err.copy_modify_hdr); + mlx5_modify_header_dealloc(mdev, rx->rx_err.copy_modify_hdr); mlx5_destroy_flow_table(rx->rx_err.ft); } -static int rx_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, - u32 family) +static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, u32 family) { - struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(priv->fs, false); - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_table *ft; int err; @@ -176,7 +175,7 @@ static int rx_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, return PTR_ERR(ft); rx->rx_err.ft = ft; - err = rx_err_add_rule(priv, rx, &rx->rx_err); + err = rx_err_add_rule(mdev, rx, &rx->rx_err); if (err) goto err_add; @@ -193,7 +192,7 @@ static int rx_create(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx, } rx->ft.sa = ft; - err = rx_fs_create(priv, rx); + err = rx_fs_create(mdev, rx); if (err) goto err_fs; @@ -203,30 +202,31 @@ err_fs: mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: mlx5_del_flow_rules(rx->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, rx->rx_err.copy_modify_hdr); + mlx5_modify_header_dealloc(mdev, rx->rx_err.copy_modify_hdr); err_add: mlx5_destroy_flow_table(rx->rx_err.ft); return err; } -static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5e_priv *priv, u32 family) +static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec *ipsec, u32 family) { - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination dest = {}; struct mlx5e_ipsec_rx *rx; int err = 0; if (family == AF_INET) - rx = priv->ipsec->rx_ipv4; + rx = ipsec->rx_ipv4; else - rx = priv->ipsec->rx_ipv6; + rx = ipsec->rx_ipv6; mutex_lock(&rx->ft.mutex); if (rx->ft.refcnt) goto skip; /* create FT */ - err = rx_create(priv, rx, family); + err = rx_create(mdev, ipsec, rx, family); if (err) goto out; @@ -244,15 +244,16 @@ out: return rx; } -static void rx_ft_put(struct mlx5e_priv *priv, u32 family) +static void rx_ft_put(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + u32 family) { - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5e_ipsec_rx *rx; if (family == AF_INET) - rx = priv->ipsec->rx_ipv4; + rx = ipsec->rx_ipv4; else - rx = priv->ipsec->rx_ipv6; + rx = ipsec->rx_ipv6; mutex_lock(&rx->ft.mutex); rx->ft.refcnt--; @@ -263,43 +264,42 @@ static void rx_ft_put(struct mlx5e_priv *priv, u32 family) mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); /* remove FT */ - rx_destroy(priv, rx); + rx_destroy(mdev, rx); out: mutex_unlock(&rx->ft.mutex); } /* IPsec TX flow steering */ -static int tx_create(struct mlx5e_priv *priv) +static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) { struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5e_ipsec *ipsec = priv->ipsec; - struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_table *ft; int err; ft_attr.max_fte = NUM_IPSEC_FTE; ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ipsec->tx->ns, &ft_attr); + ft = mlx5_create_auto_grouped_flow_table(tx->ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); mlx5_core_err(mdev, "fail to create ipsec tx ft err=%d\n", err); return err; } - ipsec->tx->ft.sa = ft; + tx->ft.sa = ft; return 0; } -static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5e_priv *priv) +static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_tx *tx = priv->ipsec->tx; + struct mlx5e_ipsec_tx *tx = ipsec->tx; int err = 0; mutex_lock(&tx->ft.mutex); if (tx->ft.refcnt) goto skip; - err = tx_create(priv); + err = tx_create(mdev, tx); if (err) goto out; skip: @@ -311,9 +311,9 @@ out: return tx; } -static void tx_ft_put(struct mlx5e_priv *priv) +static void tx_ft_put(struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_tx *tx = priv->ipsec->tx; + struct mlx5e_ipsec_tx *tx = ipsec->tx; mutex_lock(&tx->ft.mutex); tx->ft.refcnt--; @@ -382,13 +382,13 @@ static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, flow_act->flags |= FLOW_ACT_NO_APPEND; } -static int rx_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; u32 ipsec_obj_id = sa_entry->ipsec_obj_id; struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_flow_destination dest = {}; @@ -398,7 +398,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx *rx; int err = 0; - rx = rx_ft_get(priv, attrs->family); + rx = rx_ft_get(mdev, ipsec, attrs->family); if (IS_ERR(rx)) return PTR_ERR(rx); @@ -418,7 +418,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, MLX5_SET(set_action_in, action, offset, 0); MLX5_SET(set_action_in, action, length, 32); - modify_hdr = mlx5_modify_header_alloc(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL, + modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 1, action); if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); @@ -447,25 +447,25 @@ static int rx_add_rule(struct mlx5e_priv *priv, out_err: if (modify_hdr) - mlx5_modify_header_dealloc(priv->mdev, modify_hdr); - rx_ft_put(priv, attrs->family); + mlx5_modify_header_dealloc(mdev, modify_hdr); + rx_ft_put(mdev, ipsec, attrs->family); out: kvfree(spec); return err; } -static int tx_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; struct mlx5e_ipsec_tx *tx; int err = 0; - tx = tx_ft_get(priv); + tx = tx_ft_get(mdev, ipsec); if (IS_ERR(tx)) return PTR_ERR(tx); @@ -499,21 +499,19 @@ static int tx_add_rule(struct mlx5e_priv *priv, out: kvfree(spec); if (err) - tx_ft_put(priv); + tx_ft_put(ipsec); return err; } -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) - return tx_add_rule(priv, sa_entry); + return tx_add_rule(sa_entry); - return rx_add_rule(priv, sa_entry); + return rx_add_rule(sa_entry); } -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); @@ -521,12 +519,12 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, mlx5_del_flow_rules(ipsec_rule->rule); if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { - tx_ft_put(priv); + tx_ft_put(sa_entry->ipsec); return; } mlx5_modify_header_dealloc(mdev, ipsec_rule->set_modify_hdr); - rx_ft_put(priv, sa_entry->attrs.family); + rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family); } void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) From 8518d05b8f9add527041edd2f12ffc841866b01a Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:30 +0200 Subject: [PATCH 17/38] net/mlx5e: Create Advanced Steering Operation object for IPsec Setup the ASO (Advanced Steering Operation) object that is needed for IPsec to interact with SW stack about various fast changing events: replay window, lifetime limits, e.t.c Reviewed-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 1 + .../mellanox/mlx5/core/en_accel/ipsec.c | 12 +++++ .../mellanox/mlx5/core/en_accel/ipsec.h | 13 +++++ .../mlx5/core/en_accel/ipsec_offload.c | 54 +++++++++++++++++++ 4 files changed, 80 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 65790ff58a74..2d77fb8a8a01 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -1245,4 +1245,5 @@ int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_t int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi); int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats); #endif +int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); #endif /* __MLX5_EN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index f518322c1ac1..d2c814e7af97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -373,6 +373,13 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) if (!ipsec->wq) goto err_wq; + if (mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_PACKET_OFFLOAD) { + ret = mlx5e_ipsec_aso_init(ipsec); + if (ret) + goto err_aso; + } + ret = mlx5e_accel_ipsec_fs_init(ipsec); if (ret) goto err_fs_init; @@ -383,6 +390,9 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) return; err_fs_init: + if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + mlx5e_ipsec_aso_cleanup(ipsec); +err_aso: destroy_workqueue(ipsec->wq); err_wq: kfree(ipsec); @@ -398,6 +408,8 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) return; mlx5e_accel_ipsec_fs_cleanup(ipsec); + if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + mlx5e_ipsec_aso_cleanup(ipsec); destroy_workqueue(ipsec->wq); kfree(ipsec); priv->ipsec = NULL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index db0ccf2a797a..8e2f88f269ac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -39,6 +39,7 @@ #include #include #include +#include "lib/aso.h" #define MLX5E_IPSEC_SADB_RX_BITS 10 #define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L @@ -97,6 +98,14 @@ struct mlx5e_ipsec_sw_stats { struct mlx5e_ipsec_rx; struct mlx5e_ipsec_tx; +struct mlx5e_ipsec_aso { + u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; + dma_addr_t dma_addr; + struct mlx5_aso *aso; + u32 pdn; + u32 mkey; +}; + struct mlx5e_ipsec { struct mlx5_core_dev *mdev; DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); @@ -107,6 +116,7 @@ struct mlx5e_ipsec { struct mlx5e_ipsec_rx *rx_ipv4; struct mlx5e_ipsec_rx *rx_ipv6; struct mlx5e_ipsec_tx *tx; + struct mlx5e_ipsec_aso *aso; }; struct mlx5e_ipsec_esn_state { @@ -160,6 +170,9 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev); void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, const struct mlx5_accel_esp_xfrm_attrs *attrs); +int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); +void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); + static inline struct mlx5_core_dev * mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 3f2aeb07ea84..7fef5de55229 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -2,6 +2,7 @@ /* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */ #include "mlx5_core.h" +#include "en.h" #include "ipsec.h" #include "lib/mlx5.h" @@ -207,3 +208,56 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs)); } + +int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec) +{ + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_aso *aso; + struct mlx5e_hw_objs *res; + struct device *pdev; + int err; + + aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL); + if (!aso) + return -ENOMEM; + + res = &mdev->mlx5e_res.hw_objs; + + pdev = mlx5_core_dma_dev(mdev); + aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx), + DMA_BIDIRECTIONAL); + err = dma_mapping_error(pdev, aso->dma_addr); + if (err) + goto err_dma; + + aso->aso = mlx5_aso_create(mdev, res->pdn); + if (IS_ERR(aso->aso)) { + err = PTR_ERR(aso->aso); + goto err_aso_create; + } + + ipsec->aso = aso; + return 0; + +err_aso_create: + dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), + DMA_BIDIRECTIONAL); +err_dma: + kfree(aso); + return err; +} + +void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec) +{ + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_aso *aso; + struct device *pdev; + + aso = ipsec->aso; + pdev = mlx5_core_dma_dev(mdev); + + mlx5_aso_destroy(aso->aso); + dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), + DMA_BIDIRECTIONAL); + kfree(aso); +} From 8d15f364d5d39956fd6bb848ad7732e077616c9a Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:31 +0200 Subject: [PATCH 18/38] net/mlx5e: Create hardware IPsec packet offload objects Create initial hardware IPsec packet offload object and connect it to advanced steering operation (ASO) context and queue, so the data path can communicate with the stack. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 1 + .../mellanox/mlx5/core/en_accel/ipsec.h | 3 +- .../mlx5/core/en_accel/ipsec_offload.c | 37 +++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index d2c814e7af97..c5bccc0df60d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -176,6 +176,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); attrs->family = x->props.family; + attrs->type = x->xso.type; } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 8e2f88f269ac..2c9aedf6b0ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -73,6 +73,7 @@ struct mlx5_accel_esp_xfrm_attrs { u8 dir : 2; u8 esn_overlap : 1; u8 esn_trigger : 1; + u8 type : 2; u8 family; u32 replay_window; }; @@ -102,8 +103,6 @@ struct mlx5e_ipsec_aso { u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; dma_addr_t dma_addr; struct mlx5_aso *aso; - u32 pdn; - u32 mkey; }; struct mlx5e_ipsec { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 7fef5de55229..fc88454aaf8d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -53,6 +53,38 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) } EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps); +static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn, + struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + void *aso_ctx; + + aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso); + if (attrs->esn_trigger) { + MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1); + + if (attrs->dir == XFRM_DEV_OFFLOAD_IN) { + MLX5_SET(ipsec_aso, aso_ctx, window_sz, + attrs->replay_window / 64); + MLX5_SET(ipsec_aso, aso_ctx, mode, + MLX5_IPSEC_ASO_REPLAY_PROTECTION); + } + } + + /* ASO context */ + MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn); + MLX5_SET(ipsec_obj, obj, full_offload, 1); + MLX5_SET(ipsec_aso, aso_ctx, valid, 1); + /* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used + * in flow steering to perform matching against. Please be + * aware that this register was chosen arbitrary and can't + * be used in other places as long as IPsec packet offload + * active. + */ + MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5); + if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) + MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN); +} + static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; @@ -61,6 +93,7 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {}; void *obj, *salt_p, *salt_iv_p; + struct mlx5e_hw_objs *res; int err; obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object); @@ -87,6 +120,10 @@ static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC); + res = &mdev->mlx5e_res.hw_objs; + if (attrs->type == XFRM_DEV_OFFLOAD_PACKET) + mlx5e_ipsec_packet_setup(obj, res->pdn, attrs); + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) sa_entry->ipsec_obj_id = From 42ba0f9d4b3972bd9ca9c86f89d5ccc8dc5c58b6 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:32 +0200 Subject: [PATCH 19/38] net/mlx5e: Move IPsec flow table creation to separate function Even now, to support IPsec crypto, the RX and TX paths use same logic to create flow tables. In the following patches, we will add more tables to support IPsec packet offload. So reuse existing code and rewrite it to support IPsec packet offload from the beginning. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 8e87d8d02511..f65a74e3d648 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -42,6 +42,21 @@ static enum mlx5_traffic_types family2tt(u32 family) return MLX5_TT_IPV6_IPSEC_ESP; } +static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, + int level, int prio, + int max_num_groups) +{ + struct mlx5_flow_table_attr ft_attr = {}; + + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = max_num_groups; + ft_attr.max_fte = NUM_IPSEC_FTE; + ft_attr.level = level; + ft_attr.prio = prio; + + return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); +} + static int rx_err_add_rule(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx, struct mlx5e_ipsec_rx_err *rx_err) @@ -160,17 +175,13 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, { struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); - struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_table *ft; int err; rx->default_dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); - ft_attr.max_fte = 1; - ft_attr.autogroup.max_num_groups = 1; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, + MLX5E_NIC_PRIO, 1); if (IS_ERR(ft)) return PTR_ERR(ft); @@ -180,12 +191,8 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, goto err_add; /* Create FT */ - ft_attr.max_fte = NUM_IPSEC_FTE; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; - ft_attr.autogroup.num_reserved_entries = 1; - ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, + 1); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; @@ -273,18 +280,12 @@ out: /* IPsec TX flow steering */ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) { - struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_table *ft; - int err; - ft_attr.max_fte = NUM_IPSEC_FTE; - ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(tx->ns, &ft_attr); - if (IS_ERR(ft)) { - err = PTR_ERR(ft); - mlx5_core_err(mdev, "fail to create ipsec tx ft err=%d\n", err); - return err; - } + ft = ipsec_ft_create(tx->ns, 0, 0, 1); + if (IS_ERR(ft)) + return PTR_ERR(ft); + tx->ft.sa = ft; return 0; } From 35324bbb962e984fe58e783ccad20bcfccc65cba Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:33 +0200 Subject: [PATCH 20/38] net/mlx5e: Refactor FTE setup code to be more clear The policy offload logic needs to set flow steering rule that match on saddr and daddr too, so factor out this code to separate functions, together with code alignment to netdev coding pattern of relying on family type. As part of this change, let's separate more logic from setup_fte_common to make sure that the function names describe that is done in the function better than general *common* name. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 139 +++++++++++------- 1 file changed, 85 insertions(+), 54 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index f65a74e3d648..4c5904544bda 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -326,61 +326,78 @@ out: mutex_unlock(&tx->ft.mutex); } -static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, - u32 ipsec_obj_id, - struct mlx5_flow_spec *spec, - struct mlx5_flow_act *flow_act) +static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, + __be32 *daddr) { - u8 ip_version = (attrs->family == AF_INET) ? 4 : 6; + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS; - - /* ip_version */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); - /* Non fragmented */ - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); +} +static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, + __be32 *daddr) +{ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); + + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); +} + +static void setup_fte_esp(struct mlx5_flow_spec *spec) +{ /* ESP header */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); +} +static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi) +{ /* SPI number */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); +} + +static void setup_fte_no_frags(struct mlx5_flow_spec *spec) +{ + /* Non fragmented */ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); +} + +static void setup_fte_reg_a(struct mlx5_flow_spec *spec) +{ + /* Add IPsec indicator in metadata_reg_a */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); MLX5_SET(fte_match_param, spec->match_value, - misc_parameters.outer_esp_spi, attrs->spi); - - if (ip_version == 4) { - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), - &attrs->saddr.a4, 4); - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), - &attrs->daddr.a4, 4); - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); - } else { - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), - &attrs->saddr.a6, 16); - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), - &attrs->daddr.a6, 16); - memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), - 0xff, 16); - memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), - 0xff, 16); - } - - flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; - flow_act->crypto.obj_id = ipsec_obj_id; - flow_act->flags |= FLOW_ACT_NO_APPEND; + misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); } static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) @@ -390,7 +407,6 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - u32 ipsec_obj_id = sa_entry->ipsec_obj_id; struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; @@ -409,13 +425,21 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) goto out_err; } - setup_fte_common(attrs, ipsec_obj_id, spec, &flow_act); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_no_frags(spec); /* Set bit[31] ipsec marker */ /* Set bit[23-0] ipsec_obj_id */ MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); - MLX5_SET(set_action_in, action, data, (ipsec_obj_id | BIT(31))); + MLX5_SET(set_action_in, action, data, + (sa_entry->ipsec_obj_id | BIT(31))); MLX5_SET(set_action_in, action, offset, 0); MLX5_SET(set_action_in, action, length, 32); @@ -429,6 +453,9 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) goto out_err; } + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; + flow_act.flags |= FLOW_ACT_NO_APPEND; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; @@ -458,6 +485,7 @@ out: static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { + struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5_flow_act flow_act = {}; @@ -476,16 +504,19 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) goto out; } - setup_fte_common(&sa_entry->attrs, sa_entry->ipsec_obj_id, spec, - &flow_act); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); - /* Add IPsec indicator in metadata_reg_a */ - spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; - MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, - MLX5_ETH_WQE_FT_META_IPSEC); - MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, - MLX5_ETH_WQE_FT_META_IPSEC); + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_no_frags(spec); + setup_fte_reg_a(spec); + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; + flow_act.flags |= FLOW_ACT_NO_APPEND; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, NULL, 0); From 384298c28aac42fcc93e3ff83fe627c900821804 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:34 +0200 Subject: [PATCH 21/38] net/mlx5e: Flatten the IPsec RX add rule path Rewrote the IPsec RX add rule path to be less convoluted and don't rely on pre-initialized variables. The code now has clean linear flow with clean separation between error and success paths. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.h | 2 +- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 88 +++++++++++-------- 2 files changed, 53 insertions(+), 37 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 2c9aedf6b0ef..990378d52fd4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -126,7 +126,7 @@ struct mlx5e_ipsec_esn_state { struct mlx5e_ipsec_rule { struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *set_modify_hdr; + struct mlx5_modify_hdr *modify_hdr; }; struct mlx5e_ipsec_modify_state_work { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 4c5904544bda..b81046c71e6c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -400,20 +400,52 @@ static void setup_fte_reg_a(struct mlx5_flow_spec *spec) misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); } -static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) +static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, + struct mlx5_flow_act *flow_act) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + enum mlx5_flow_namespace_type ns_type; + struct mlx5_modify_hdr *modify_hdr; + + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + switch (dir) { + case XFRM_DEV_OFFLOAD_IN: + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_B); + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + break; + default: + return -EINVAL; + } + + MLX5_SET(set_action_in, action, data, val); + MLX5_SET(set_action_in, action, offset, 0); + MLX5_SET(set_action_in, action, length, 32); + + modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); + if (IS_ERR(modify_hdr)) { + mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", + PTR_ERR(modify_hdr)); + return PTR_ERR(modify_hdr); + } + + flow_act->modify_hdr = modify_hdr; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + return 0; +} + +static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) +{ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; struct mlx5e_ipsec_rx *rx; - int err = 0; + int err; rx = rx_ft_get(mdev, ipsec, attrs->family); if (IS_ERR(rx)) @@ -422,7 +454,7 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { err = -ENOMEM; - goto out_err; + goto err_alloc; } if (attrs->family == AF_INET) @@ -434,52 +466,36 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) setup_fte_esp(spec); setup_fte_no_frags(spec); - /* Set bit[31] ipsec marker */ - /* Set bit[23-0] ipsec_obj_id */ - MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); - MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); - MLX5_SET(set_action_in, action, data, - (sa_entry->ipsec_obj_id | BIT(31))); - MLX5_SET(set_action_in, action, offset, 0); - MLX5_SET(set_action_in, action, length, 32); - - modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, - 1, action); - if (IS_ERR(modify_hdr)) { - err = PTR_ERR(modify_hdr); - mlx5_core_err(mdev, - "fail to alloc ipsec set modify_header_id err=%d\n", err); - modify_hdr = NULL; - goto out_err; - } + err = setup_modify_header(mdev, sa_entry->ipsec_obj_id | BIT(31), + XFRM_DEV_OFFLOAD_IN, &flow_act); + if (err) + goto err_mod_header; flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; flow_act.flags |= FLOW_ACT_NO_APPEND; - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | - MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - flow_act.modify_hdr = modify_hdr; dest.ft = rx->rx_err.ft; rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); - goto out_err; + goto err_add_flow; } + kvfree(spec); ipsec_rule->rule = rule; - ipsec_rule->set_modify_hdr = modify_hdr; - goto out; + ipsec_rule->modify_hdr = flow_act.modify_hdr; + return 0; -out_err: - if (modify_hdr) - mlx5_modify_header_dealloc(mdev, modify_hdr); - rx_ft_put(mdev, ipsec, attrs->family); - -out: +err_add_flow: + mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); +err_mod_header: kvfree(spec); +err_alloc: + rx_ft_put(mdev, ipsec, attrs->family); return err; } @@ -555,7 +571,7 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) return; } - mlx5_modify_header_dealloc(mdev, ipsec_rule->set_modify_hdr); + mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family); } From 27ebe531d2d99886a189ce3d48b9872c3216bb9e Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:35 +0200 Subject: [PATCH 22/38] net/mlx5e: Make clear what IPsec rx_err does Reuse existing struct what holds all information about modify header pointer and rule. This helps to reduce ambiguity from the name _err_ that doesn't describe the real purpose of that flow table, rule and function - to copy status result from HW to the stack. Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index b81046c71e6c..b89001538abd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,15 +9,10 @@ #define NUM_IPSEC_FTE BIT(15) -struct mlx5e_ipsec_rx_err { - struct mlx5_flow_table *ft; - struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *copy_modify_hdr; -}; - struct mlx5e_ipsec_ft { struct mutex mutex; /* Protect changes to this struct */ struct mlx5_flow_table *sa; + struct mlx5_flow_table *status; u32 refcnt; }; @@ -26,7 +21,7 @@ struct mlx5e_ipsec_rx { struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; struct mlx5_flow_destination default_dest; - struct mlx5e_ipsec_rx_err rx_err; + struct mlx5e_ipsec_rule status; }; struct mlx5e_ipsec_tx { @@ -57,9 +52,8 @@ static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); } -static int rx_err_add_rule(struct mlx5_core_dev *mdev, - struct mlx5e_ipsec_rx *rx, - struct mlx5e_ipsec_rx_err *rx_err) +static int ipsec_status_rule(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec_rx *rx) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5_flow_act flow_act = {}; @@ -94,7 +88,7 @@ static int rx_err_add_rule(struct mlx5_core_dev *mdev, flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; flow_act.modify_hdr = modify_hdr; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, + fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, &rx->default_dest, 1); if (IS_ERR(fte)) { err = PTR_ERR(fte); @@ -103,8 +97,8 @@ static int rx_err_add_rule(struct mlx5_core_dev *mdev, } kvfree(spec); - rx_err->rule = fte; - rx_err->copy_modify_hdr = modify_hdr; + rx->status.rule = fte; + rx->status.modify_hdr = modify_hdr; return 0; out: @@ -165,9 +159,9 @@ static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) mlx5_destroy_flow_group(rx->miss_group); mlx5_destroy_flow_table(rx->ft.sa); - mlx5_del_flow_rules(rx->rx_err.rule); - mlx5_modify_header_dealloc(mdev, rx->rx_err.copy_modify_hdr); - mlx5_destroy_flow_table(rx->rx_err.ft); + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); + mlx5_destroy_flow_table(rx->ft.status); } static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, @@ -185,8 +179,8 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, if (IS_ERR(ft)) return PTR_ERR(ft); - rx->rx_err.ft = ft; - err = rx_err_add_rule(mdev, rx, &rx->rx_err); + rx->ft.status = ft; + err = ipsec_status_rule(mdev, rx); if (err) goto err_add; @@ -208,10 +202,10 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, err_fs: mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: - mlx5_del_flow_rules(rx->rx_err.rule); - mlx5_modify_header_dealloc(mdev, rx->rx_err.copy_modify_hdr); + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); err_add: - mlx5_destroy_flow_table(rx->rx_err.ft); + mlx5_destroy_flow_table(rx->ft.status); return err; } @@ -477,7 +471,7 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = rx->rx_err.ft; + dest.ft = rx->ft.status; rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); From d7ec2b76026bcd72dec47dfeeb79c1a542868204 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:36 +0200 Subject: [PATCH 23/38] net/mlx5e: Group IPsec miss handles into separate struct Move miss handles into dedicated struct, so we can reuse it in next patch when creating IPsec policy flow table. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index b89001538abd..dfdda5ae2245 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -16,10 +16,14 @@ struct mlx5e_ipsec_ft { u32 refcnt; }; +struct mlx5e_ipsec_miss { + struct mlx5_flow_group *group; + struct mlx5_flow_handle *rule; +}; + struct mlx5e_ipsec_rx { struct mlx5e_ipsec_ft ft; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; + struct mlx5e_ipsec_miss sa; struct mlx5_flow_destination default_dest; struct mlx5e_ipsec_rule status; }; @@ -135,18 +139,18 @@ static int rx_fs_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) mlx5_core_err(mdev, "fail to create ipsec rx miss_group err=%d\n", err); goto out; } - rx->miss_group = miss_group; + rx->sa.group = miss_group; /* Create miss rule */ miss_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &rx->default_dest, 1); if (IS_ERR(miss_rule)) { - mlx5_destroy_flow_group(rx->miss_group); + mlx5_destroy_flow_group(rx->sa.group); err = PTR_ERR(miss_rule); mlx5_core_err(mdev, "fail to create ipsec rx miss_rule err=%d\n", err); goto out; } - rx->miss_rule = miss_rule; + rx->sa.rule = miss_rule; out: kvfree(flow_group_in); kvfree(spec); @@ -155,8 +159,8 @@ out: static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) { - mlx5_del_flow_rules(rx->miss_rule); - mlx5_destroy_flow_group(rx->miss_group); + mlx5_del_flow_rules(rx->sa.rule); + mlx5_destroy_flow_group(rx->sa.group); mlx5_destroy_flow_table(rx->ft.sa); mlx5_del_flow_rules(rx->status.rule); From a8e052932a2aaeccb979fddbca6f018ef8b458eb Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:10:37 +0200 Subject: [PATCH 24/38] net/mlx5e: Generalize creation of default IPsec miss group and rule Create general function that sets miss group and rule to forward all not-matched traffic to the next table. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index dfdda5ae2245..5bc6f9d1f5a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -24,7 +24,6 @@ struct mlx5e_ipsec_miss { struct mlx5e_ipsec_rx { struct mlx5e_ipsec_ft ft; struct mlx5e_ipsec_miss sa; - struct mlx5_flow_destination default_dest; struct mlx5e_ipsec_rule status; }; @@ -57,7 +56,8 @@ static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, } static int ipsec_status_rule(struct mlx5_core_dev *mdev, - struct mlx5e_ipsec_rx *rx) + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5_flow_act flow_act = {}; @@ -92,8 +92,7 @@ static int ipsec_status_rule(struct mlx5_core_dev *mdev, flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; flow_act.modify_hdr = modify_hdr; - fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, - &rx->default_dest, 1); + fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 1); if (IS_ERR(fte)) { err = PTR_ERR(fte); mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); @@ -112,12 +111,12 @@ out_spec: return err; } -static int rx_fs_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) +static int ipsec_miss_create(struct mlx5_core_dev *mdev, + struct mlx5_flow_table *ft, + struct mlx5e_ipsec_miss *miss, + struct mlx5_flow_destination *dest) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - struct mlx5_flow_table *ft = rx->ft.sa; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_spec *spec; u32 *flow_group_in; @@ -133,24 +132,23 @@ static int rx_fs_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) /* Create miss_group */ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); - miss_group = mlx5_create_flow_group(ft, flow_group_in); - if (IS_ERR(miss_group)) { - err = PTR_ERR(miss_group); - mlx5_core_err(mdev, "fail to create ipsec rx miss_group err=%d\n", err); + miss->group = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(miss->group)) { + err = PTR_ERR(miss->group); + mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", + err); goto out; } - rx->sa.group = miss_group; /* Create miss rule */ - miss_rule = - mlx5_add_flow_rules(ft, spec, &flow_act, &rx->default_dest, 1); - if (IS_ERR(miss_rule)) { - mlx5_destroy_flow_group(rx->sa.group); - err = PTR_ERR(miss_rule); - mlx5_core_err(mdev, "fail to create ipsec rx miss_rule err=%d\n", err); + miss->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); + if (IS_ERR(miss->rule)) { + mlx5_destroy_flow_group(miss->group); + err = PTR_ERR(miss->rule); + mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", + err); goto out; } - rx->sa.rule = miss_rule; out: kvfree(flow_group_in); kvfree(spec); @@ -173,18 +171,19 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, { struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + struct mlx5_flow_destination dest; struct mlx5_flow_table *ft; int err; - rx->default_dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); - ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, MLX5E_NIC_PRIO, 1); if (IS_ERR(ft)) return PTR_ERR(ft); rx->ft.status = ft; - err = ipsec_status_rule(mdev, rx); + + dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); + err = ipsec_status_rule(mdev, rx, &dest); if (err) goto err_add; @@ -197,7 +196,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, } rx->ft.sa = ft; - err = rx_fs_create(mdev, rx); + err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, &dest); if (err) goto err_fs; From 8c17295bd4991b6d275248ff7f987f54dd5eb0e6 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:45 +0200 Subject: [PATCH 25/38] net/mlx5e: Create IPsec policy offload tables Add empty table to be used for IPsec policy offload. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../net/ethernet/mellanox/mlx5/core/en/fs.h | 3 +- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 53 ++++++++++++++++++- .../net/ethernet/mellanox/mlx5/core/fs_core.c | 6 +-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index bf2741eb7f9b..379c6dc9a3be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -84,7 +84,8 @@ enum { MLX5E_ARFS_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, #endif #ifdef CONFIG_MLX5_EN_IPSEC - MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_POL_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, #endif }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 5bc6f9d1f5a6..a3c7d0f142c0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -11,6 +11,7 @@ struct mlx5e_ipsec_ft { struct mutex mutex; /* Protect changes to this struct */ + struct mlx5_flow_table *pol; struct mlx5_flow_table *sa; struct mlx5_flow_table *status; u32 refcnt; @@ -23,12 +24,14 @@ struct mlx5e_ipsec_miss { struct mlx5e_ipsec_rx { struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; struct mlx5e_ipsec_miss sa; struct mlx5e_ipsec_rule status; }; struct mlx5e_ipsec_tx { struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; struct mlx5_flow_namespace *ns; }; @@ -157,6 +160,10 @@ out: static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) { + mlx5_del_flow_rules(rx->pol.rule); + mlx5_destroy_flow_group(rx->pol.group); + mlx5_destroy_flow_table(rx->ft.pol); + mlx5_del_flow_rules(rx->sa.rule); mlx5_destroy_flow_group(rx->sa.group); mlx5_destroy_flow_table(rx->ft.sa); @@ -200,8 +207,27 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, if (err) goto err_fs; + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO, + 1); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + goto err_pol_ft; + } + rx->ft.pol = ft; + memset(&dest, 0x00, sizeof(dest)); + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = rx->ft.sa; + err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, &dest); + if (err) + goto err_pol_miss; + return 0; +err_pol_miss: + mlx5_destroy_flow_table(rx->ft.pol); +err_pol_ft: + mlx5_del_flow_rules(rx->sa.rule); + mlx5_destroy_flow_group(rx->sa.group); err_fs: mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: @@ -236,7 +262,7 @@ static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, /* connect */ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = rx->ft.sa; + dest.ft = rx->ft.pol; mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest); skip: @@ -277,14 +303,34 @@ out: /* IPsec TX flow steering */ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) { + struct mlx5_flow_destination dest = {}; struct mlx5_flow_table *ft; + int err; - ft = ipsec_ft_create(tx->ns, 0, 0, 1); + ft = ipsec_ft_create(tx->ns, 1, 0, 1); if (IS_ERR(ft)) return PTR_ERR(ft); tx->ft.sa = ft; + + ft = ipsec_ft_create(tx->ns, 0, 0, 1); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + goto err_pol_ft; + } + tx->ft.pol = ft; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = tx->ft.sa; + err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); + if (err) + goto err_pol_miss; return 0; + +err_pol_miss: + mlx5_destroy_flow_table(tx->ft.pol); +err_pol_ft: + mlx5_destroy_flow_table(tx->ft.sa); + return err; } static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, @@ -318,6 +364,9 @@ static void tx_ft_put(struct mlx5e_ipsec *ipsec) if (tx->ft.refcnt) goto out; + mlx5_del_flow_rules(tx->pol.rule); + mlx5_destroy_flow_group(tx->pol.group); + mlx5_destroy_flow_table(tx->ft.pol); mlx5_destroy_flow_table(tx->ft.sa); out: mutex_unlock(&tx->ft.mutex); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index d53749248fa0..9995307d374b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -111,8 +111,8 @@ #define ETHTOOL_PRIO_NUM_LEVELS 1 #define ETHTOOL_NUM_PRIOS 11 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS) -/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}} */ -#define KERNEL_NIC_PRIO_NUM_LEVELS 7 +/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}}, IPsec policy */ +#define KERNEL_NIC_PRIO_NUM_LEVELS 8 #define KERNEL_NIC_NUM_PRIOS 1 /* One more level for tc */ #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1) @@ -133,7 +133,7 @@ #define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1) #define KERNEL_TX_IPSEC_NUM_PRIOS 1 -#define KERNEL_TX_IPSEC_NUM_LEVELS 1 +#define KERNEL_TX_IPSEC_NUM_LEVELS 2 #define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS) #define KERNEL_TX_MACSEC_NUM_PRIOS 1 From a5b8ca9471d394459152a2a7f8f5f8227ae3e3c0 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:46 +0200 Subject: [PATCH 26/38] net/mlx5e: Add XFRM policy offload logic Implement mlx5 flow steering logic and mlx5 IPsec code support XFRM policy offload. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 118 +++++++++++++- .../mellanox/mlx5/core/en_accel/ipsec.h | 32 ++++ .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 146 ++++++++++++++++++ 3 files changed, 295 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index c5bccc0df60d..b22a31e178cf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -45,6 +45,11 @@ static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; } +static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x) +{ + return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle; +} + struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, unsigned int handle) { @@ -446,6 +451,101 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) queue_work(sa_entry->ipsec->wq, &modify_work->work); } +static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x) +{ + struct net_device *netdev = x->xdo.real_dev; + + if (x->type != XFRM_POLICY_TYPE_MAIN) { + netdev_info(netdev, "Cannot offload non-main policy types\n"); + return -EINVAL; + } + + /* Please pay attention that we support only one template */ + if (x->xfrm_nr > 1) { + netdev_info(netdev, "Cannot offload more than one template\n"); + return -EINVAL; + } + + if (x->xdo.dir != XFRM_DEV_OFFLOAD_IN && + x->xdo.dir != XFRM_DEV_OFFLOAD_OUT) { + netdev_info(netdev, "Cannot offload forward policy\n"); + return -EINVAL; + } + + if (!x->xfrm_vec[0].reqid) { + netdev_info(netdev, "Cannot offload policy without reqid\n"); + return -EINVAL; + } + + if (x->xdo.type != XFRM_DEV_OFFLOAD_PACKET) { + netdev_info(netdev, "Unsupported xfrm offload type\n"); + return -EINVAL; + } + + return 0; +} + +static void +mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, + struct mlx5_accel_pol_xfrm_attrs *attrs) +{ + struct xfrm_policy *x = pol_entry->x; + struct xfrm_selector *sel; + + sel = &x->selector; + memset(attrs, 0, sizeof(*attrs)); + + memcpy(&attrs->saddr, sel->saddr.a6, sizeof(attrs->saddr)); + memcpy(&attrs->daddr, sel->daddr.a6, sizeof(attrs->daddr)); + attrs->family = sel->family; + attrs->dir = x->xdo.dir; + attrs->action = x->action; + attrs->type = XFRM_DEV_OFFLOAD_PACKET; +} + +static int mlx5e_xfrm_add_policy(struct xfrm_policy *x) +{ + struct net_device *netdev = x->xdo.real_dev; + struct mlx5e_ipsec_pol_entry *pol_entry; + struct mlx5e_priv *priv; + int err; + + priv = netdev_priv(netdev); + if (!priv->ipsec) + return -EOPNOTSUPP; + + err = mlx5e_xfrm_validate_policy(x); + if (err) + return err; + + pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); + if (!pol_entry) + return -ENOMEM; + + pol_entry->x = x; + pol_entry->ipsec = priv->ipsec; + + mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs); + err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); + if (err) + goto err_fs; + + x->xdo.offload_handle = (unsigned long)pol_entry; + return 0; + +err_fs: + kfree(pol_entry); + return err; +} + +static void mlx5e_xfrm_free_policy(struct xfrm_policy *x) +{ + struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(x); + + mlx5e_accel_ipsec_fs_del_pol(pol_entry); + kfree(pol_entry); +} + static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_state_add = mlx5e_xfrm_add_state, .xdo_dev_state_delete = mlx5e_xfrm_del_state, @@ -454,6 +554,17 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, }; +static const struct xfrmdev_ops mlx5e_ipsec_packet_xfrmdev_ops = { + .xdo_dev_state_add = mlx5e_xfrm_add_state, + .xdo_dev_state_delete = mlx5e_xfrm_del_state, + .xdo_dev_state_free = mlx5e_xfrm_free_state, + .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, + .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, + + .xdo_dev_policy_add = mlx5e_xfrm_add_policy, + .xdo_dev_policy_free = mlx5e_xfrm_free_policy, +}; + void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; @@ -463,7 +574,12 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) return; mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); - netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; + + if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + netdev->xfrmdev_ops = &mlx5e_ipsec_packet_xfrmdev_ops; + else + netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; + netdev->features |= NETIF_F_HW_ESP; netdev->hw_enc_features |= NETIF_F_HW_ESP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 990378d52fd4..8d1a0d053eb4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -149,6 +149,30 @@ struct mlx5e_ipsec_sa_entry { struct mlx5e_ipsec_modify_state_work modify_work; }; +struct mlx5_accel_pol_xfrm_attrs { + union { + __be32 a4; + __be32 a6[4]; + } saddr; + + union { + __be32 a4; + __be32 a6[4]; + } daddr; + + u8 family; + u8 action; + u8 type : 2; + u8 dir : 2; +}; + +struct mlx5e_ipsec_pol_entry { + struct xfrm_policy *x; + struct mlx5e_ipsec *ipsec; + struct mlx5_flow_handle *rule; + struct mlx5_accel_pol_xfrm_attrs attrs; +}; + void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); @@ -160,6 +184,8 @@ void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry); void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry); +void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry); int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); @@ -177,6 +203,12 @@ mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry) { return sa_entry->ipsec->mdev; } + +static inline struct mlx5_core_dev * +mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + return pol_entry->ipsec->mdev; +} #else static inline void mlx5e_ipsec_init(struct mlx5e_priv *priv) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index a3c7d0f142c0..27f866a158de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -597,6 +597,130 @@ out: return err; } +static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_tx *tx; + int err; + + tx = tx_ft_get(mdev, pol_entry->ipsec); + if (IS_ERR(tx)) + return PTR_ERR(tx); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto err_alloc; + } + + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + switch (attrs->action) { + case XFRM_POLICY_ALLOW: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + break; + case XFRM_POLICY_BLOCK: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; + break; + default: + WARN_ON(true); + err = -EINVAL; + goto err_action; + } + + flow_act.flags |= FLOW_ACT_NO_APPEND; + dest.ft = tx->ft.sa; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); + goto err_action; + } + + kvfree(spec); + pol_entry->rule = rule; + return 0; + +err_action: + kvfree(spec); +err_alloc: + tx_ft_put(pol_entry->ipsec); + return err; +} + +static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_rx *rx; + int err; + + rx = rx_ft_get(mdev, pol_entry->ipsec, attrs->family); + if (IS_ERR(rx)) + return PTR_ERR(rx); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto err_alloc; + } + + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + switch (attrs->action) { + case XFRM_POLICY_ALLOW: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + break; + case XFRM_POLICY_BLOCK: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; + break; + default: + WARN_ON(true); + err = -EINVAL; + goto err_action; + } + + flow_act.flags |= FLOW_ACT_NO_APPEND; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = rx->ft.sa; + rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); + goto err_action; + } + + kvfree(spec); + pol_entry->rule = rule; + return 0; + +err_action: + kvfree(spec); +err_alloc: + rx_ft_put(mdev, pol_entry->ipsec, attrs->family); + return err; +} + int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) @@ -621,6 +745,28 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family); } +int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) + return tx_add_policy(pol_entry); + + return rx_add_policy(pol_entry); +} + +void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + + mlx5_del_flow_rules(pol_entry->rule); + + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { + tx_ft_put(pol_entry->ipsec); + return; + } + + rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family); +} + void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) { if (!ipsec->tx) From 9af594d8a99ef726b5785a309fb230265d16fd8b Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:47 +0200 Subject: [PATCH 27/38] net/mlx5e: Use same coding pattern for Rx and Tx flows Remove intermediate variable in favor of having similar coding style for Rx and Tx add rule functions. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 27f866a158de..893c1862e211 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -482,7 +482,6 @@ static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { - struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec *ipsec = sa_entry->ipsec; @@ -532,8 +531,8 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) } kvfree(spec); - ipsec_rule->rule = rule; - ipsec_rule->modify_hdr = flow_act.modify_hdr; + sa_entry->ipsec_rule.rule = rule; + sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; return 0; err_add_flow: From 6b5c45e16e434efc6d323a8bc79e0eba49cd13f2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:48 +0200 Subject: [PATCH 28/38] net/mlx5e: Configure IPsec packet offload flow steering In packet offload mode, the HW is responsible to handle ESP headers, SPI numbers and trailers (ICV) together with different logic for RX and TX paths. In order to support packet offload mode, special logic is added to flow steering rules. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 2 + .../mellanox/mlx5/core/en_accel/ipsec.h | 2 + .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 101 +++++++++++++++--- 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index b22a31e178cf..65f73a5c29ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -162,6 +162,8 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, sizeof(aes_gcm->salt)); + attrs->authsize = crypto_aead_authsize(aead) / 4; /* in dwords */ + /* iv len */ aes_gcm->icv_len = x->aead->alg_icv_len; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 8d1a0d053eb4..25b865590488 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -76,6 +76,7 @@ struct mlx5_accel_esp_xfrm_attrs { u8 type : 2; u8 family; u32 replay_window; + u32 authsize; }; enum mlx5_ipsec_cap { @@ -127,6 +128,7 @@ struct mlx5e_ipsec_esn_state { struct mlx5e_ipsec_rule { struct mlx5_flow_handle *rule; struct mlx5_modify_hdr *modify_hdr; + struct mlx5_pkt_reformat *pkt_reformat; }; struct mlx5e_ipsec_modify_state_work { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 893c1862e211..dbe35accaebf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -480,6 +480,48 @@ static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, return 0; } +static int setup_pkt_reformat(struct mlx5_core_dev *mdev, + struct mlx5_accel_esp_xfrm_attrs *attrs, + struct mlx5_flow_act *flow_act) +{ + enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5_pkt_reformat *pkt_reformat; + u8 reformatbf[16] = {}; + __be32 spi; + + if (attrs->dir == XFRM_DEV_OFFLOAD_IN) { + reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + goto cmd; + } + + if (attrs->family == AF_INET) + reformat_params.type = + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; + else + reformat_params.type = + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; + + /* convert to network format */ + spi = htonl(attrs->spi); + memcpy(reformatbf, &spi, 4); + + reformat_params.param_0 = attrs->authsize; + reformat_params.size = sizeof(reformatbf); + reformat_params.data = &reformatbf; + +cmd: + pkt_reformat = + mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); + if (IS_ERR(pkt_reformat)) + return PTR_ERR(pkt_reformat); + + flow_act->pkt_reformat = pkt_reformat; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + return 0; +} + static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; @@ -516,6 +558,16 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) if (err) goto err_mod_header; + switch (attrs->type) { + case XFRM_DEV_OFFLOAD_PACKET: + err = setup_pkt_reformat(mdev, attrs, &flow_act); + if (err) + goto err_pkt_reformat; + break; + default: + break; + } + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; flow_act.flags |= FLOW_ACT_NO_APPEND; @@ -533,9 +585,13 @@ static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) sa_entry->ipsec_rule.rule = rule; sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; + sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; return 0; err_add_flow: + if (flow_act.pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); +err_pkt_reformat: mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); err_mod_header: kvfree(spec); @@ -562,7 +618,7 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { err = -ENOMEM; - goto out; + goto err_alloc; } if (attrs->family == AF_INET) @@ -570,29 +626,47 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) else setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); - setup_fte_spi(spec, attrs->spi); - setup_fte_esp(spec); setup_fte_no_frags(spec); - setup_fte_reg_a(spec); + + switch (attrs->type) { + case XFRM_DEV_OFFLOAD_CRYPTO: + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_reg_a(spec); + break; + case XFRM_DEV_OFFLOAD_PACKET: + err = setup_pkt_reformat(mdev, attrs, &flow_act); + if (err) + goto err_pkt_reformat; + break; + default: + break; + } flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; flow_act.flags |= FLOW_ACT_NO_APPEND; - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); - goto out; + goto err_add_flow; } - sa_entry->ipsec_rule.rule = rule; - -out: kvfree(spec); - if (err) - tx_ft_put(ipsec); + sa_entry->ipsec_rule.rule = rule; + sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; + return 0; + +err_add_flow: + if (flow_act.pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); +err_pkt_reformat: + kvfree(spec); +err_alloc: + tx_ft_put(ipsec); return err; } @@ -735,6 +809,9 @@ void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) mlx5_del_flow_rules(ipsec_rule->rule); + if (ipsec_rule->pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { tx_ft_put(sa_entry->ipsec); return; From 18f38fd267abf3be8330dd73ddeb838f2dd98f29 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:49 +0200 Subject: [PATCH 29/38] net/mlx5e: Improve IPsec flow steering autogroup Flow steering API separates newly created rules based on their match criteria. Right now, all IPsec tables are created with one group and suffers from not-optimal FS performance. Count number of different match criteria for relevant tables, and set proper value at the table creation. Reviewed-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index dbe35accaebf..0af831128c4e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -196,7 +196,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, /* Create FT */ ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, - 1); + 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; @@ -208,7 +208,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, goto err_fs; ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO, - 1); + 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_pol_ft; @@ -307,13 +307,13 @@ static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) struct mlx5_flow_table *ft; int err; - ft = ipsec_ft_create(tx->ns, 1, 0, 1); + ft = ipsec_ft_create(tx->ns, 1, 0, 4); if (IS_ERR(ft)) return PTR_ERR(ft); tx->ft.sa = ft; - ft = ipsec_ft_create(tx->ns, 0, 0, 1); + ft = ipsec_ft_create(tx->ns, 0, 0, 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_pol_ft; From 81f8fba5ecaad53b6b32a81fe7ef06b772557647 Mon Sep 17 00:00:00 2001 From: Raed Salem Date: Fri, 2 Dec 2022 22:14:50 +0200 Subject: [PATCH 30/38] net/mlx5e: Add statistics for Rx/Tx IPsec offloaded flows Add the following statistics: RX successfully IPsec flows: ipsec_rx_pkts : Number of packets passed Rx IPsec flow ipsec_rx_bytes : Number of bytes passed Rx IPsec flow Rx dropped IPsec policy packets: ipsec_rx_drop_pkts: Number of packets dropped in Rx datapath due to IPsec drop policy ipsec_rx_drop_bytes: Number of bytes dropped in Rx datapath due to IPsec drop policy TX successfully encrypted and encapsulated IPsec packets: ipsec_tx_pkts : Number of packets encrypted and encapsulated successfully ipsec_tx_bytes : Number of bytes encrypted and encapsulated successfully Tx dropped IPsec policy packets: ipsec_tx_drop_pkts: Number of packets dropped in Tx datapath due to IPsec drop policy ipsec_tx_drop_bytes: Number of bytes dropped in Tx datapath due to IPsec drop policy The above can be seen using: ethtool -S |grep ipsec Signed-off-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.h | 15 ++ .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 188 +++++++++++++++--- .../mellanox/mlx5/core/en_accel/ipsec_stats.c | 52 +++++ .../ethernet/mellanox/mlx5/core/en_stats.c | 1 + .../ethernet/mellanox/mlx5/core/en_stats.h | 1 + 5 files changed, 233 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 25b865590488..2e7654e90314 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -87,6 +87,17 @@ enum mlx5_ipsec_cap { struct mlx5e_priv; +struct mlx5e_ipsec_hw_stats { + u64 ipsec_rx_pkts; + u64 ipsec_rx_bytes; + u64 ipsec_rx_drop_pkts; + u64 ipsec_rx_drop_bytes; + u64 ipsec_tx_pkts; + u64 ipsec_tx_bytes; + u64 ipsec_tx_drop_pkts; + u64 ipsec_tx_drop_bytes; +}; + struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_rx_drop_sp_alloc; atomic64_t ipsec_rx_drop_sadb_miss; @@ -111,6 +122,7 @@ struct mlx5e_ipsec { DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); spinlock_t sadb_rx_lock; /* Protects sadb_rx */ struct mlx5e_ipsec_sw_stats sw_stats; + struct mlx5e_ipsec_hw_stats hw_stats; struct workqueue_struct *wq; struct mlx5e_flow_steering *fs; struct mlx5e_ipsec_rx *rx_ipv4; @@ -200,6 +212,9 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); +void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, + void *ipsec_stats); + static inline struct mlx5_core_dev * mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 0af831128c4e..8ba92b00afdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,6 +9,11 @@ #define NUM_IPSEC_FTE BIT(15) +struct mlx5e_ipsec_fc { + struct mlx5_fc *cnt; + struct mlx5_fc *drop; +}; + struct mlx5e_ipsec_ft { struct mutex mutex; /* Protect changes to this struct */ struct mlx5_flow_table *pol; @@ -27,12 +32,14 @@ struct mlx5e_ipsec_rx { struct mlx5e_ipsec_miss pol; struct mlx5e_ipsec_miss sa; struct mlx5e_ipsec_rule status; + struct mlx5e_ipsec_fc *fc; }; struct mlx5e_ipsec_tx { struct mlx5e_ipsec_ft ft; struct mlx5e_ipsec_miss pol; struct mlx5_flow_namespace *ns; + struct mlx5e_ipsec_fc *fc; }; /* IPsec RX flow steering */ @@ -93,9 +100,10 @@ static int ipsec_status_rule(struct mlx5_core_dev *mdev, /* create fte */ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | - MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; flow_act.modify_hdr = modify_hdr; - fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 1); + fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); if (IS_ERR(fte)) { err = PTR_ERR(fte); mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); @@ -178,7 +186,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, { struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); - struct mlx5_flow_destination dest; + struct mlx5_flow_destination dest[2]; struct mlx5_flow_table *ft; int err; @@ -189,8 +197,10 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, rx->ft.status = ft; - dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); - err = ipsec_status_rule(mdev, rx, &dest); + dest[0] = mlx5_ttc_get_default_dest(ttc, family2tt(family)); + dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); + err = ipsec_status_rule(mdev, rx, dest); if (err) goto err_add; @@ -203,7 +213,7 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, } rx->ft.sa = ft; - err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, &dest); + err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, dest); if (err) goto err_fs; @@ -214,10 +224,10 @@ static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, goto err_pol_ft; } rx->ft.pol = ft; - memset(&dest, 0x00, sizeof(dest)); - dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = rx->ft.sa; - err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, &dest); + memset(dest, 0x00, 2 * sizeof(*dest)); + dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[0].ft = rx->ft.sa; + err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, dest); if (err) goto err_pol_miss; @@ -605,6 +615,7 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; @@ -647,8 +658,11 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; flow_act.flags |= FLOW_ACT_NO_APPEND; flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; - rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, NULL, 0); + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(tx->fc->cnt); + rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); @@ -674,12 +688,12 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) { struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); - struct mlx5_flow_destination dest = {}; + struct mlx5_flow_destination dest[2] = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; struct mlx5e_ipsec_tx *tx; - int err; + int err, dstn = 0; tx = tx_ft_get(mdev, pol_entry->ipsec); if (IS_ERR(tx)) @@ -703,7 +717,11 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; break; case XFRM_POLICY_BLOCK: - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); + dstn++; break; default: WARN_ON(true); @@ -712,9 +730,10 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) } flow_act.flags |= FLOW_ACT_NO_APPEND; - dest.ft = tx->ft.sa; - dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, &dest, 1); + dest[dstn].ft = tx->ft.sa; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dstn++; + rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, dest, dstn); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); @@ -736,12 +755,12 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) { struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); - struct mlx5_flow_destination dest = {}; + struct mlx5_flow_destination dest[2]; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; struct mlx5e_ipsec_rx *rx; - int err; + int err, dstn = 0; rx = rx_ft_get(mdev, pol_entry->ipsec, attrs->family); if (IS_ERR(rx)) @@ -765,7 +784,10 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; break; case XFRM_POLICY_BLOCK: - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); + dstn++; break; default: WARN_ON(true); @@ -774,9 +796,10 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) } flow_act.flags |= FLOW_ACT_NO_APPEND; - dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = rx->ft.sa; - rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, &dest, 1); + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[dstn].ft = rx->ft.sa; + dstn++; + rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, dest, dstn); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); @@ -794,6 +817,116 @@ err_alloc: return err; } +static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx *tx = ipsec->tx; + + mlx5_fc_destroy(mdev, tx->fc->drop); + mlx5_fc_destroy(mdev, tx->fc->cnt); + kfree(tx->fc); + mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); + mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); + kfree(rx_ipv4->fc); +} + +static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; + struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_fc *fc; + struct mlx5_fc *counter; + int err; + + fc = kzalloc(sizeof(*rx_ipv4->fc), GFP_KERNEL); + if (!fc) + return -ENOMEM; + + /* Both IPv4 and IPv6 point to same flow counters struct. */ + rx_ipv4->fc = fc; + rx_ipv6->fc = fc; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_rx_cnt; + } + + fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_rx_drop; + } + + fc->drop = counter; + fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); + if (!fc) { + err = -ENOMEM; + goto err_tx_fc; + } + + tx->fc = fc; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_tx_cnt; + } + + fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_tx_drop; + } + + fc->drop = counter; + return 0; + +err_tx_drop: + mlx5_fc_destroy(mdev, tx->fc->cnt); +err_tx_cnt: + kfree(tx->fc); +err_tx_fc: + mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); +err_rx_drop: + mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); +err_rx_cnt: + kfree(rx_ipv4->fc); + return err; +} + +void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5e_ipsec_hw_stats *stats; + struct mlx5e_ipsec_fc *fc; + + stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; + + stats->ipsec_rx_pkts = 0; + stats->ipsec_rx_bytes = 0; + stats->ipsec_rx_drop_pkts = 0; + stats->ipsec_rx_drop_bytes = 0; + stats->ipsec_tx_pkts = 0; + stats->ipsec_tx_bytes = 0; + stats->ipsec_tx_drop_pkts = 0; + stats->ipsec_tx_drop_bytes = 0; + + fc = ipsec->rx_ipv4->fc; + mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_rx_pkts, &stats->ipsec_rx_bytes); + mlx5_fc_query(mdev, fc->drop, &stats->ipsec_rx_drop_pkts, + &stats->ipsec_rx_drop_bytes); + + fc = ipsec->tx->fc; + mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); + mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, + &stats->ipsec_tx_drop_bytes); +} + int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) @@ -848,6 +981,7 @@ void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) if (!ipsec->tx) return; + ipsec_fs_destroy_counters(ipsec); mutex_destroy(&ipsec->tx->ft.mutex); WARN_ON(ipsec->tx->ft.refcnt); kfree(ipsec->tx); @@ -883,6 +1017,10 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) if (!ipsec->rx_ipv6) goto err_rx_ipv6; + err = ipsec_fs_init_counters(ipsec); + if (err) + goto err_counters; + mutex_init(&ipsec->tx->ft.mutex); mutex_init(&ipsec->rx_ipv4->ft.mutex); mutex_init(&ipsec->rx_ipv6->ft.mutex); @@ -890,6 +1028,8 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) return 0; +err_counters: + kfree(ipsec->rx_ipv6); err_rx_ipv6: kfree(ipsec->rx_ipv4); err_rx_ipv4: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c index 9de84821dafb..e0e36a09721c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c @@ -37,6 +37,17 @@ #include "en.h" #include "ipsec.h" +static const struct counter_desc mlx5e_ipsec_hw_stats_desc[] = { + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_drop_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_rx_drop_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_drop_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_hw_stats, ipsec_tx_drop_bytes) }, +}; + static const struct counter_desc mlx5e_ipsec_sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_sp_alloc) }, { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_sadb_miss) }, @@ -50,8 +61,48 @@ static const struct counter_desc mlx5e_ipsec_sw_stats_desc[] = { #define MLX5E_READ_CTR_ATOMIC64(ptr, dsc, i) \ atomic64_read((atomic64_t *)((char *)(ptr) + (dsc)[i].offset)) +#define NUM_IPSEC_HW_COUNTERS ARRAY_SIZE(mlx5e_ipsec_hw_stats_desc) #define NUM_IPSEC_SW_COUNTERS ARRAY_SIZE(mlx5e_ipsec_sw_stats_desc) +static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ipsec_hw) +{ + if (!priv->ipsec) + return 0; + + return NUM_IPSEC_HW_COUNTERS; +} + +static inline MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(ipsec_hw) {} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(ipsec_hw) +{ + unsigned int i; + + if (!priv->ipsec) + return idx; + + for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, + mlx5e_ipsec_hw_stats_desc[i].format); + + return idx; +} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(ipsec_hw) +{ + int i; + + if (!priv->ipsec) + return idx; + + mlx5e_accel_ipsec_fs_read_stats(priv, &priv->ipsec->hw_stats); + for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) + data[idx++] = MLX5E_READ_CTR_ATOMIC64(&priv->ipsec->hw_stats, + mlx5e_ipsec_hw_stats_desc, i); + + return idx; +} + static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ipsec_sw) { return priv->ipsec ? NUM_IPSEC_SW_COUNTERS : 0; @@ -81,4 +132,5 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(ipsec_sw) return idx; } +MLX5E_DEFINE_STATS_GRP(ipsec_hw, 0); MLX5E_DEFINE_STATS_GRP(ipsec_sw, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 70c4ea3841d7..6687b8136e44 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -2480,6 +2480,7 @@ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = { &MLX5E_STATS_GRP(per_prio), &MLX5E_STATS_GRP(pme), #ifdef CONFIG_MLX5_EN_IPSEC + &MLX5E_STATS_GRP(ipsec_hw), &MLX5E_STATS_GRP(ipsec_sw), #endif &MLX5E_STATS_GRP(tls), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index cbc831ca646b..712cac10ba49 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -490,6 +490,7 @@ extern MLX5E_DECLARE_STATS_GRP(per_prio); extern MLX5E_DECLARE_STATS_GRP(pme); extern MLX5E_DECLARE_STATS_GRP(channels); extern MLX5E_DECLARE_STATS_GRP(per_port_buff_congest); +extern MLX5E_DECLARE_STATS_GRP(ipsec_hw); extern MLX5E_DECLARE_STATS_GRP(ipsec_sw); extern MLX5E_DECLARE_STATS_GRP(ptp); extern MLX5E_DECLARE_STATS_GRP(macsec_hw); From 6721239672fe38187119afdafe996902ab24fd11 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:51 +0200 Subject: [PATCH 31/38] net/mlx5e: Skip IPsec encryption for TX path without matching policy Software implementation of IPsec skips encryption of packets in TX path if no matching policy is found. So align HW implementation to this behavior, by requiring matching reqid for offloaded policy and SA. Reviewed-by: Raed Salem Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 7 ++++ .../mellanox/mlx5/core/en_accel/ipsec.h | 4 +- .../mellanox/mlx5/core/en_accel/ipsec_fs.c | 39 ++++++++++++++++--- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 65f73a5c29ba..8f08dbf2206e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -184,6 +184,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); attrs->family = x->props.family; attrs->type = x->xso.type; + attrs->reqid = x->props.reqid; } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) @@ -267,6 +268,11 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) x->replay_esn->replay_window); return -EINVAL; } + + if (!x->props.reqid) { + netdev_info(netdev, "Cannot offload without reqid\n"); + return -EINVAL; + } } return 0; } @@ -503,6 +509,7 @@ mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, attrs->dir = x->xdo.dir; attrs->action = x->action; attrs->type = XFRM_DEV_OFFLOAD_PACKET; + attrs->reqid = x->xfrm_vec[0].reqid; } static int mlx5e_xfrm_add_policy(struct xfrm_policy *x) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 2e7654e90314..492be255d267 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -77,6 +77,7 @@ struct mlx5_accel_esp_xfrm_attrs { u8 family; u32 replay_window; u32 authsize; + u32 reqid; }; enum mlx5_ipsec_cap { @@ -178,12 +179,13 @@ struct mlx5_accel_pol_xfrm_attrs { u8 action; u8 type : 2; u8 dir : 2; + u32 reqid; }; struct mlx5e_ipsec_pol_entry { struct xfrm_policy *x; struct mlx5e_ipsec *ipsec; - struct mlx5_flow_handle *rule; + struct mlx5e_ipsec_rule ipsec_rule; struct mlx5_accel_pol_xfrm_attrs attrs; }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 8ba92b00afdb..9f19f4b59a70 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -456,6 +456,17 @@ static void setup_fte_reg_a(struct mlx5_flow_spec *spec) misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); } +static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) +{ + /* Pass policy check before choosing this SA */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_c_0, reqid); + MLX5_SET(fte_match_param, spec->match_value, + misc_parameters_2.metadata_reg_c_0, reqid); +} + static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, struct mlx5_flow_act *flow_act) { @@ -470,6 +481,11 @@ static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, MLX5_ACTION_IN_FIELD_METADATA_REG_B); ns_type = MLX5_FLOW_NAMESPACE_KERNEL; break; + case XFRM_DEV_OFFLOAD_OUT: + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); + ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + break; default: return -EINVAL; } @@ -646,6 +662,7 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) setup_fte_reg_a(spec); break; case XFRM_DEV_OFFLOAD_PACKET: + setup_fte_reg_c0(spec, attrs->reqid); err = setup_pkt_reformat(mdev, attrs, &flow_act); if (err) goto err_pkt_reformat; @@ -712,6 +729,11 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) setup_fte_no_frags(spec); + err = setup_modify_header(mdev, attrs->reqid, XFRM_DEV_OFFLOAD_OUT, + &flow_act); + if (err) + goto err_mod_header; + switch (attrs->action) { case XFRM_POLICY_ALLOW: flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; @@ -741,10 +763,13 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) } kvfree(spec); - pol_entry->rule = rule; + pol_entry->ipsec_rule.rule = rule; + pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; return 0; err_action: + mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); +err_mod_header: kvfree(spec); err_alloc: tx_ft_put(pol_entry->ipsec); @@ -807,7 +832,7 @@ static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) } kvfree(spec); - pol_entry->rule = rule; + pol_entry->ipsec_rule.rule = rule; return 0; err_action: @@ -964,16 +989,18 @@ int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) { + struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); - mlx5_del_flow_rules(pol_entry->rule); + mlx5_del_flow_rules(ipsec_rule->rule); - if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { - tx_ft_put(pol_entry->ipsec); + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { + rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family); return; } - rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family); + mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); + tx_ft_put(pol_entry->ipsec); } void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) From 7bddb659bd0fc4e9022d3a3a2260540c01856e68 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:52 +0200 Subject: [PATCH 32/38] net/mlx5e: Provide intermediate pointer to access IPsec struct Improve readability by providing direct pointer to struct mlx5e_ipsec. Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec_rxtx.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 6859f1c1a831..9f07e58f7737 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -312,23 +312,23 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, struct mlx5_cqe64 *cqe) { u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata); - struct mlx5e_priv *priv; + struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_ipsec *ipsec = priv->ipsec; struct xfrm_offload *xo; struct xfrm_state *xs; struct sec_path *sp; u32 sa_handle; sa_handle = MLX5_IPSEC_METADATA_HANDLE(ipsec_meta_data); - priv = netdev_priv(netdev); sp = secpath_set(skb); if (unlikely(!sp)) { - atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc); + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sp_alloc); return; } - xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle); + xs = mlx5e_ipsec_sadb_rx_lookup(ipsec, sa_handle); if (unlikely(!xs)) { - atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss); + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss); return; } @@ -349,6 +349,6 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, xo->status = CRYPTO_INVALID_PACKET_SYNTAX; break; default: - atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome); + atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_syndrome); } } From 403b383a3ce3ebe12c26317978c34dd3d653c867 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:53 +0200 Subject: [PATCH 33/38] net/mlx5e: Store all XFRM SAs in Xarray Instead of performing custom hash calculations, rely on FW that returns unique identifier to every created SA. That identifier is Xarray ready, which provides better semantic with efficient access. In addition, store both TX and RX SAs to allow correlation between event generated by HW when limits are armed and XFRM states. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 82 +++++-------------- .../mellanox/mlx5/core/en_accel/ipsec.h | 8 +- .../mellanox/mlx5/core/en_accel/ipsec_rxtx.c | 12 ++- 3 files changed, 28 insertions(+), 74 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 8f08dbf2206e..fe10f1a2a04a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -50,57 +50,6 @@ static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x) return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle; } -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, - unsigned int handle) -{ - struct mlx5e_ipsec_sa_entry *sa_entry; - struct xfrm_state *ret = NULL; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) - if (sa_entry->handle == handle) { - ret = sa_entry->x; - xfrm_state_hold(ret); - break; - } - rcu_read_unlock(); - - return ret; -} - -static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - unsigned int handle = sa_entry->ipsec_obj_id; - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5e_ipsec_sa_entry *_sa_entry; - unsigned long flags; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) - if (_sa_entry->handle == handle) { - rcu_read_unlock(); - return -EEXIST; - } - rcu_read_unlock(); - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - sa_entry->handle = handle; - hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); - - return 0; -} - -static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - unsigned long flags; - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - hash_del_rcu(&sa_entry->hlist); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); -} - static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) { struct xfrm_replay_state_esn *replay_esn; @@ -291,6 +240,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = NULL; struct net_device *netdev = x->xso.real_dev; + struct mlx5e_ipsec *ipsec; struct mlx5e_priv *priv; int err; @@ -298,6 +248,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (!priv->ipsec) return -EOPNOTSUPP; + ipsec = priv->ipsec; err = mlx5e_xfrm_validate_state(x); if (err) return err; @@ -309,7 +260,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) } sa_entry->x = x; - sa_entry->ipsec = priv->ipsec; + sa_entry->ipsec = ipsec; /* check esn */ mlx5e_ipsec_update_esn_state(sa_entry); @@ -324,18 +275,22 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (err) goto err_hw_ctx; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { - err = mlx5e_ipsec_sadb_rx_add(sa_entry); - if (err) - goto err_add_rule; - } else { + /* We use *_bh() variant because xfrm_timer_handler(), which runs + * in softirq context, can reach our state delete logic and we need + * xa_erase_bh() there. + */ + err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry, + GFP_KERNEL); + if (err) + goto err_add_rule; + + if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; - } INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); x->xso.offload_handle = (unsigned long)sa_entry; - goto out; + return 0; err_add_rule: mlx5e_accel_ipsec_fs_del_rule(sa_entry); @@ -350,9 +305,11 @@ out: static void mlx5e_xfrm_del_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_sa_entry *old; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) - mlx5e_ipsec_sadb_rx_del(sa_entry); + old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id); + WARN_ON(old != sa_entry); } static void mlx5e_xfrm_free_state(struct xfrm_state *x) @@ -379,8 +336,7 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) if (!ipsec) return; - hash_init(ipsec->sadb_rx); - spin_lock_init(&ipsec->sadb_rx_lock); + xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC); ipsec->mdev = priv->mdev; ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, priv->netdev->name); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 492be255d267..724f2df14a97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -120,8 +120,7 @@ struct mlx5e_ipsec_aso { struct mlx5e_ipsec { struct mlx5_core_dev *mdev; - DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); - spinlock_t sadb_rx_lock; /* Protects sadb_rx */ + struct xarray sadb; struct mlx5e_ipsec_sw_stats sw_stats; struct mlx5e_ipsec_hw_stats hw_stats; struct workqueue_struct *wq; @@ -150,9 +149,7 @@ struct mlx5e_ipsec_modify_state_work { }; struct mlx5e_ipsec_sa_entry { - struct hlist_node hlist; /* Item in SADB_RX hashtable */ struct mlx5e_ipsec_esn_state esn_state; - unsigned int handle; /* Handle in SADB_RX */ struct xfrm_state *x; struct mlx5e_ipsec *ipsec; struct mlx5_accel_esp_xfrm_attrs attrs; @@ -193,9 +190,6 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev, - unsigned int handle); - void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 9f07e58f7737..eab5bc718771 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -314,8 +314,8 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata); struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5e_ipsec_sa_entry *sa_entry; struct xfrm_offload *xo; - struct xfrm_state *xs; struct sec_path *sp; u32 sa_handle; @@ -326,13 +326,17 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, return; } - xs = mlx5e_ipsec_sadb_rx_lookup(ipsec, sa_handle); - if (unlikely(!xs)) { + rcu_read_lock(); + sa_entry = xa_load(&ipsec->sadb, sa_handle); + if (unlikely(!sa_entry)) { + rcu_read_unlock(); atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss); return; } + xfrm_state_hold(sa_entry->x); + rcu_read_unlock(); - sp->xvec[sp->len++] = xs; + sp->xvec[sp->len++] = sa_entry->x; sp->olen++; xo = xfrm_offload(skb); From 1ed78fc033074c55221a80498204c539a3696877 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:54 +0200 Subject: [PATCH 34/38] net/mlx5e: Update IPsec soft and hard limits Implement mlx5 IPsec callback to update current lifetime counters. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 63 +++++++++++++++++++ .../mellanox/mlx5/core/en_accel/ipsec.h | 6 ++ .../mlx5/core/en_accel/ipsec_offload.c | 57 +++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/lib/aso.h | 1 + 4 files changed, 127 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index fe10f1a2a04a..8d0c605d4cdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -83,6 +83,31 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) return false; } +static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + struct xfrm_state *x = sa_entry->x; + + attrs->hard_packet_limit = x->lft.hard_packet_limit; + if (x->lft.soft_packet_limit == XFRM_INF) + return; + + /* Hardware decrements hard_packet_limit counter through + * the operation. While fires an event when soft_packet_limit + * is reached. It emans that we need substitute the numbers + * in order to properly count soft limit. + * + * As an example: + * XFRM user sets soft limit is 2 and hard limit is 9 and + * expects to see soft event after 2 packets and hard event + * after 9 packets. In our case, the hard limit will be set + * to 9 and soft limit is comparator to 7 so user gets the + * soft event after 2 packeta + */ + attrs->soft_packet_limit = + x->lft.hard_packet_limit - x->lft.soft_packet_limit; +} + static void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) @@ -134,6 +159,8 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->family = x->props.family; attrs->type = x->xso.type; attrs->reqid = x->props.reqid; + + mlx5e_ipsec_init_limits(sa_entry, attrs); } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) @@ -222,6 +249,21 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Cannot offload without reqid\n"); return -EINVAL; } + + if (x->lft.hard_byte_limit != XFRM_INF || + x->lft.soft_byte_limit != XFRM_INF) { + netdev_info(netdev, + "Device doesn't support limits in bytes\n"); + return -EINVAL; + } + + if (x->lft.soft_packet_limit >= x->lft.hard_packet_limit && + x->lft.hard_packet_limit != XFRM_INF) { + /* XFRM stack doesn't prevent such configuration :(. */ + netdev_info(netdev, + "Hard packet limit must be greater than soft one\n"); + return -EINVAL; + } } return 0; } @@ -415,6 +457,26 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) queue_work(sa_entry->ipsec->wq, &modify_work->work); } +static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) +{ + struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + int err; + + lockdep_assert_held(&x->lock); + + if (sa_entry->attrs.soft_packet_limit == XFRM_INF) + /* Limits are not configured, as soft limit + * must be lowever than hard limit. + */ + return; + + err = mlx5e_ipsec_aso_query(sa_entry); + if (err) + return; + + mlx5e_ipsec_aso_update_curlft(sa_entry, &x->curlft.packets); +} + static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x) { struct net_device *netdev = x->xdo.real_dev; @@ -526,6 +588,7 @@ static const struct xfrmdev_ops mlx5e_ipsec_packet_xfrmdev_ops = { .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, + .xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft, .xdo_dev_policy_add = mlx5e_xfrm_add_policy, .xdo_dev_policy_free = mlx5e_xfrm_free_policy, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 724f2df14a97..aac1e6a83631 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -78,6 +78,8 @@ struct mlx5_accel_esp_xfrm_attrs { u32 replay_window; u32 authsize; u32 reqid; + u64 hard_packet_limit; + u64 soft_packet_limit; }; enum mlx5_ipsec_cap { @@ -208,6 +210,10 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry); +void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, + u64 *packets); + void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index fc88454aaf8d..8790558ea859 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -83,6 +83,20 @@ static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn, MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5); if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN); + + if (attrs->hard_packet_limit != XFRM_INF) { + MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt, + lower_32_bits(attrs->hard_packet_limit)); + MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1); + MLX5_SET(ipsec_aso, aso_ctx, remove_flow_enable, 1); + } + + if (attrs->soft_packet_limit != XFRM_INF) { + MLX5_SET(ipsec_aso, aso_ctx, remove_flow_soft_lft, + lower_32_bits(attrs->soft_packet_limit)); + + MLX5_SET(ipsec_aso, aso_ctx, soft_lft_arm, 1); + } } static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry) @@ -298,3 +312,46 @@ void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec) DMA_BIDIRECTIONAL); kfree(aso); } + +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) +{ + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_aso *aso = ipsec->aso; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5_wqe_aso_ctrl_seg *ctrl; + struct mlx5e_hw_objs *res; + struct mlx5_aso_wqe *wqe; + u8 ds_cnt; + + res = &mdev->mlx5e_res.hw_objs; + + memset(aso->ctx, 0, sizeof(aso->ctx)); + wqe = mlx5_aso_get_wqe(aso->aso); + ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS); + mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id, + MLX5_ACCESS_ASO_OPC_MOD_IPSEC); + + ctrl = &wqe->aso_ctrl; + ctrl->va_l = + cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN); + ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr)); + ctrl->l_key = cpu_to_be32(res->mkey); + + mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl); + return mlx5_aso_poll_cq(aso->aso, false); +} + +void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, + u64 *packets) +{ + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_aso *aso = ipsec->aso; + u64 hard_cnt; + + hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt); + /* HW decresases the limit till it reaches zero to fire an avent. + * We need to fix the calculations, so the returned count is a total + * number of passed packets and not how much left. + */ + *packets = sa_entry->attrs.hard_packet_limit - hard_cnt; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h index c8fc3c838642..afb078bbb8ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -15,6 +15,7 @@ #define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24 #define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS)) +#define ASO_CTRL_READ_EN BIT(0) struct mlx5_wqe_aso_ctrl_seg { __be32 va_h; __be32 va_l; /* include read_enable */ From 8c582ddfbb473c1d799c40b5140aed81278e2837 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:55 +0200 Subject: [PATCH 35/38] net/mlx5e: Handle hardware IPsec limits events Enable object changed event to signal IPsec about hitting soft and hard limits. Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 2 +- .../mellanox/mlx5/core/en_accel/ipsec.h | 19 +++- .../mlx5/core/en_accel/ipsec_offload.c | 97 ++++++++++++++++++- drivers/net/ethernet/mellanox/mlx5/core/eq.c | 5 + 4 files changed, 118 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 8d0c605d4cdb..4f176bd8395a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -470,7 +470,7 @@ static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) */ return; - err = mlx5e_ipsec_aso_query(sa_entry); + err = mlx5e_ipsec_aso_query(sa_entry, NULL); if (err) return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index aac1e6a83631..e7f21e449268 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -34,8 +34,6 @@ #ifndef __MLX5E_IPSEC_H__ #define __MLX5E_IPSEC_H__ -#ifdef CONFIG_MLX5_EN_IPSEC - #include #include #include @@ -114,10 +112,21 @@ struct mlx5e_ipsec_sw_stats { struct mlx5e_ipsec_rx; struct mlx5e_ipsec_tx; +struct mlx5e_ipsec_work { + struct work_struct work; + struct mlx5e_ipsec *ipsec; + u32 id; +}; + struct mlx5e_ipsec_aso { u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; dma_addr_t dma_addr; struct mlx5_aso *aso; + /* IPsec ASO caches data on every query call, + * so in nested calls, we can use this boolean to save + * recursive calls to mlx5e_ipsec_aso_query() + */ + u8 use_cache : 1; }; struct mlx5e_ipsec { @@ -131,6 +140,7 @@ struct mlx5e_ipsec { struct mlx5e_ipsec_rx *rx_ipv6; struct mlx5e_ipsec_tx *tx; struct mlx5e_ipsec_aso *aso; + struct notifier_block nb; }; struct mlx5e_ipsec_esn_state { @@ -188,6 +198,8 @@ struct mlx5e_ipsec_pol_entry { struct mlx5_accel_pol_xfrm_attrs attrs; }; +#ifdef CONFIG_MLX5_EN_IPSEC + void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); @@ -210,7 +222,8 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); -int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data); void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, u64 *packets); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 8790558ea859..1b5014ffa257 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -260,6 +260,73 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs)); } +static void mlx5e_ipsec_handle_event(struct work_struct *_work) +{ + struct mlx5e_ipsec_work *work = + container_of(_work, struct mlx5e_ipsec_work, work); + struct mlx5_accel_esp_xfrm_attrs *attrs; + struct mlx5e_ipsec_sa_entry *sa_entry; + struct mlx5e_ipsec_aso *aso; + struct mlx5e_ipsec *ipsec; + int ret; + + sa_entry = xa_load(&work->ipsec->sadb, work->id); + if (!sa_entry) + goto out; + + ipsec = sa_entry->ipsec; + aso = ipsec->aso; + attrs = &sa_entry->attrs; + + spin_lock(&sa_entry->x->lock); + ret = mlx5e_ipsec_aso_query(sa_entry, NULL); + if (ret) + goto unlock; + + aso->use_cache = true; + if (attrs->soft_packet_limit != XFRM_INF) + if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) || + !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) || + !MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable)) + xfrm_state_check_expire(sa_entry->x); + aso->use_cache = false; + +unlock: + spin_unlock(&sa_entry->x->lock); +out: + kfree(work); +} + +static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event, + void *data) +{ + struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb); + struct mlx5_eqe_obj_change *object; + struct mlx5e_ipsec_work *work; + struct mlx5_eqe *eqe = data; + u16 type; + + if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE) + return NOTIFY_DONE; + + object = &eqe->data.obj_change; + type = be16_to_cpu(object->obj_type); + + if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5e_ipsec_handle_event); + work->ipsec = ipsec; + work->id = be32_to_cpu(object->obj_id); + + queue_work(ipsec->wq, &work->work); + return NOTIFY_OK; +} + int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec) { struct mlx5_core_dev *mdev = ipsec->mdev; @@ -287,6 +354,9 @@ int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec) goto err_aso_create; } + ipsec->nb.notifier_call = mlx5e_ipsec_event; + mlx5_notifier_register(mdev, &ipsec->nb); + ipsec->aso = aso; return 0; @@ -307,13 +377,33 @@ void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec) aso = ipsec->aso; pdev = mlx5_core_dma_dev(mdev); + mlx5_notifier_unregister(mdev, &ipsec->nb); mlx5_aso_destroy(aso->aso); dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL); kfree(aso); } -int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) +static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl, + struct mlx5_wqe_aso_ctrl_seg *data) +{ + if (!data) + return; + + ctrl->data_mask_mode = data->data_mask_mode; + ctrl->condition_1_0_operand = data->condition_1_0_operand; + ctrl->condition_1_0_offset = data->condition_1_0_offset; + ctrl->data_offset_condition_operand = data->data_offset_condition_operand; + ctrl->condition_0_data = data->condition_0_data; + ctrl->condition_0_mask = data->condition_0_mask; + ctrl->condition_1_data = data->condition_1_data; + ctrl->condition_1_mask = data->condition_1_mask; + ctrl->bitwise_data = data->bitwise_data; + ctrl->data_mask = data->data_mask; +} + +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data) { struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5e_ipsec_aso *aso = ipsec->aso; @@ -323,6 +413,10 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) struct mlx5_aso_wqe *wqe; u8 ds_cnt; + lockdep_assert_held(&sa_entry->x->lock); + if (aso->use_cache) + return 0; + res = &mdev->mlx5e_res.hw_objs; memset(aso->ctx, 0, sizeof(aso->ctx)); @@ -336,6 +430,7 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN); ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr)); ctrl->l_key = cpu_to_be32(res->mkey); + mlx5e_ipsec_aso_copy(ctrl, data); mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl); return mlx5_aso_poll_cq(aso->aso, false); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index a0242dc15741..8f7580fec193 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -19,6 +19,7 @@ #include "diag/fw_tracer.h" #include "mlx5_irq.h" #include "devlink.h" +#include "en_accel/ipsec.h" enum { MLX5_EQE_OWNER_INIT_VAL = 0x1, @@ -578,6 +579,10 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + async_event_mask |= + (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + mask[0] = async_event_mask; if (MLX5_CAP_GEN(dev, event_cap)) From cee137a634318bfbda18ee5af45d300153b57fa5 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:56 +0200 Subject: [PATCH 36/38] net/mlx5e: Handle ESN update events Extend event logic to update ESN state (esn_msb, esn_overlap) for an IPsec Offload context. Reviewed-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 5 +-- .../mellanox/mlx5/core/en_accel/ipsec.h | 2 + .../mlx5/core/en_accel/ipsec_offload.c | 44 +++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 4f176bd8395a..f5f930ea3f0f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -108,9 +108,8 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, x->lft.hard_packet_limit - x->lft.soft_packet_limit; } -static void -mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, - struct mlx5_accel_esp_xfrm_attrs *attrs) +void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs) { struct xfrm_state *x = sa_entry->x; struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index e7f21e449268..a92e19c4c499 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -230,6 +230,8 @@ void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats); +void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs); static inline struct mlx5_core_dev * mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 1b5014ffa257..8e3614218fc4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -6,6 +6,10 @@ #include "ipsec.h" #include "lib/mlx5.h" +enum { + MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET, +}; + u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev) { u32 caps = 0; @@ -260,6 +264,39 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs)); } +static void +mlx5e_ipsec_aso_update_esn(struct mlx5e_ipsec_sa_entry *sa_entry, + const struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + struct mlx5_wqe_aso_ctrl_seg data = {}; + + data.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6; + data.condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE + << 4; + data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET; + data.bitwise_data = cpu_to_be64(BIT_ULL(54)); + data.data_mask = data.bitwise_data; + + mlx5e_ipsec_aso_query(sa_entry, &data); +} + +static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry, + u32 mode_param) +{ + struct mlx5_accel_esp_xfrm_attrs attrs = {}; + + if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) { + sa_entry->esn_state.esn++; + sa_entry->esn_state.overlap = 0; + } else { + sa_entry->esn_state.overlap = 1; + } + + mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs); + mlx5_accel_esp_modify_xfrm(sa_entry, &attrs); + mlx5e_ipsec_aso_update_esn(sa_entry, &attrs); +} + static void mlx5e_ipsec_handle_event(struct work_struct *_work) { struct mlx5e_ipsec_work *work = @@ -284,6 +321,13 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) goto unlock; aso->use_cache = true; + if (attrs->esn_trigger && + !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) { + u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter); + + mlx5e_ipsec_update_esn_state(sa_entry, mode_param); + } + if (attrs->soft_packet_limit != XFRM_INF) if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) || !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) || From 37d244ad183f42f91910e47984a7636e970e9d1b Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 2 Dec 2022 22:14:57 +0200 Subject: [PATCH 37/38] net/mlx5e: Open mlx5 driver to accept IPsec packet offload Enable configuration of IPsec packet offload through XFRM state add interface together with moving specific to IPsec packet mode limitations to specific switch-case section. Reviewed-by: Raed Salem Signed-off-by: Leon Romanovsky Signed-off-by: Steffen Klassert --- .../mellanox/mlx5/core/en_accel/ipsec.c | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index f5f930ea3f0f..bb9023957f74 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -191,11 +191,6 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n"); return -EINVAL; } - if (x->props.mode != XFRM_MODE_TRANSPORT && - x->props.mode != XFRM_MODE_TUNNEL) { - dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n"); - return -EINVAL; - } if (x->id.proto != IPPROTO_ESP) { netdev_info(netdev, "Only ESP xfrm state may be offloaded\n"); return -EINVAL; @@ -229,11 +224,32 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } - if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { - netdev_info(netdev, "Unsupported xfrm offload type\n"); - return -EINVAL; - } - if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) { + switch (x->xso.type) { + case XFRM_DEV_OFFLOAD_CRYPTO: + if (!(mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_CRYPTO)) { + netdev_info(netdev, "Crypto offload is not supported\n"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT && + x->props.mode != XFRM_MODE_TUNNEL) { + netdev_info(netdev, "Only transport and tunnel xfrm states may be offloaded\n"); + return -EINVAL; + } + break; + case XFRM_DEV_OFFLOAD_PACKET: + if (!(mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { + netdev_info(netdev, "Packet offload is not supported\n"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT) { + netdev_info(netdev, "Only transport xfrm states may be offloaded in packet mode\n"); + return -EINVAL; + } + if (x->replay_esn && x->replay_esn->replay_window != 32 && x->replay_esn->replay_window != 64 && x->replay_esn->replay_window != 128 && @@ -263,6 +279,11 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) "Hard packet limit must be greater than soft one\n"); return -EINVAL; } + break; + default: + netdev_info(netdev, "Unsupported xfrm offload type %d\n", + x->xso.type); + return -EINVAL; } return 0; } From abe2343d37c2b4361547d5d31e17340ff9ec7356 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 7 Dec 2022 09:23:14 +0000 Subject: [PATCH 38/38] xfrm: Fix spelling mistake "oflload" -> "offload" There is a spelling mistake in a NL_SET_ERR_MSG message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Steffen Klassert --- net/xfrm/xfrm_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 3e9e874522a8..4aff76c6f12e 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -379,7 +379,7 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, default: xdo->dev = NULL; dev_put(dev); - NL_SET_ERR_MSG(extack, "Unrecognized oflload direction"); + NL_SET_ERR_MSG(extack, "Unrecognized offload direction"); return -EINVAL; }