diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 8f08dbf2206e..fe10f1a2a04a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -50,57 +50,6 @@ static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x) return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle; } -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, - unsigned int handle) -{ - struct mlx5e_ipsec_sa_entry *sa_entry; - struct xfrm_state *ret = NULL; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) - if (sa_entry->handle == handle) { - ret = sa_entry->x; - xfrm_state_hold(ret); - break; - } - rcu_read_unlock(); - - return ret; -} - -static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - unsigned int handle = sa_entry->ipsec_obj_id; - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5e_ipsec_sa_entry *_sa_entry; - unsigned long flags; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) - if (_sa_entry->handle == handle) { - rcu_read_unlock(); - return -EEXIST; - } - rcu_read_unlock(); - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - sa_entry->handle = handle; - hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); - - return 0; -} - -static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - unsigned long flags; - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - hash_del_rcu(&sa_entry->hlist); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); -} - static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) { struct xfrm_replay_state_esn *replay_esn; @@ -291,6 +240,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = NULL; struct net_device *netdev = x->xso.real_dev; + struct mlx5e_ipsec *ipsec; struct mlx5e_priv *priv; int err; @@ -298,6 +248,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (!priv->ipsec) return -EOPNOTSUPP; + ipsec = priv->ipsec; err = mlx5e_xfrm_validate_state(x); if (err) return err; @@ -309,7 +260,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) } sa_entry->x = x; - sa_entry->ipsec = priv->ipsec; + sa_entry->ipsec = ipsec; /* check esn */ mlx5e_ipsec_update_esn_state(sa_entry); @@ -324,18 +275,22 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (err) goto err_hw_ctx; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { - err = mlx5e_ipsec_sadb_rx_add(sa_entry); - if (err) - goto err_add_rule; - } else { + /* We use *_bh() variant because xfrm_timer_handler(), which runs + * in softirq context, can reach our state delete logic and we need + * xa_erase_bh() there. + */ + err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry, + GFP_KERNEL); + if (err) + goto err_add_rule; + + if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; - } INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); x->xso.offload_handle = (unsigned long)sa_entry; - goto out; + return 0; err_add_rule: mlx5e_accel_ipsec_fs_del_rule(sa_entry); @@ -350,9 +305,11 @@ out: static void mlx5e_xfrm_del_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_sa_entry *old; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) - mlx5e_ipsec_sadb_rx_del(sa_entry); + old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id); + WARN_ON(old != sa_entry); } static void mlx5e_xfrm_free_state(struct xfrm_state *x) @@ -379,8 +336,7 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) if (!ipsec) return; - hash_init(ipsec->sadb_rx); - spin_lock_init(&ipsec->sadb_rx_lock); + xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC); ipsec->mdev = priv->mdev; ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, priv->netdev->name); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 492be255d267..724f2df14a97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -120,8 +120,7 @@ struct mlx5e_ipsec_aso { struct mlx5e_ipsec { struct mlx5_core_dev *mdev; - DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); - spinlock_t sadb_rx_lock; /* Protects sadb_rx */ + struct xarray sadb; struct mlx5e_ipsec_sw_stats sw_stats; struct mlx5e_ipsec_hw_stats hw_stats; struct workqueue_struct *wq; @@ -150,9 +149,7 @@ struct mlx5e_ipsec_modify_state_work { }; struct mlx5e_ipsec_sa_entry { - struct hlist_node hlist; /* Item in SADB_RX hashtable */ struct mlx5e_ipsec_esn_state esn_state; - unsigned int handle; /* Handle in SADB_RX */ struct xfrm_state *x; struct mlx5e_ipsec *ipsec; struct mlx5_accel_esp_xfrm_attrs attrs; @@ -193,9 +190,6 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev, - unsigned int handle); - void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 9f07e58f7737..eab5bc718771 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -314,8 +314,8 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata); struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5e_ipsec_sa_entry *sa_entry; struct xfrm_offload *xo; - struct xfrm_state *xs; struct sec_path *sp; u32 sa_handle; @@ -326,13 +326,17 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, return; } - xs = mlx5e_ipsec_sadb_rx_lookup(ipsec, sa_handle); - if (unlikely(!xs)) { + rcu_read_lock(); + sa_entry = xa_load(&ipsec->sadb, sa_handle); + if (unlikely(!sa_entry)) { + rcu_read_unlock(); atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss); return; } + xfrm_state_hold(sa_entry->x); + rcu_read_unlock(); - sp->xvec[sp->len++] = xs; + sp->xvec[sp->len++] = sa_entry->x; sp->olen++; xo = xfrm_offload(skb);