iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

netfilter: x_tables: improve limit_mt scalability

Browse Source 
We've seen this spin_lock show up high in profiles. Let's introduce a
lockless version. I've tested this using pktgen_sample01_simple.sh.

Signed-off-by: Jason Baron <jbaron@akamai.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
This commit is contained in:
Jason Baron 2021-04-29 12:26:13 -04:00 committed by Pablo Neira Ayuso
parent 06f0299302
commit 07df3fc90a
1 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
net/netfilter/xt_limit.c
View File

@ -8,16 +8,14 @@
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/skbuff.h> #include <linux/skbuff.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h> #include <linux/interrupt.h>
#include <linux/netfilter/x_tables.h> #include <linux/netfilter/x_tables.h>
#include <linux/netfilter/xt_limit.h> #include <linux/netfilter/xt_limit.h>
struct xt_limit_priv { struct xt_limit_priv {
spinlock_t lock;
unsigned long prev; unsigned long prev;
uint32_t credit; u32 credit;
}; };
MODULE_LICENSE("GPL"); MODULE_LICENSE("GPL");
@ -66,22 +64,31 @@ limit_mt(const struct sk_buff *skb, struct xt_action_param *par)
{ {
const struct xt_rateinfo *r = par->matchinfo; const struct xt_rateinfo *r = par->matchinfo;
struct xt_limit_priv *priv = r->master; struct xt_limit_priv *priv = r->master;
unsigned long now = jiffies; unsigned long now;
u32 old_credit, new_credit, credit_increase = 0;
bool ret;
spin_lock_bh(&priv->lock); /* fastpath if there is nothing to update */
priv->credit += (now - xchg(&priv->prev, now)) * CREDITS_PER_JIFFY; if ((READ_ONCE(priv->credit) < r->cost) && (READ_ONCE(priv->prev) == jiffies))
if (priv->credit > r->credit_cap) return false;
priv->credit = r->credit_cap;
if (priv->credit >= r->cost) { do {
/* We're not limited. */ now = jiffies;
priv->credit -= r->cost; credit_increase += (now - xchg(&priv->prev, now)) * CREDITS_PER_JIFFY;
spin_unlock_bh(&priv->lock); old_credit = READ_ONCE(priv->credit);
return true; new_credit = old_credit;
} new_credit += credit_increase;
if (new_credit > r->credit_cap)
new_credit = r->credit_cap;
if (new_credit >= r->cost) {
ret = true;
new_credit -= r->cost;
} else {
ret = false;
}
} while (cmpxchg(&priv->credit, old_credit, new_credit) != old_credit);
spin_unlock_bh(&priv->lock); return ret;
return false;
} }
/* Precision saver. */ /* Precision saver. */
@ -122,7 +129,6 @@ static int limit_mt_check(const struct xt_mtchk_param *par)
r->credit_cap = priv->credit; /* Credits full. */ r->credit_cap = priv->credit; /* Credits full. */
r->cost = user2credits(r->avg); r->cost = user2credits(r->avg);
} }
spin_lock_init(&priv->lock);
return 0; return 0;
} }