iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'ip-faster-in-order-IP-fragments'

Browse Source 
Peter Oskolkov says:

====================
ip: faster in-order IP fragments

Added "Signed-off-by" in v2.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2018-08-11 17:54:18 -07:00
commit 78cbac647e
3 changed files with 149 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/net/inet_frag.h
View File

@ -57,7 +57,9 @@ struct frag_v6_compare_key {
* @lock: spinlock protecting this frag * @lock: spinlock protecting this frag
* @refcnt: reference count of the queue * @refcnt: reference count of the queue
* @fragments: received fragments head * @fragments: received fragments head
* @rb_fragments: received fragments rb-tree root
* @fragments_tail: received fragments tail * @fragments_tail: received fragments tail
* @last_run_head: the head of the last "run". see ip_fragment.c
* @stamp: timestamp of the last received fragment * @stamp: timestamp of the last received fragment
* @len: total length of the original datagram * @len: total length of the original datagram
* @meat: length of received fragments so far * @meat: length of received fragments so far
@ -78,6 +80,7 @@ struct inet_frag_queue {
struct sk_buff *fragments; /* Used in IPv6. */ struct sk_buff *fragments; /* Used in IPv6. */
struct rb_root rb_fragments; /* Used in IPv4. */ struct rb_root rb_fragments; /* Used in IPv4. */
struct sk_buff *fragments_tail; struct sk_buff *fragments_tail;
struct sk_buff *last_run_head;
ktime_t stamp; ktime_t stamp;
int len; int len;
int meat; int meat;
@ -113,6 +116,9 @@ void inet_frag_kill(struct inet_frag_queue *q);
void inet_frag_destroy(struct inet_frag_queue *q); void inet_frag_destroy(struct inet_frag_queue *q);
struct inet_frag_queue *inet_frag_find(struct netns_frags *nf, void *key); struct inet_frag_queue *inet_frag_find(struct netns_frags *nf, void *key);
/* Free all skbs in the queue; return the sum of their truesizes. */
unsigned int inet_frag_rbtree_purge(struct rb_root *root);
static inline void inet_frag_put(struct inet_frag_queue *q) static inline void inet_frag_put(struct inet_frag_queue *q)
{ {
if (refcount_dec_and_test(&q->refcnt)) if (refcount_dec_and_test(&q->refcnt))

2
net/ipv4/inet_fragment.c
View File

@ -146,7 +146,7 @@ void inet_frag_destroy(struct inet_frag_queue *q)
fp = xp; fp = xp;
} while (fp); } while (fp);
} else { } else {
sum_truesize = skb_rbtree_purge(&q->rb_fragments); sum_truesize = inet_frag_rbtree_purge(&q->rb_fragments);
} }
sum = sum_truesize + f->qsize; sum = sum_truesize + f->qsize;

159
net/ipv4/ip_fragment.c
View File

@ -57,6 +57,57 @@
*/ */
static const char ip_frag_cache_name[] = "ip4-frags"; static const char ip_frag_cache_name[] = "ip4-frags";
/* Use skb->cb to track consecutive/adjacent fragments coming at
* the end of the queue. Nodes in the rb-tree queue will
* contain "runs" of one or more adjacent fragments.
*
* Invariants:
* - next_frag is NULL at the tail of a "run";
* - the head of a "run" has the sum of all fragment lengths in frag_run_len.
*/
struct ipfrag_skb_cb {
struct inet_skb_parm h;
struct sk_buff *next_frag;
int frag_run_len;
};
#define FRAG_CB(skb) ((struct ipfrag_skb_cb *)((skb)->cb))
static void ip4_frag_init_run(struct sk_buff *skb)
{
BUILD_BUG_ON(sizeof(struct ipfrag_skb_cb) > sizeof(skb->cb));
FRAG_CB(skb)->next_frag = NULL;
FRAG_CB(skb)->frag_run_len = skb->len;
}
/* Append skb to the last "run". */
static void ip4_frag_append_to_last_run(struct inet_frag_queue *q,
struct sk_buff *skb)
{
RB_CLEAR_NODE(&skb->rbnode);
FRAG_CB(skb)->next_frag = NULL;
FRAG_CB(q->last_run_head)->frag_run_len += skb->len;
FRAG_CB(q->fragments_tail)->next_frag = skb;
q->fragments_tail = skb;
}
/* Create a new "run" with the skb. */
static void ip4_frag_create_run(struct inet_frag_queue *q, struct sk_buff *skb)
{
if (q->last_run_head)
rb_link_node(&skb->rbnode, &q->last_run_head->rbnode,
&q->last_run_head->rbnode.rb_right);
else
rb_link_node(&skb->rbnode, NULL, &q->rb_fragments.rb_node);
rb_insert_color(&skb->rbnode, &q->rb_fragments);
ip4_frag_init_run(skb);
q->fragments_tail = skb;
q->last_run_head = skb;
}
/* Describe an entry in the "incomplete datagrams" queue. */ /* Describe an entry in the "incomplete datagrams" queue. */
struct ipq { struct ipq {
struct inet_frag_queue q; struct inet_frag_queue q;
@ -75,8 +126,8 @@ static u8 ip4_frag_ecn(u8 tos)
static struct inet_frags ip4_frags; static struct inet_frags ip4_frags;
static int ip_frag_reasm(struct ipq *qp, struct sk_buff *prev, static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
struct net_device *dev); struct sk_buff *prev_tail, struct net_device *dev);
static void ip4_frag_init(struct inet_frag_queue *q, const void *a) static void ip4_frag_init(struct inet_frag_queue *q, const void *a)
@ -168,6 +219,11 @@ static void ip_expire(struct timer_list *t)
head = skb_rb_first(&qp->q.rb_fragments); head = skb_rb_first(&qp->q.rb_fragments);
if (!head) if (!head)
goto out; goto out;
if (FRAG_CB(head)->next_frag)
rb_replace_node(&head->rbnode,
&FRAG_CB(head)->next_frag->rbnode,
&qp->q.rb_fragments);
else
rb_erase(&head->rbnode, &qp->q.rb_fragments); rb_erase(&head->rbnode, &qp->q.rb_fragments);
memset(&head->rbnode, 0, sizeof(head->rbnode)); memset(&head->rbnode, 0, sizeof(head->rbnode));
barrier(); barrier();
@ -269,7 +325,7 @@ static int ip_frag_reinit(struct ipq *qp)
return -ETIMEDOUT; return -ETIMEDOUT;
} }
sum_truesize = skb_rbtree_purge(&qp->q.rb_fragments); sum_truesize = inet_frag_rbtree_purge(&qp->q.rb_fragments);
sub_frag_mem_limit(qp->q.net, sum_truesize); sub_frag_mem_limit(qp->q.net, sum_truesize);
qp->q.flags = 0; qp->q.flags = 0;
@ -278,6 +334,7 @@ static int ip_frag_reinit(struct ipq *qp)
qp->q.fragments = NULL; qp->q.fragments = NULL;
qp->q.rb_fragments = RB_ROOT; qp->q.rb_fragments = RB_ROOT;
qp->q.fragments_tail = NULL; qp->q.fragments_tail = NULL;
qp->q.last_run_head = NULL;
qp->iif = 0; qp->iif = 0;
qp->ecn = 0; qp->ecn = 0;
@ -289,7 +346,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb)
{ {
struct net *net = container_of(qp->q.net, struct net, ipv4.frags); struct net *net = container_of(qp->q.net, struct net, ipv4.frags);
struct rb_node **rbn, *parent; struct rb_node **rbn, *parent;
struct sk_buff *skb1; struct sk_buff *skb1, *prev_tail;
struct net_device *dev; struct net_device *dev;
unsigned int fragsize; unsigned int fragsize;
int flags, offset; int flags, offset;
@ -367,38 +424,41 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb)
*/ */
/* Find out where to put this fragment. */ /* Find out where to put this fragment. */
skb1 = qp->q.fragments_tail; prev_tail = qp->q.fragments_tail;
if (!skb1) { if (!prev_tail)
/* This is the first fragment we've received. */ ip4_frag_create_run(&qp->q, skb); /* First fragment. */
rb_link_node(&skb->rbnode, NULL, &qp->q.rb_fragments.rb_node); else if (prev_tail->ip_defrag_offset + prev_tail->len < end) {
qp->q.fragments_tail = skb; /* This is the common case: skb goes to the end. */
} else if ((skb1->ip_defrag_offset + skb1->len) < end) {
/* This is the common/special case: skb goes to the end. */
/* Detect and discard overlaps. */ /* Detect and discard overlaps. */
if (offset < (skb1->ip_defrag_offset + skb1->len)) if (offset < prev_tail->ip_defrag_offset + prev_tail->len)
goto discard_qp; goto discard_qp;
/* Insert after skb1. */ if (offset == prev_tail->ip_defrag_offset + prev_tail->len)
rb_link_node(&skb->rbnode, &skb1->rbnode, &skb1->rbnode.rb_right); ip4_frag_append_to_last_run(&qp->q, skb);
qp->q.fragments_tail = skb; else
ip4_frag_create_run(&qp->q, skb);
} else { } else {
/* Binary search. Note that skb can become the first fragment, but /* Binary search. Note that skb can become the first fragment,
* not the last (covered above). */ * but not the last (covered above).
*/
rbn = &qp->q.rb_fragments.rb_node; rbn = &qp->q.rb_fragments.rb_node;
do { do {
parent = *rbn; parent = *rbn;
skb1 = rb_to_skb(parent); skb1 = rb_to_skb(parent);
if (end <= skb1->ip_defrag_offset) if (end <= skb1->ip_defrag_offset)
rbn = &parent->rb_left; rbn = &parent->rb_left;
else if (offset >= skb1->ip_defrag_offset + skb1->len) else if (offset >= skb1->ip_defrag_offset +
FRAG_CB(skb1)->frag_run_len)
rbn = &parent->rb_right; rbn = &parent->rb_right;
else /* Found an overlap with skb1. */ else /* Found an overlap with skb1. */
goto discard_qp; goto discard_qp;
} while (*rbn); } while (*rbn);
/* Here we have parent properly set, and rbn pointing to /* Here we have parent properly set, and rbn pointing to
* one of its NULL left/right children. Insert skb. */ * one of its NULL left/right children. Insert skb.
*/
ip4_frag_init_run(skb);
rb_link_node(&skb->rbnode, parent, rbn); rb_link_node(&skb->rbnode, parent, rbn);
}
rb_insert_color(&skb->rbnode, &qp->q.rb_fragments); rb_insert_color(&skb->rbnode, &qp->q.rb_fragments);
}
if (dev) if (dev)
qp->iif = dev->ifindex; qp->iif = dev->ifindex;
@ -425,7 +485,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb)
unsigned long orefdst = skb->_skb_refdst; unsigned long orefdst = skb->_skb_refdst;
skb->_skb_refdst = 0UL; skb->_skb_refdst = 0UL;
err = ip_frag_reasm(qp, skb, dev); err = ip_frag_reasm(qp, skb, prev_tail, dev);
skb->_skb_refdst = orefdst; skb->_skb_refdst = orefdst;
return err; return err;
} }
@ -444,7 +504,7 @@ err:
/* Build a new IP datagram from all its fragments. */ /* Build a new IP datagram from all its fragments. */
static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
struct net_device *dev) struct sk_buff *prev_tail, struct net_device *dev)
{ {
struct net *net = container_of(qp->q.net, struct net, ipv4.frags); struct net *net = container_of(qp->q.net, struct net, ipv4.frags);
struct iphdr *iph; struct iphdr *iph;
@ -468,10 +528,16 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
fp = skb_clone(skb, GFP_ATOMIC); fp = skb_clone(skb, GFP_ATOMIC);
if (!fp) if (!fp)
goto out_nomem; goto out_nomem;
rb_replace_node(&skb->rbnode, &fp->rbnode, &qp->q.rb_fragments); FRAG_CB(fp)->next_frag = FRAG_CB(skb)->next_frag;
if (RB_EMPTY_NODE(&skb->rbnode))
FRAG_CB(prev_tail)->next_frag = fp;
else
rb_replace_node(&skb->rbnode, &fp->rbnode,
&qp->q.rb_fragments);
if (qp->q.fragments_tail == skb) if (qp->q.fragments_tail == skb)
qp->q.fragments_tail = fp; qp->q.fragments_tail = fp;
skb_morph(skb, head); skb_morph(skb, head);
FRAG_CB(skb)->next_frag = FRAG_CB(head)->next_frag;
rb_replace_node(&head->rbnode, &skb->rbnode, rb_replace_node(&head->rbnode, &skb->rbnode,
&qp->q.rb_fragments); &qp->q.rb_fragments);
consume_skb(head); consume_skb(head);
@ -507,7 +573,7 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
for (i = 0; i < skb_shinfo(head)->nr_frags; i++) for (i = 0; i < skb_shinfo(head)->nr_frags; i++)
plen += skb_frag_size(&skb_shinfo(head)->frags[i]); plen += skb_frag_size(&skb_shinfo(head)->frags[i]);
clone->len = clone->data_len = head->data_len - plen; clone->len = clone->data_len = head->data_len - plen;
skb->truesize += clone->truesize; head->truesize += clone->truesize;
clone->csum = 0; clone->csum = 0;
clone->ip_summed = head->ip_summed; clone->ip_summed = head->ip_summed;
add_frag_mem_limit(qp->q.net, clone->truesize); add_frag_mem_limit(qp->q.net, clone->truesize);
@ -520,13 +586,15 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
skb_push(head, head->data - skb_network_header(head)); skb_push(head, head->data - skb_network_header(head));
/* Traverse the tree in order, to build frag_list. */ /* Traverse the tree in order, to build frag_list. */
fp = FRAG_CB(head)->next_frag;
rbn = rb_next(&head->rbnode); rbn = rb_next(&head->rbnode);
rb_erase(&head->rbnode, &qp->q.rb_fragments); rb_erase(&head->rbnode, &qp->q.rb_fragments);
while (rbn) { while (rbn || fp) {
struct rb_node *rbnext = rb_next(rbn); /* fp points to the next sk_buff in the current run;
fp = rb_to_skb(rbn); * rbn points to the next run.
rb_erase(rbn, &qp->q.rb_fragments); */
rbn = rbnext; /* Go through the current run. */
while (fp) {
*nextp = fp; *nextp = fp;
nextp = &fp->next; nextp = &fp->next;
fp->prev = NULL; fp->prev = NULL;
@ -538,6 +606,16 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
else if (head->ip_summed == CHECKSUM_COMPLETE) else if (head->ip_summed == CHECKSUM_COMPLETE)
head->csum = csum_add(head->csum, fp->csum); head->csum = csum_add(head->csum, fp->csum);
head->truesize += fp->truesize; head->truesize += fp->truesize;
fp = FRAG_CB(fp)->next_frag;
}
/* Move to the next run. */
if (rbn) {
struct rb_node *rbnext = rb_next(rbn);
fp = rb_to_skb(rbn);
rb_erase(rbn, &qp->q.rb_fragments);
rbn = rbnext;
}
} }
sub_frag_mem_limit(qp->q.net, head->truesize); sub_frag_mem_limit(qp->q.net, head->truesize);
@ -573,6 +651,7 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,
qp->q.fragments = NULL; qp->q.fragments = NULL;
qp->q.rb_fragments = RB_ROOT; qp->q.rb_fragments = RB_ROOT;
qp->q.fragments_tail = NULL; qp->q.fragments_tail = NULL;
qp->q.last_run_head = NULL;
return 0; return 0;
out_nomem: out_nomem:
@ -654,6 +733,28 @@ struct sk_buff *ip_check_defrag(struct net *net, struct sk_buff *skb, u32 user)
} }
EXPORT_SYMBOL(ip_check_defrag); EXPORT_SYMBOL(ip_check_defrag);
unsigned int inet_frag_rbtree_purge(struct rb_root *root)
{
struct rb_node *p = rb_first(root);
unsigned int sum = 0;
while (p) {
struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
p = rb_next(p);
rb_erase(&skb->rbnode, root);
while (skb) {
struct sk_buff *next = FRAG_CB(skb)->next_frag;
sum += skb->truesize;
kfree_skb(skb);
skb = next;
}
}
return sum;
}
EXPORT_SYMBOL(inet_frag_rbtree_purge);
#ifdef CONFIG_SYSCTL #ifdef CONFIG_SYSCTL
static int dist_min; static int dist_min;