From e75ec151c1088c1aa7a49ff16a2adcaddb24a861 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 18 Mar 2021 18:42:23 +0000 Subject: [PATCH 1/4] gro: make net/gro.h self-contained MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If some source file includes , but doesn't include : In file included from net/8021q/vlan_core.c:7: ./include/net/gro.h:6:1: warning: data definition has no type or storage class 6 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, | ^~~~~~~~~~~~~~~~~~~~~~~~~ ./include/net/gro.h:6:1: error: type defaults to ‘int’ in declaration of ‘INDIRECT_CALLABLE_DECLARE’ [-Werror=implicit-int] [...] Include directly. It's small and won't pull lots of dependencies. Also add some incomplete struct declarations to be fully stacked. Fixes: 04f00ab2275f ("net/core: move gro function declarations to separate header ") Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- include/net/gro.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/net/gro.h b/include/net/gro.h index 8a6eb5303cc4..27c38b36df16 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -3,6 +3,11 @@ #ifndef _NET_IPV6_GRO_H #define _NET_IPV6_GRO_H +#include + +struct list_head; +struct sk_buff; + INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); From 86af2c82c28499b4b509d6b15637a96bd829201b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 18 Mar 2021 18:42:30 +0000 Subject: [PATCH 2/4] gro: add combined call_gro_receive() + INDIRECT_CALL_INET() helper call_gro_receive() is used to limit GRO recursion, but it works only with callback pointers. There's a combined version of call_gro_receive() + INDIRECT_CALL_2() in , but it doesn't check for IPv6 modularity. Add a similar new helper to cover both of these. It can and will be used to avoid retpoline overhead when IP header lies behind another offloaded proto. Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- include/net/gro.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/net/gro.h b/include/net/gro.h index 27c38b36df16..01edaf3fdda0 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -14,4 +14,12 @@ INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); + +#define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ +({ \ + unlikely(gro_recursion_inc_test(skb)) ? \ + NAPI_GRO_CB(skb)->flush |= 1, NULL : \ + INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ +}) + #endif /* _NET_IPV6_GRO_H */ From 4a6e7ec93a602b1e386704aeb0ce76bfc1ba8030 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 18 Mar 2021 18:42:34 +0000 Subject: [PATCH 3/4] vlan/8021q: avoid retpoline overhead on GRO The two most popular headers going after VLAN are IPv4 and IPv6. Retpoline overhead for them is addressed only in dev_gro_receive(), when they lie right after the outermost Ethernet header. Use the indirect call wrappers in VLAN GRO receive code to reduce the penalty on receiving tagged frames (when hardware stripping is off or not available). Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- net/8021q/vlan_core.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c index 78ec2e1b14d1..59bc13b5f14f 100644 --- a/net/8021q/vlan_core.c +++ b/net/8021q/vlan_core.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "vlan.h" bool vlan_do_receive(struct sk_buff **skbp) @@ -495,7 +496,10 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head, skb_gro_pull(skb, sizeof(*vhdr)); skb_gro_postpull_rcsum(skb, vhdr, sizeof(*vhdr)); - pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + + pp = indirect_call_gro_receive_inet(ptype->callbacks.gro_receive, + ipv6_gro_receive, inet_gro_receive, + head, skb); out_unlock: rcu_read_unlock(); @@ -515,7 +519,9 @@ static int vlan_gro_complete(struct sk_buff *skb, int nhoff) rcu_read_lock(); ptype = gro_find_complete_by_type(type); if (ptype) - err = ptype->callbacks.gro_complete(skb, nhoff + sizeof(*vhdr)); + err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, + ipv6_gro_complete, inet_gro_complete, + skb, nhoff + sizeof(*vhdr)); rcu_read_unlock(); return err; From 5588796e89777318267ff13f2bf322b2c48843af Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 18 Mar 2021 18:42:39 +0000 Subject: [PATCH 4/4] ethernet: avoid retpoline overhead on TEB (GENEVE, NvGRE, VxLAN) GRO The two most popular headers going after Ethernet are IPv4 and IPv6. Retpoline overhead for them is addressed only in dev_gro_receive(), when they lie right after the outermost Ethernet header. Use the indirect call wrappers in TEB (Transparent Ethernet Bridging, such as GENEVE, NvGRE, VxLAN etc.) GRO receive code to reduce the penalty when processing the inner headers. Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- net/ethernet/eth.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index e01cf766d2c5..933b427122be 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include @@ -449,7 +450,10 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb) skb_gro_pull(skb, sizeof(*eh)); skb_gro_postpull_rcsum(skb, eh, sizeof(*eh)); - pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + + pp = indirect_call_gro_receive_inet(ptype->callbacks.gro_receive, + ipv6_gro_receive, inet_gro_receive, + head, skb); out_unlock: rcu_read_unlock(); @@ -473,8 +477,9 @@ int eth_gro_complete(struct sk_buff *skb, int nhoff) rcu_read_lock(); ptype = gro_find_complete_by_type(type); if (ptype != NULL) - err = ptype->callbacks.gro_complete(skb, nhoff + - sizeof(struct ethhdr)); + err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, + ipv6_gro_complete, inet_gro_complete, + skb, nhoff + sizeof(*eh)); rcu_read_unlock(); return err;