iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

net: move sysctl_mem_pcpu_rsv to net_hotdata

Browse Source 
sysctl_mem_pcpu_rsv is used in TCP fast path,
move it to net_hodata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240429134025.1233626-6-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet 2024-04-29 13:40:25 +00:00 committed by Jakub Kicinski
parent f3d93817fb
commit c204fef97e
5 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/net/hotdata.h
View File

@ -40,6 +40,7 @@ struct net_hotdata {
int dev_rx_weight; int dev_rx_weight;
int sysctl_max_skb_frags; int sysctl_max_skb_frags;
int sysctl_skb_defer_max; int sysctl_skb_defer_max;
int sysctl_mem_pcpu_rsv;
}; };
#define inet_ehash_secret net_hotdata.tcp_protocol.secret #define inet_ehash_secret net_hotdata.tcp_protocol.secret

6
include/net/proto_memory.h
View File

@ -3,10 +3,10 @@
#define _PROTO_MEMORY_H #define _PROTO_MEMORY_H
#include <net/sock.h> #include <net/sock.h>
#include <net/hotdata.h>
/* 1 MB per cpu, in page units */ /* 1 MB per cpu, in page units */
#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
extern int sysctl_mem_pcpu_rsv;
static inline bool sk_has_memory_pressure(const struct sock *sk) static inline bool sk_has_memory_pressure(const struct sock *sk)
{ {
@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val)
val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val); val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv))) if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
proto_memory_pcpu_drain(proto); proto_memory_pcpu_drain(proto);
} }
@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val)
val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val); val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv))) if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
proto_memory_pcpu_drain(proto); proto_memory_pcpu_drain(proto);
} }

5
net/core/hotdata.c
View File

@ -1,9 +1,9 @@
// SPDX-License-Identifier: GPL-2.0-or-later // SPDX-License-Identifier: GPL-2.0-or-later
#include <net/hotdata.h>
#include <linux/cache.h> #include <linux/cache.h>
#include <linux/jiffies.h> #include <linux/jiffies.h>
#include <linux/list.h> #include <linux/list.h>
#include <net/hotdata.h>
#include <net/proto_memory.h>
struct net_hotdata net_hotdata __cacheline_aligned = { struct net_hotdata net_hotdata __cacheline_aligned = {
.offload_base = LIST_HEAD_INIT(net_hotdata.offload_base), .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
.dev_rx_weight = 64, .dev_rx_weight = 64,
.sysctl_max_skb_frags = MAX_SKB_FRAGS, .sysctl_max_skb_frags = MAX_SKB_FRAGS,
.sysctl_skb_defer_max = 64, .sysctl_skb_defer_max = 64,
.sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE
}; };
EXPORT_SYMBOL(net_hotdata); EXPORT_SYMBOL(net_hotdata);

1
net/core/sock.c
View File

@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
EXPORT_SYMBOL(sysctl_rmem_max); EXPORT_SYMBOL(sysctl_rmem_max);
__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE;
int sysctl_tstamp_allow_data __read_mostly = 1; int sysctl_tstamp_allow_data __read_mostly = 1;

2
net/core/sysctl_net_core.c
View File

@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = {
}, },
{ {
.procname = "mem_pcpu_rsv", .procname = "mem_pcpu_rsv",
.data = &sysctl_mem_pcpu_rsv, .data = &net_hotdata.sysctl_mem_pcpu_rsv,
.maxlen = sizeof(int), .maxlen = sizeof(int),
.mode = 0644, .mode = 0644,
.proc_handler = proc_dointvec_minmax, .proc_handler = proc_dointvec_minmax,