iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branches 'slab/for-5.19/stackdepot' and 'slab/for-5.19/refactor' into slab/for-linus

Browse Source
This commit is contained in:
Vlastimil Babka
2022-05-23 11:14:32 +02:00
parent f8d9f46e87 9f04b55f00 23587f7c5d
commit e001897da6
9 changed files with 238 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
mm/slub.c
View File

@ -26,6 +26,7 @@
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
#include <linux/stackdepot.h>
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/kfence.h>
@ -37,6 +38,7 @@
#include <linux/memcontrol.h>
#include <linux/random.h>
#include <kunit/test.h>
#include <linux/sort.h>
#include <linux/debugfs.h>
#include <trace/events/kmem.h>
@ -264,8 +266,8 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
#define TRACK_ADDRS_COUNT 16
struct track {
unsigned long addr; /* Called from address */
#ifdef CONFIG_STACKTRACE
unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
#ifdef CONFIG_STACKDEPOT
depot_stack_handle_t handle;
#endif
int cpu; /* Was running on cpu */
int pid; /* Pid context */
@ -724,57 +726,51 @@ static struct track *get_track(struct kmem_cache *s, void *object,
return kasan_reset_tag(p + alloc);
}
static void set_track(struct kmem_cache *s, void *object,
static void noinline set_track(struct kmem_cache *s, void *object,
enum track_item alloc, unsigned long addr)
{
struct track *p = get_track(s, object, alloc);
if (addr) {
#ifdef CONFIG_STACKTRACE
unsigned int nr_entries;
#ifdef CONFIG_STACKDEPOT
unsigned long entries[TRACK_ADDRS_COUNT];
unsigned int nr_entries;
metadata_access_enable();
nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
TRACK_ADDRS_COUNT, 3);
metadata_access_disable();
if (nr_entries < TRACK_ADDRS_COUNT)
p->addrs[nr_entries] = 0;
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
p->handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);
#endif
p->addr = addr;
p->cpu = smp_processor_id();
p->pid = current->pid;
p->when = jiffies;
} else {
memset(p, 0, sizeof(struct track));
}
p->addr = addr;
p->cpu = smp_processor_id();
p->pid = current->pid;
p->when = jiffies;
}
static void init_tracking(struct kmem_cache *s, void *object)
{
struct track *p;
if (!(s->flags & SLAB_STORE_USER))
return;
set_track(s, object, TRACK_FREE, 0UL);
set_track(s, object, TRACK_ALLOC, 0UL);
p = get_track(s, object, TRACK_ALLOC);
memset(p, 0, 2*sizeof(struct track));
}
static void print_track(const char *s, struct track *t, unsigned long pr_time)
{
depot_stack_handle_t handle __maybe_unused;
if (!t->addr)
return;
pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
#ifdef CONFIG_STACKTRACE
{
int i;
for (i = 0; i < TRACK_ADDRS_COUNT; i++)
if (t->addrs[i])
pr_err("\t%pS\n", (void *)t->addrs[i]);
else
break;
}
#ifdef CONFIG_STACKDEPOT
handle = READ_ONCE(t->handle);
if (handle)
stack_depot_print(handle);
else
pr_err("object allocation/free stack trace missing\n");
#endif
}
@ -1532,6 +1528,8 @@ static int __init setup_slub_debug(char *str)
global_slub_debug_changed = true;
} else {
slab_list_specified = true;
if (flags & SLAB_STORE_USER)
stack_depot_want_early_init();
}
}
@ -1549,6 +1547,8 @@ static int __init setup_slub_debug(char *str)
}
out:
slub_debug = global_flags;
if (slub_debug & SLAB_STORE_USER)
stack_depot_want_early_init();
if (slub_debug != 0 || slub_debug_string)
static_branch_enable(&slub_debug_enabled);
else
@ -4162,8 +4162,6 @@ static int calculate_sizes(struct kmem_cache *s)
*/
s->oo = oo_make(order, size);
s->min = oo_make(get_order(size), size);
if (oo_objects(s->oo) > oo_objects(s->max))
s->max = s->oo;
return !!oo_objects(s->oo);
}
@ -4341,18 +4339,26 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
objp = fixup_red_left(s, objp);
trackp = get_track(s, objp, TRACK_ALLOC);
kpp->kp_ret = (void *)trackp->addr;
#ifdef CONFIG_STACKTRACE
for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
kpp->kp_stack[i] = (void *)trackp->addrs[i];
if (!kpp->kp_stack[i])
break;
}
#ifdef CONFIG_STACKDEPOT
{
depot_stack_handle_t handle;
unsigned long *entries;
unsigned int nr_entries;
trackp = get_track(s, objp, TRACK_FREE);
for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
kpp->kp_free_stack[i] = (void *)trackp->addrs[i];
if (!kpp->kp_free_stack[i])
break;
handle = READ_ONCE(trackp->handle);
if (handle) {
nr_entries = stack_depot_fetch(handle, &entries);
for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
kpp->kp_stack[i] = (void *)entries[i];
}
trackp = get_track(s, objp, TRACK_FREE);
handle = READ_ONCE(trackp->handle);
if (handle) {
nr_entries = stack_depot_fetch(handle, &entries);
for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
kpp->kp_free_stack[i] = (void *)entries[i];
}
}
#endif
#endif
@ -5054,6 +5060,7 @@ EXPORT_SYMBOL(validate_slab_cache);
*/
struct location {
depot_stack_handle_t handle;
unsigned long count;
unsigned long addr;
long long sum_time;
@ -5106,9 +5113,13 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
{
long start, end, pos;
struct location *l;
unsigned long caddr;
unsigned long caddr, chandle;
unsigned long age = jiffies - track->when;
depot_stack_handle_t handle = 0;
#ifdef CONFIG_STACKDEPOT
handle = READ_ONCE(track->handle);
#endif
start = -1;
end = t->count;
@ -5123,7 +5134,8 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
break;
caddr = t->loc[pos].addr;
if (track->addr == caddr) {
chandle = t->loc[pos].handle;
if ((track->addr == caddr) && (handle == chandle)) {
l = &t->loc[pos];
l->count++;
@ -5148,6 +5160,8 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
if (track->addr < caddr)
end = pos;
else if (track->addr == caddr && handle < chandle)
end = pos;
else
start = pos;
}
@ -5170,6 +5184,7 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
l->max_time = age;
l->min_pid = track->pid;
l->max_pid = track->pid;
l->handle = handle;
cpumask_clear(to_cpumask(l->cpus));
cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
nodes_clear(l->nodes);
@ -6079,6 +6094,21 @@ static int slab_debugfs_show(struct seq_file *seq, void *v)
seq_printf(seq, " nodes=%*pbl",
nodemask_pr_args(&l->nodes));
#ifdef CONFIG_STACKDEPOT
{
depot_stack_handle_t handle;
unsigned long *entries;
unsigned int nr_entries, j;
handle = READ_ONCE(l->handle);
if (handle) {
nr_entries = stack_depot_fetch(handle, &entries);
seq_puts(seq, "\n");
for (j = 0; j < nr_entries; j++)
seq_printf(seq, " %pS\n", (void *)entries[j]);
}
}
#endif
seq_puts(seq, "\n");
}
@ -6103,6 +6133,17 @@ static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
return NULL;
}
static int cmp_loc_by_count(const void *a, const void *b, const void *data)
{
struct location *loc1 = (struct location *)a;
struct location *loc2 = (struct location *)b;
if (loc1->count > loc2->count)
return -1;
else
return 1;
}
static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
{
struct loc_track *t = seq->private;
@ -6164,6 +6205,10 @@ static int slab_debug_trace_open(struct inode *inode, struct file *filep)
spin_unlock_irqrestore(&n->list_lock, flags);
}
/* Sort locations by count */
sort_r(t->loc, t->count, sizeof(struct location),
cmp_loc_by_count, NULL, NULL);
bitmap_free(obj_map);
return 0;
}