iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mm, slub: move disabling irqs closer to get_partial() in ___slab_alloc()

Browse Source 
Continue reducing the irq disabled scope. Check for per-cpu partial slabs with
first with irqs enabled and then recheck with irqs disabled before grabbing
the slab page. Mostly preparatory for the following patches.

Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
This commit is contained in:
Vlastimil Babka 2021-05-10 13:56:17 +02:00
parent 0b303fb402
commit fa417ab750
1 changed files with 25 additions and 9 deletions

34
mm/slub.c

@ -2706,11 +2706,6 @@ reread_page:
if (unlikely(node != NUMA_NO_NODE &&
!node_isset(node, slab_nodes)))
node = NUMA_NO_NODE;
local_irq_save(flags);
if (unlikely(c->page)) {
local_irq_restore(flags);
goto reread_page;
}
goto new_slab;
}
redo:
@ -2751,6 +2746,7 @@ redo:
if (!freelist) {
c->page = NULL;
local_irq_restore(flags);
stat(s, DEACTIVATE_BYPASS);
goto new_slab;
}
@ -2780,12 +2776,19 @@ deactivate_slab:
goto reread_page;
}
deactivate_slab(s, page, c->freelist, c);
local_irq_restore(flags);
new_slab:
lockdep_assert_irqs_disabled();
if (slub_percpu_partial(c)) {
local_irq_save(flags);
if (unlikely(c->page)) {
local_irq_restore(flags);
goto reread_page;
}
if (unlikely(!slub_percpu_partial(c)))
goto new_objects; /* stolen by an IRQ handler */
page = c->page = slub_percpu_partial(c);
slub_set_percpu_partial(c, page);
local_irq_restore(flags);
@ -2793,6 +2796,16 @@ new_slab:
goto redo;
}
local_irq_save(flags);
if (unlikely(c->page)) {
local_irq_restore(flags);
goto reread_page;
}
new_objects:
lockdep_assert_irqs_disabled();
freelist = get_partial(s, gfpflags, node, &page);
if (freelist) {
c->page = page;
@ -2825,15 +2838,18 @@ new_slab:
check_new_page:
if (kmem_cache_debug(s)) {
if (!alloc_debug_processing(s, page, freelist, addr))
if (!alloc_debug_processing(s, page, freelist, addr)) {
/* Slab failed checks. Next slab needed */
c->page = NULL;
local_irq_restore(flags);
goto new_slab;
else
} else {
/*
* For debug case, we don't load freelist so that all
* allocations go through alloc_debug_processing()
*/
goto return_single;
}
}
if (unlikely(!pfmemalloc_match(page, gfpflags)))