slab,slub: don't enable interrupts during early boot
As explained by Benjamin Herrenschmidt: Oh and btw, your patch alone doesn't fix powerpc, because it's missing a whole bunch of GFP_KERNEL's in the arch code... You would have to grep the entire kernel for things that check slab_is_available() and even then you'll be missing some. For example, slab_is_available() didn't always exist, and so in the early days on powerpc, we used a mem_init_done global that is set form mem_init() (not perfect but works in practice). And we still have code using that to do the test. Therefore, mask out __GFP_WAIT, __GFP_IO, and __GFP_FS in the slab allocators in early boot code to avoid enabling interrupts. Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>
This commit is contained in:
parent
eb91f1d0a5
commit
7e85ee0c1d
@ -85,6 +85,9 @@ struct vm_area_struct;
|
||||
__GFP_NOWARN|__GFP_REPEAT|__GFP_NOFAIL|\
|
||||
__GFP_NORETRY|__GFP_NOMEMALLOC)
|
||||
|
||||
/* Control slab gfp mask during early boot */
|
||||
#define SLAB_GFP_BOOT_MASK __GFP_BITS_MASK & ~(__GFP_WAIT|__GFP_IO|__GFP_FS)
|
||||
|
||||
/* Control allocation constraints */
|
||||
#define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE)
|
||||
|
||||
|
@ -319,4 +319,6 @@ static inline void *kzalloc_node(size_t size, gfp_t flags, int node)
|
||||
return kmalloc_node(size, flags | __GFP_ZERO, node);
|
||||
}
|
||||
|
||||
void __init kmem_cache_init_late(void);
|
||||
|
||||
#endif /* _LINUX_SLAB_H */
|
||||
|
@ -34,4 +34,9 @@ static __always_inline void *__kmalloc(size_t size, gfp_t flags)
|
||||
return kmalloc(size, flags);
|
||||
}
|
||||
|
||||
static inline void kmem_cache_init_late(void)
|
||||
{
|
||||
/* Nothing to do */
|
||||
}
|
||||
|
||||
#endif /* __LINUX_SLOB_DEF_H */
|
||||
|
@ -302,4 +302,6 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
}
|
||||
#endif
|
||||
|
||||
void __init kmem_cache_init_late(void);
|
||||
|
||||
#endif /* _LINUX_SLUB_DEF_H */
|
||||
|
@ -640,6 +640,7 @@ asmlinkage void __init start_kernel(void)
|
||||
"enabled early\n");
|
||||
early_boot_irqs_on();
|
||||
local_irq_enable();
|
||||
kmem_cache_init_late();
|
||||
|
||||
/*
|
||||
* HACK ALERT! This is early. We're enabling the console before
|
||||
|
18
mm/slab.c
18
mm/slab.c
@ -303,6 +303,12 @@ struct kmem_list3 {
|
||||
int free_touched; /* updated without locking */
|
||||
};
|
||||
|
||||
/*
|
||||
* The slab allocator is initialized with interrupts disabled. Therefore, make
|
||||
* sure early boot allocations don't accidentally enable interrupts.
|
||||
*/
|
||||
static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
|
||||
|
||||
/*
|
||||
* Need this for bootstrapping a per node allocator.
|
||||
*/
|
||||
@ -1654,6 +1660,14 @@ void __init kmem_cache_init(void)
|
||||
*/
|
||||
}
|
||||
|
||||
void __init kmem_cache_init_late(void)
|
||||
{
|
||||
/*
|
||||
* Interrupts are enabled now so all GFP allocations are safe.
|
||||
*/
|
||||
slab_gfp_mask = __GFP_BITS_MASK;
|
||||
}
|
||||
|
||||
static int __init cpucache_init(void)
|
||||
{
|
||||
int cpu;
|
||||
@ -3354,6 +3368,8 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
|
||||
unsigned long save_flags;
|
||||
void *ptr;
|
||||
|
||||
flags &= slab_gfp_mask;
|
||||
|
||||
lockdep_trace_alloc(flags);
|
||||
|
||||
if (slab_should_failslab(cachep, flags))
|
||||
@ -3434,6 +3450,8 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
|
||||
unsigned long save_flags;
|
||||
void *objp;
|
||||
|
||||
flags &= slab_gfp_mask;
|
||||
|
||||
lockdep_trace_alloc(flags);
|
||||
|
||||
if (slab_should_failslab(cachep, flags))
|
||||
|
16
mm/slub.c
16
mm/slub.c
@ -178,6 +178,12 @@ static enum {
|
||||
SYSFS /* Sysfs up */
|
||||
} slab_state = DOWN;
|
||||
|
||||
/*
|
||||
* The slab allocator is initialized with interrupts disabled. Therefore, make
|
||||
* sure early boot allocations don't accidentally enable interrupts.
|
||||
*/
|
||||
static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
|
||||
|
||||
/* A list of all slab caches on the system */
|
||||
static DECLARE_RWSEM(slub_lock);
|
||||
static LIST_HEAD(slab_caches);
|
||||
@ -1595,6 +1601,8 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
|
||||
unsigned long flags;
|
||||
unsigned int objsize;
|
||||
|
||||
gfpflags &= slab_gfp_mask;
|
||||
|
||||
lockdep_trace_alloc(gfpflags);
|
||||
might_sleep_if(gfpflags & __GFP_WAIT);
|
||||
|
||||
@ -3104,6 +3112,14 @@ void __init kmem_cache_init(void)
|
||||
nr_cpu_ids, nr_node_ids);
|
||||
}
|
||||
|
||||
void __init kmem_cache_init_late(void)
|
||||
{
|
||||
/*
|
||||
* Interrupts are enabled now so all GFP allocations are safe.
|
||||
*/
|
||||
slab_gfp_mask = __GFP_BITS_MASK;
|
||||
}
|
||||
|
||||
/*
|
||||
* Find a mergeable slab cache
|
||||
*/
|
||||
|
Loading…
Reference in New Issue
Block a user