iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PATCH] x86: Allow percpu variables to be page-aligned

Browse Source 
Let's allow page-alignment in general for per-cpu data (wanted by Xen, and
Ingo suggested KVM as well).

Because larger alignments can use more room, we increase the max per-cpu
memory to 64k rather than 32k: it's getting a little tight.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Jeremy Fitzhardinge 2007-05-02 19:27:12 +02:00 committed by Andi Kleen
parent de90c5ce83
commit b6e3590f81
21 changed files with 29 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/alpha/kernel/vmlinux.lds.S
View File

@ -69,7 +69,7 @@ SECTIONS
. = ALIGN(8); . = ALIGN(8);
SECURITY_INIT SECURITY_INIT
. = ALIGN(64); . = ALIGN(8192);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/arm/kernel/vmlinux.lds.S
View File

@ -59,7 +59,7 @@ SECTIONS
usr/built-in.o(.init.ramfs) usr/built-in.o(.init.ramfs)
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(64); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
*(.data.percpu) *(.data.percpu)
__per_cpu_end = .; __per_cpu_end = .;

1
arch/cris/arch-v32/vmlinux.lds.S
View File

@ -91,6 +91,7 @@ SECTIONS
} }
SECURITY_INIT SECURITY_INIT
. = ALIGN (8192);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

1
arch/frv/kernel/vmlinux.lds.S
View File

@ -57,6 +57,7 @@ SECTIONS
__alt_instructions_end = .; __alt_instructions_end = .;
.altinstr_replacement : { *(.altinstr_replacement) } .altinstr_replacement : { *(.altinstr_replacement) }
. = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/i386/kernel/vmlinux.lds.S
View File

@ -194,7 +194,7 @@ SECTIONS
__initramfs_end = .; __initramfs_end = .;
} }
#endif #endif
. = ALIGN(L1_CACHE_BYTES); . = ALIGN(4096);
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
__per_cpu_start = .; __per_cpu_start = .;
*(.data.percpu) *(.data.percpu)

2
arch/m32r/kernel/vmlinux.lds.S
View File

@ -110,7 +110,7 @@ SECTIONS
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(32); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/mips/kernel/vmlinux.lds.S
View File

@ -119,7 +119,7 @@ SECTIONS
.init.ramfs : { *(.init.ramfs) } .init.ramfs : { *(.init.ramfs) }
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(32); . = ALIGN(_PAGE_SIZE);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/parisc/kernel/vmlinux.lds.S
View File

@ -181,7 +181,7 @@ SECTIONS
.init.ramfs : { *(.init.ramfs) } .init.ramfs : { *(.init.ramfs) }
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(32); . = ALIGN(ASM_PAGE_SIZE);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

4
arch/powerpc/kernel/setup_64.c
View File

@ -582,14 +582,14 @@ void __init setup_per_cpu_areas(void)
char *ptr; char *ptr;
/* Copy section for each CPU (we discard the original) */ /* Copy section for each CPU (we discard the original) */
size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES); size = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE);
#ifdef CONFIG_MODULES #ifdef CONFIG_MODULES
if (size < PERCPU_ENOUGH_ROOM) if (size < PERCPU_ENOUGH_ROOM)
size = PERCPU_ENOUGH_ROOM; size = PERCPU_ENOUGH_ROOM;
#endif #endif
for_each_possible_cpu(i) { for_each_possible_cpu(i) {
ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size); ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
if (!ptr) if (!ptr)
panic("Cannot allocate cpu data for CPU %d\n", i); panic("Cannot allocate cpu data for CPU %d\n", i);

6
arch/powerpc/kernel/vmlinux.lds.S
View File

@ -139,11 +139,7 @@ SECTIONS
__initramfs_end = .; __initramfs_end = .;
} }
#endif #endif
#ifdef CONFIG_PPC32 . = ALIGN(PAGE_SIZE);
. = ALIGN(32);
#else
. = ALIGN(128);
#endif
.data.percpu : { .data.percpu : {
__per_cpu_start = .; __per_cpu_start = .;
*(.data.percpu) *(.data.percpu)

2
arch/ppc/kernel/vmlinux.lds.S
View File

@ -130,7 +130,7 @@ SECTIONS
__ftr_fixup : { *(__ftr_fixup) } __ftr_fixup : { *(__ftr_fixup) }
__stop___ftr_fixup = .; __stop___ftr_fixup = .;
. = ALIGN(32); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/s390/kernel/vmlinux.lds.S
View File

@ -107,7 +107,7 @@ SECTIONS
. = ALIGN(2); . = ALIGN(2);
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(256); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/sh/kernel/vmlinux.lds.S
View File

@ -54,7 +54,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
.data.page_aligned : { *(.data.page_aligned) } .data.page_aligned : { *(.data.page_aligned) }
. = ALIGN(L1_CACHE_BYTES); . = ALIGN(PAGE_SIZE);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/sh64/kernel/vmlinux.lds.S
View File

@ -85,7 +85,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
.data.page_aligned : C_PHYS(.data.page_aligned) { *(.data.page_aligned) } .data.page_aligned : C_PHYS(.data.page_aligned) { *(.data.page_aligned) }
. = ALIGN(L1_CACHE_BYTES); . = ALIGN(PAGE_SIZE);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : C_PHYS(.data.percpu) { *(.data.percpu) } .data.percpu : C_PHYS(.data.percpu) { *(.data.percpu) }
__per_cpu_end = . ; __per_cpu_end = . ;

2
arch/sparc/kernel/vmlinux.lds.S
View File

@ -65,7 +65,7 @@ SECTIONS
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(32); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

6
arch/sparc64/kernel/smp.c
View File

@ -1343,11 +1343,11 @@ void __init setup_per_cpu_areas(void)
/* Copy section for each CPU (we discard the original) */ /* Copy section for each CPU (we discard the original) */
goal = PERCPU_ENOUGH_ROOM; goal = PERCPU_ENOUGH_ROOM;
__per_cpu_shift = 0; __per_cpu_shift = PAGE_SHIFT;
for (size = 1UL; size < goal; size <<= 1UL) for (size = PAGE_SIZE; size < goal; size <<= 1UL)
__per_cpu_shift++; __per_cpu_shift++;
ptr = alloc_bootmem(size * NR_CPUS); ptr = alloc_bootmem_pages(size * NR_CPUS);
__per_cpu_base = ptr - __per_cpu_start; __per_cpu_base = ptr - __per_cpu_start;

4
arch/x86_64/kernel/setup64.c
View File

@ -103,9 +103,9 @@ void __init setup_per_cpu_areas(void)
if (!NODE_DATA(cpu_to_node(i))) { if (!NODE_DATA(cpu_to_node(i))) {
printk("cpu with no node %d, num_online_nodes %d\n", printk("cpu with no node %d, num_online_nodes %d\n",
i, num_online_nodes()); i, num_online_nodes());
ptr = alloc_bootmem(size); ptr = alloc_bootmem_pages(size);
} else { } else {
ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size); ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
} }
if (!ptr) if (!ptr)
panic("Cannot allocate cpu data for CPU %d\n", i); panic("Cannot allocate cpu data for CPU %d\n", i);

2
arch/x86_64/kernel/vmlinux.lds.S
View File

@ -195,7 +195,7 @@ SECTIONS
__initramfs_end = .; __initramfs_end = .;
#endif #endif
. = ALIGN(CONFIG_X86_L1_CACHE_BYTES); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { *(.data.percpu) } .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

2
arch/xtensa/kernel/vmlinux.lds.S
View File

@ -198,7 +198,7 @@ SECTIONS
__ftr_fixup : { *(__ftr_fixup) } __ftr_fixup : { *(__ftr_fixup) }
__stop___ftr_fixup = .; __stop___ftr_fixup = .;
. = ALIGN(32); . = ALIGN(4096);
__per_cpu_start = .; __per_cpu_start = .;
.data.percpu : { *(.data.percpu) } .data.percpu : { *(.data.percpu) }
__per_cpu_end = .; __per_cpu_end = .;

8
init/main.c
View File

@ -369,12 +369,8 @@ static void __init setup_per_cpu_areas(void)
unsigned long nr_possible_cpus = num_possible_cpus(); unsigned long nr_possible_cpus = num_possible_cpus();
/* Copy section for each CPU (we discard the original) */ /* Copy section for each CPU (we discard the original) */
size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES); size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
#ifdef CONFIG_MODULES ptr = alloc_bootmem_pages(size * nr_possible_cpus);
if (size < PERCPU_ENOUGH_ROOM)
size = PERCPU_ENOUGH_ROOM;
#endif
ptr = alloc_bootmem(size * nr_possible_cpus);
for_each_possible_cpu(i) { for_each_possible_cpu(i) {
__per_cpu_offset[i] = ptr - __per_cpu_start; __per_cpu_offset[i] = ptr - __per_cpu_start;

8
kernel/module.c
View File

@ -346,10 +346,10 @@ static void *percpu_modalloc(unsigned long size, unsigned long align,
unsigned int i; unsigned int i;
void *ptr; void *ptr;
if (align > SMP_CACHE_BYTES) { if (align > PAGE_SIZE) {
printk(KERN_WARNING "%s: per-cpu alignment %li > %i\n", printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
name, align, SMP_CACHE_BYTES); name, align, PAGE_SIZE);
align = SMP_CACHE_BYTES; align = PAGE_SIZE;
} }
ptr = __per_cpu_start; ptr = __per_cpu_start;