iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
ed391f4ebf
linux/arch/arm/lib/uaccess_with_memcpy.c

229 lines
5.2 KiB
C
Raw Normal View History

[ARM] alternative copy_to_user/clear_user implementation This implements {copy_to,clear}_user() by faulting in the userland pages and then using the regular kernel mem{cpy,set}() to copy the data (while holding the page table lock). This is a win if the regular mem{cpy,set}() implementations are faster than the user copy functions, which is the case e.g. on Feroceon, where 8-word STMs (which memcpy() uses under the right conditions) give significantly higher memory write throughput than a sequence of individual 32bit stores. Here are numbers for page sized buffers on some Feroceon cores: - copy_to_user on Orion5x goes from 51 MB/s to 83 MB/s - clear_user on Orion5x goes from 89MB/s to 314MB/s - copy_to_user on Kirkwood goes from 240 MB/s to 356 MB/s - clear_user on Kirkwood goes from 367 MB/s to 1108 MB/s - copy_to_user on Disco-Duo goes from 248 MB/s to 398 MB/s - clear_user on Disco-Duo goes from 328 MB/s to 1741 MB/s Because the setup cost is non negligible, this is worthwhile only if the amount of data to copy is large enough. The operation falls back to the standard implementation when the amount of data is below a certain threshold. This threshold was determined empirically, however some targets could benefit from a lower runtime determined value for optimal results eventually. In the copy_from_user() case, this technique does not provide any worthwhile performance gain due to the fact that any kind of read access allocates the cache and subsequent 32bit loads are just as fast as the equivalent 8-word LDM. Signed-off-by: Lennert Buytenhek <buytenh@marvell.com> Signed-off-by: Nicolas Pitre <nico@marvell.com> Tested-by: Martin Michlmayr <tbm@cyrius.com>
2009-03-09 21:30:09 +03:00
/*
* linux/arch/arm/lib/uaccess_with_memcpy.c
*
* Written by: Lennert Buytenhek and Nicolas Pitre
* Copyright (C) 2009 Marvell Semiconductor
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/kernel.h>
#include <linux/ctype.h>
#include <linux/uaccess.h>
#include <linux/rwsem.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/hardirq.h> /* for in_atomic() */
#include <asm/current.h>
#include <asm/page.h>
static int
pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
{
unsigned long addr = (unsigned long)_addr;
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
pgd = pgd_offset(current->mm, addr);
if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
return 0;
pmd = pmd_offset(pgd, addr);
if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
return 0;
pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
if (unlikely(!pte_present(*pte) || !pte_young(*pte) ||
!pte_write(*pte) || !pte_dirty(*pte))) {
pte_unmap_unlock(pte, ptl);
return 0;
}
*ptep = pte;
*ptlp = ptl;
return 1;
}
[ARM] lower overhead with alternative copy_to_user for small copies Because the alternate copy_to_user implementation has a higher setup cost than the standard implementation, the size of the memory area to copy is tested and the standard implementation invoked instead when that size is too small. Still, that test is made after the processor has preserved a bunch of registers on the stack which have to be reloaded right away needlessly in that case, causing a measurable performance regression compared to plain usage of the standard implementation only. To make the size test overhead negligible, let's factorize it out of the alternate copy_to_user function where it is clear to the compiler that no stack frame is needed. Thanks to CONFIG_ARM_UNWIND allowing for frame pointers to be disabled and tail call optimization to kick in, the overhead in the small copy case becomes only 3 assembly instructions. A similar trick is applied to clear_user as well. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-22 06:17:17 +04:00
static unsigned long noinline
__copy_to_user_memcpy(void __user *to, const void *from, unsigned long n)
[ARM] alternative copy_to_user/clear_user implementation This implements {copy_to,clear}_user() by faulting in the userland pages and then using the regular kernel mem{cpy,set}() to copy the data (while holding the page table lock). This is a win if the regular mem{cpy,set}() implementations are faster than the user copy functions, which is the case e.g. on Feroceon, where 8-word STMs (which memcpy() uses under the right conditions) give significantly higher memory write throughput than a sequence of individual 32bit stores. Here are numbers for page sized buffers on some Feroceon cores: - copy_to_user on Orion5x goes from 51 MB/s to 83 MB/s - clear_user on Orion5x goes from 89MB/s to 314MB/s - copy_to_user on Kirkwood goes from 240 MB/s to 356 MB/s - clear_user on Kirkwood goes from 367 MB/s to 1108 MB/s - copy_to_user on Disco-Duo goes from 248 MB/s to 398 MB/s - clear_user on Disco-Duo goes from 328 MB/s to 1741 MB/s Because the setup cost is non negligible, this is worthwhile only if the amount of data to copy is large enough. The operation falls back to the standard implementation when the amount of data is below a certain threshold. This threshold was determined empirically, however some targets could benefit from a lower runtime determined value for optimal results eventually. In the copy_from_user() case, this technique does not provide any worthwhile performance gain due to the fact that any kind of read access allocates the cache and subsequent 32bit loads are just as fast as the equivalent 8-word LDM. Signed-off-by: Lennert Buytenhek <buytenh@marvell.com> Signed-off-by: Nicolas Pitre <nico@marvell.com> Tested-by: Martin Michlmayr <tbm@cyrius.com>
2009-03-09 21:30:09 +03:00
{
int atomic;
if (unlikely(segment_eq(get_fs(), KERNEL_DS))) {
memcpy((void *)to, from, n);
return 0;
}
/* the mmap semaphore is taken only if not in an atomic context */
atomic = in_atomic();
if (!atomic)
down_read(&current->mm->mmap_sem);
while (n) {
pte_t *pte;
spinlock_t *ptl;
int tocopy;
while (!pin_page_for_write(to, &pte, &ptl)) {
if (!atomic)
up_read(&current->mm->mmap_sem);
if (__put_user(0, (char __user *)to))
goto out;
if (!atomic)
down_read(&current->mm->mmap_sem);
}
tocopy = (~(unsigned long)to & ~PAGE_MASK) + 1;
if (tocopy > n)
tocopy = n;
memcpy((void *)to, from, tocopy);
to += tocopy;
from += tocopy;
n -= tocopy;
pte_unmap_unlock(pte, ptl);
}
if (!atomic)
up_read(&current->mm->mmap_sem);
out:
return n;
}
[ARM] lower overhead with alternative copy_to_user for small copies Because the alternate copy_to_user implementation has a higher setup cost than the standard implementation, the size of the memory area to copy is tested and the standard implementation invoked instead when that size is too small. Still, that test is made after the processor has preserved a bunch of registers on the stack which have to be reloaded right away needlessly in that case, causing a measurable performance regression compared to plain usage of the standard implementation only. To make the size test overhead negligible, let's factorize it out of the alternate copy_to_user function where it is clear to the compiler that no stack frame is needed. Thanks to CONFIG_ARM_UNWIND allowing for frame pointers to be disabled and tail call optimization to kick in, the overhead in the small copy case becomes only 3 assembly instructions. A similar trick is applied to clear_user as well. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-22 06:17:17 +04:00
unsigned long
__copy_to_user(void __user *to, const void *from, unsigned long n)
{
/*
* This test is stubbed out of the main function above to keep
* the overhead for small copies low by avoiding a large
* register dump on the stack just to reload them right away.
* With frame pointer disabled, tail call optimization kicks in
* as well making this test almost invisible.
*/
[ARM] alternative copy_to_user: more precise fallback threshold Previous size thresholds were guessed from various user space benchmarks using a kernel with and without the alternative uaccess option. This is however not as precise as a kernel based test to measure the real speed of each method. This adds a simple test bench to show the time needed for each method. With this, the optimal size treshold for the alternative implementation can be determined with more confidence. It appears that the optimal threshold for both copy_to_user and clear_user is around 64 bytes. This is not a surprise knowing that the memcpy and memset implementations need at least 64 bytes to achieve maximum throughput. One might suggest that such test be used to determine the optimal threshold at run time instead, but results are near enough to 64 on tested targets concerned by this alternative copy_to_user implementation, so adding some overhead associated with a variable threshold is probably not worth it for now. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-30 05:55:50 +04:00
if (n < 64)
[ARM] lower overhead with alternative copy_to_user for small copies Because the alternate copy_to_user implementation has a higher setup cost than the standard implementation, the size of the memory area to copy is tested and the standard implementation invoked instead when that size is too small. Still, that test is made after the processor has preserved a bunch of registers on the stack which have to be reloaded right away needlessly in that case, causing a measurable performance regression compared to plain usage of the standard implementation only. To make the size test overhead negligible, let's factorize it out of the alternate copy_to_user function where it is clear to the compiler that no stack frame is needed. Thanks to CONFIG_ARM_UNWIND allowing for frame pointers to be disabled and tail call optimization to kick in, the overhead in the small copy case becomes only 3 assembly instructions. A similar trick is applied to clear_user as well. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-22 06:17:17 +04:00
return __copy_to_user_std(to, from, n);
return __copy_to_user_memcpy(to, from, n);
}
static unsigned long noinline
__clear_user_memset(void __user *addr, unsigned long n)
[ARM] alternative copy_to_user/clear_user implementation This implements {copy_to,clear}_user() by faulting in the userland pages and then using the regular kernel mem{cpy,set}() to copy the data (while holding the page table lock). This is a win if the regular mem{cpy,set}() implementations are faster than the user copy functions, which is the case e.g. on Feroceon, where 8-word STMs (which memcpy() uses under the right conditions) give significantly higher memory write throughput than a sequence of individual 32bit stores. Here are numbers for page sized buffers on some Feroceon cores: - copy_to_user on Orion5x goes from 51 MB/s to 83 MB/s - clear_user on Orion5x goes from 89MB/s to 314MB/s - copy_to_user on Kirkwood goes from 240 MB/s to 356 MB/s - clear_user on Kirkwood goes from 367 MB/s to 1108 MB/s - copy_to_user on Disco-Duo goes from 248 MB/s to 398 MB/s - clear_user on Disco-Duo goes from 328 MB/s to 1741 MB/s Because the setup cost is non negligible, this is worthwhile only if the amount of data to copy is large enough. The operation falls back to the standard implementation when the amount of data is below a certain threshold. This threshold was determined empirically, however some targets could benefit from a lower runtime determined value for optimal results eventually. In the copy_from_user() case, this technique does not provide any worthwhile performance gain due to the fact that any kind of read access allocates the cache and subsequent 32bit loads are just as fast as the equivalent 8-word LDM. Signed-off-by: Lennert Buytenhek <buytenh@marvell.com> Signed-off-by: Nicolas Pitre <nico@marvell.com> Tested-by: Martin Michlmayr <tbm@cyrius.com>
2009-03-09 21:30:09 +03:00
{
if (unlikely(segment_eq(get_fs(), KERNEL_DS))) {
memset((void *)addr, 0, n);
return 0;
}
down_read(&current->mm->mmap_sem);
while (n) {
pte_t *pte;
spinlock_t *ptl;
int tocopy;
while (!pin_page_for_write(addr, &pte, &ptl)) {
up_read(&current->mm->mmap_sem);
if (__put_user(0, (char __user *)addr))
goto out;
down_read(&current->mm->mmap_sem);
}
tocopy = (~(unsigned long)addr & ~PAGE_MASK) + 1;
if (tocopy > n)
tocopy = n;
memset((void *)addr, 0, tocopy);
addr += tocopy;
n -= tocopy;
pte_unmap_unlock(pte, ptl);
}
up_read(&current->mm->mmap_sem);
out:
return n;
}
[ARM] lower overhead with alternative copy_to_user for small copies Because the alternate copy_to_user implementation has a higher setup cost than the standard implementation, the size of the memory area to copy is tested and the standard implementation invoked instead when that size is too small. Still, that test is made after the processor has preserved a bunch of registers on the stack which have to be reloaded right away needlessly in that case, causing a measurable performance regression compared to plain usage of the standard implementation only. To make the size test overhead negligible, let's factorize it out of the alternate copy_to_user function where it is clear to the compiler that no stack frame is needed. Thanks to CONFIG_ARM_UNWIND allowing for frame pointers to be disabled and tail call optimization to kick in, the overhead in the small copy case becomes only 3 assembly instructions. A similar trick is applied to clear_user as well. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-22 06:17:17 +04:00
unsigned long __clear_user(void __user *addr, unsigned long n)
{
/* See rational for this in __copy_to_user() above. */
[ARM] alternative copy_to_user: more precise fallback threshold Previous size thresholds were guessed from various user space benchmarks using a kernel with and without the alternative uaccess option. This is however not as precise as a kernel based test to measure the real speed of each method. This adds a simple test bench to show the time needed for each method. With this, the optimal size treshold for the alternative implementation can be determined with more confidence. It appears that the optimal threshold for both copy_to_user and clear_user is around 64 bytes. This is not a surprise knowing that the memcpy and memset implementations need at least 64 bytes to achieve maximum throughput. One might suggest that such test be used to determine the optimal threshold at run time instead, but results are near enough to 64 on tested targets concerned by this alternative copy_to_user implementation, so adding some overhead associated with a variable threshold is probably not worth it for now. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-30 05:55:50 +04:00
if (n < 64)
[ARM] lower overhead with alternative copy_to_user for small copies Because the alternate copy_to_user implementation has a higher setup cost than the standard implementation, the size of the memory area to copy is tested and the standard implementation invoked instead when that size is too small. Still, that test is made after the processor has preserved a bunch of registers on the stack which have to be reloaded right away needlessly in that case, causing a measurable performance regression compared to plain usage of the standard implementation only. To make the size test overhead negligible, let's factorize it out of the alternate copy_to_user function where it is clear to the compiler that no stack frame is needed. Thanks to CONFIG_ARM_UNWIND allowing for frame pointers to be disabled and tail call optimization to kick in, the overhead in the small copy case becomes only 3 assembly instructions. A similar trick is applied to clear_user as well. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-22 06:17:17 +04:00
return __clear_user_std(addr, n);
return __clear_user_memset(addr, n);
}
[ARM] alternative copy_to_user: more precise fallback threshold Previous size thresholds were guessed from various user space benchmarks using a kernel with and without the alternative uaccess option. This is however not as precise as a kernel based test to measure the real speed of each method. This adds a simple test bench to show the time needed for each method. With this, the optimal size treshold for the alternative implementation can be determined with more confidence. It appears that the optimal threshold for both copy_to_user and clear_user is around 64 bytes. This is not a surprise knowing that the memcpy and memset implementations need at least 64 bytes to achieve maximum throughput. One might suggest that such test be used to determine the optimal threshold at run time instead, but results are near enough to 64 on tested targets concerned by this alternative copy_to_user implementation, so adding some overhead associated with a variable threshold is probably not worth it for now. Signed-off-by: Nicolas Pitre <nico@marvell.com>
2009-05-30 05:55:50 +04:00
#if 0
/*
* This code is disabled by default, but kept around in case the chosen
* thresholds need to be revalidated. Some overhead (small but still)
* would be implied by a runtime determined variable threshold, and
* so far the measurement on concerned targets didn't show a worthwhile
* variation.
*
* Note that a fairly precise sched_clock() implementation is needed
* for results to make some sense.
*/
#include <linux/vmalloc.h>
static int __init test_size_treshold(void)
{
struct page *src_page, *dst_page;
void *user_ptr, *kernel_ptr;
unsigned long long t0, t1, t2;
int size, ret;
ret = -ENOMEM;
src_page = alloc_page(GFP_KERNEL);
if (!src_page)
goto no_src;
dst_page = alloc_page(GFP_KERNEL);
if (!dst_page)
goto no_dst;
kernel_ptr = page_address(src_page);
user_ptr = vmap(&dst_page, 1, VM_IOREMAP, __pgprot(__P010));
if (!user_ptr)
goto no_vmap;
/* warm up the src page dcache */
ret = __copy_to_user_memcpy(user_ptr, kernel_ptr, PAGE_SIZE);
for (size = PAGE_SIZE; size >= 4; size /= 2) {
t0 = sched_clock();
ret |= __copy_to_user_memcpy(user_ptr, kernel_ptr, size);
t1 = sched_clock();
ret |= __copy_to_user_std(user_ptr, kernel_ptr, size);
t2 = sched_clock();
printk("copy_to_user: %d %llu %llu\n", size, t1 - t0, t2 - t1);
}
for (size = PAGE_SIZE; size >= 4; size /= 2) {
t0 = sched_clock();
ret |= __clear_user_memset(user_ptr, size);
t1 = sched_clock();
ret |= __clear_user_std(user_ptr, size);
t2 = sched_clock();
printk("clear_user: %d %llu %llu\n", size, t1 - t0, t2 - t1);
}
if (ret)
ret = -EFAULT;
vunmap(user_ptr);
no_vmap:
put_page(dst_page);
no_dst:
put_page(src_page);
no_src:
return ret;
}
subsys_initcall(test_size_treshold);
#endif
Reference in New Issue Copy Permalink