Merge branch 'for-next/futex' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux into for-next/core
This commit is contained in:
Will Deacon
commit
9431ac2bf6
@ -218,5 +218,4 @@ All other architectures should build just fine too - but they won't have
|
||||
the new syscalls yet.
|
||||
|
||||
Architectures need to implement the new futex_atomic_cmpxchg_inatomic()
|
||||
inline function before writing up the syscalls (that function returns
|
||||
-ENOSYS right now).
|
||||
inline function before writing up the syscalls.
|
||||
|
||||
@ -23,26 +23,34 @@
|
||||
|
||||
#include <asm/errno.h>
|
||||
|
||||
#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
|
||||
|
||||
#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
|
||||
do { \
|
||||
unsigned int loops = FUTEX_MAX_LOOPS; \
|
||||
\
|
||||
uaccess_enable(); \
|
||||
asm volatile( \
|
||||
" prfm pstl1strm, %2\n" \
|
||||
"1: ldxr %w1, %2\n" \
|
||||
insn "\n" \
|
||||
"2: stlxr %w3, %w0, %2\n" \
|
||||
" cbnz %w3, 1b\n" \
|
||||
" dmb ish\n" \
|
||||
"2: stlxr %w0, %w3, %2\n" \
|
||||
" cbz %w0, 3f\n" \
|
||||
" sub %w4, %w4, %w0\n" \
|
||||
" cbnz %w4, 1b\n" \
|
||||
" mov %w0, %w7\n" \
|
||||
"3:\n" \
|
||||
" dmb ish\n" \
|
||||
" .pushsection .fixup,\"ax\"\n" \
|
||||
" .align 2\n" \
|
||||
"4: mov %w0, %w5\n" \
|
||||
"4: mov %w0, %w6\n" \
|
||||
" b 3b\n" \
|
||||
" .popsection\n" \
|
||||
_ASM_EXTABLE(1b, 4b) \
|
||||
_ASM_EXTABLE(2b, 4b) \
|
||||
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
|
||||
: "r" (oparg), "Ir" (-EFAULT) \
|
||||
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
|
||||
"+r" (loops) \
|
||||
: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
|
||||
: "memory"); \
|
||||
uaccess_disable(); \
|
||||
} while (0)
|
||||
@ -50,30 +58,30 @@ do { \
|
||||
static inline int
|
||||
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
|
||||
{
|
||||
int oldval = 0, ret, tmp;
|
||||
int oldval, ret, tmp;
|
||||
u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
|
||||
pagefault_disable();
|
||||
|
||||
switch (op) {
|
||||
case FUTEX_OP_SET:
|
||||
__futex_atomic_op("mov %w0, %w4",
|
||||
__futex_atomic_op("mov %w3, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
break;
|
||||
case FUTEX_OP_ADD:
|
||||
__futex_atomic_op("add %w0, %w1, %w4",
|
||||
__futex_atomic_op("add %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
break;
|
||||
case FUTEX_OP_OR:
|
||||
__futex_atomic_op("orr %w0, %w1, %w4",
|
||||
__futex_atomic_op("orr %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
break;
|
||||
case FUTEX_OP_ANDN:
|
||||
__futex_atomic_op("and %w0, %w1, %w4",
|
||||
__futex_atomic_op("and %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, ~oparg);
|
||||
break;
|
||||
case FUTEX_OP_XOR:
|
||||
__futex_atomic_op("eor %w0, %w1, %w4",
|
||||
__futex_atomic_op("eor %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
break;
|
||||
default:
|
||||
@ -93,6 +101,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
|
||||
u32 oldval, u32 newval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
u32 __user *uaddr;
|
||||
|
||||
@ -104,24 +113,30 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
|
||||
asm volatile("// futex_atomic_cmpxchg_inatomic\n"
|
||||
" prfm pstl1strm, %2\n"
|
||||
"1: ldxr %w1, %2\n"
|
||||
" sub %w3, %w1, %w4\n"
|
||||
" cbnz %w3, 3f\n"
|
||||
"2: stlxr %w3, %w5, %2\n"
|
||||
" cbnz %w3, 1b\n"
|
||||
" dmb ish\n"
|
||||
" sub %w3, %w1, %w5\n"
|
||||
" cbnz %w3, 4f\n"
|
||||
"2: stlxr %w3, %w6, %2\n"
|
||||
" cbz %w3, 3f\n"
|
||||
" sub %w4, %w4, %w3\n"
|
||||
" cbnz %w4, 1b\n"
|
||||
" mov %w0, %w8\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
" .pushsection .fixup,\"ax\"\n"
|
||||
"4: mov %w0, %w6\n"
|
||||
" b 3b\n"
|
||||
"5: mov %w0, %w7\n"
|
||||
" b 4b\n"
|
||||
" .popsection\n"
|
||||
_ASM_EXTABLE(1b, 4b)
|
||||
_ASM_EXTABLE(2b, 4b)
|
||||
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
|
||||
: "r" (oldval), "r" (newval), "Ir" (-EFAULT)
|
||||
_ASM_EXTABLE(1b, 5b)
|
||||
_ASM_EXTABLE(2b, 5b)
|
||||
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
|
||||
: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable();
|
||||
|
||||
*uval = val;
|
||||
if (!ret)
|
||||
*uval = val;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@ -23,7 +23,9 @@
|
||||
*
|
||||
* Return:
|
||||
* 0 - On success
|
||||
* <0 - On error
|
||||
* -EFAULT - User access resulted in a page fault
|
||||
* -EAGAIN - Atomic operation was unable to complete due to contention
|
||||
* -ENOSYS - Operation not supported
|
||||
*/
|
||||
static inline int
|
||||
arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr)
|
||||
@ -85,7 +87,9 @@ out_pagefault_enable:
|
||||
*
|
||||
* Return:
|
||||
* 0 - On success
|
||||
* <0 - On error
|
||||
* -EFAULT - User access resulted in a page fault
|
||||
* -EAGAIN - Atomic operation was unable to complete due to contention
|
||||
* -ENOSYS - Function not implemented (only if !HAVE_FUTEX_CMPXCHG)
|
||||
*/
|
||||
static inline int
|
||||
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
|
||||
|
||||
188
kernel/futex.c
188
kernel/futex.c
@ -1311,13 +1311,15 @@ static int lookup_pi_state(u32 __user *uaddr, u32 uval,
|
||||
|
||||
static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
|
||||
{
|
||||
int err;
|
||||
u32 uninitialized_var(curval);
|
||||
|
||||
if (unlikely(should_fail_futex(true)))
|
||||
return -EFAULT;
|
||||
|
||||
if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
|
||||
return -EFAULT;
|
||||
err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
|
||||
/* If user space value changed, let the caller retry */
|
||||
return curval != uval ? -EAGAIN : 0;
|
||||
@ -1502,10 +1504,8 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_
|
||||
if (unlikely(should_fail_futex(true)))
|
||||
ret = -EFAULT;
|
||||
|
||||
if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)) {
|
||||
ret = -EFAULT;
|
||||
|
||||
} else if (curval != uval) {
|
||||
ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
|
||||
if (!ret && (curval != uval)) {
|
||||
/*
|
||||
* If a unconditional UNLOCK_PI operation (user space did not
|
||||
* try the TID->0 transition) raced with a waiter setting the
|
||||
@ -1700,32 +1700,32 @@ retry_private:
|
||||
double_lock_hb(hb1, hb2);
|
||||
op_ret = futex_atomic_op_inuser(op, uaddr2);
|
||||
if (unlikely(op_ret < 0)) {
|
||||
|
||||
double_unlock_hb(hb1, hb2);
|
||||
|
||||
#ifndef CONFIG_MMU
|
||||
/*
|
||||
* we don't get EFAULT from MMU faults if we don't have an MMU,
|
||||
* but we might get them from range checking
|
||||
*/
|
||||
ret = op_ret;
|
||||
goto out_put_keys;
|
||||
#endif
|
||||
|
||||
if (unlikely(op_ret != -EFAULT)) {
|
||||
if (!IS_ENABLED(CONFIG_MMU) ||
|
||||
unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
|
||||
/*
|
||||
* we don't get EFAULT from MMU faults if we don't have
|
||||
* an MMU, but we might get them from range checking
|
||||
*/
|
||||
ret = op_ret;
|
||||
goto out_put_keys;
|
||||
}
|
||||
|
||||
ret = fault_in_user_writeable(uaddr2);
|
||||
if (ret)
|
||||
goto out_put_keys;
|
||||
if (op_ret == -EFAULT) {
|
||||
ret = fault_in_user_writeable(uaddr2);
|
||||
if (ret)
|
||||
goto out_put_keys;
|
||||
}
|
||||
|
||||
if (!(flags & FLAGS_SHARED))
|
||||
if (!(flags & FLAGS_SHARED)) {
|
||||
cond_resched();
|
||||
goto retry_private;
|
||||
}
|
||||
|
||||
put_futex_key(&key2);
|
||||
put_futex_key(&key1);
|
||||
cond_resched();
|
||||
goto retry;
|
||||
}
|
||||
|
||||
@ -2350,7 +2350,7 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
|
||||
u32 uval, uninitialized_var(curval), newval;
|
||||
struct task_struct *oldowner, *newowner;
|
||||
u32 newtid;
|
||||
int ret;
|
||||
int ret, err = 0;
|
||||
|
||||
lockdep_assert_held(q->lock_ptr);
|
||||
|
||||
@ -2421,14 +2421,17 @@ retry:
|
||||
if (!pi_state->owner)
|
||||
newtid |= FUTEX_OWNER_DIED;
|
||||
|
||||
if (get_futex_value_locked(&uval, uaddr))
|
||||
goto handle_fault;
|
||||
err = get_futex_value_locked(&uval, uaddr);
|
||||
if (err)
|
||||
goto handle_err;
|
||||
|
||||
for (;;) {
|
||||
newval = (uval & FUTEX_OWNER_DIED) | newtid;
|
||||
|
||||
if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
|
||||
goto handle_fault;
|
||||
err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
|
||||
if (err)
|
||||
goto handle_err;
|
||||
|
||||
if (curval == uval)
|
||||
break;
|
||||
uval = curval;
|
||||
@ -2456,23 +2459,37 @@ retry:
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* To handle the page fault we need to drop the locks here. That gives
|
||||
* the other task (either the highest priority waiter itself or the
|
||||
* task which stole the rtmutex) the chance to try the fixup of the
|
||||
* pi_state. So once we are back from handling the fault we need to
|
||||
* check the pi_state after reacquiring the locks and before trying to
|
||||
* do another fixup. When the fixup has been done already we simply
|
||||
* return.
|
||||
* In order to reschedule or handle a page fault, we need to drop the
|
||||
* locks here. In the case of a fault, this gives the other task
|
||||
* (either the highest priority waiter itself or the task which stole
|
||||
* the rtmutex) the chance to try the fixup of the pi_state. So once we
|
||||
* are back from handling the fault we need to check the pi_state after
|
||||
* reacquiring the locks and before trying to do another fixup. When
|
||||
* the fixup has been done already we simply return.
|
||||
*
|
||||
* Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely
|
||||
* drop hb->lock since the caller owns the hb -> futex_q relation.
|
||||
* Dropping the pi_mutex->wait_lock requires the state revalidate.
|
||||
*/
|
||||
handle_fault:
|
||||
handle_err:
|
||||
raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
|
||||
spin_unlock(q->lock_ptr);
|
||||
|
||||
ret = fault_in_user_writeable(uaddr);
|
||||
switch (err) {
|
||||
case -EFAULT:
|
||||
ret = fault_in_user_writeable(uaddr);
|
||||
break;
|
||||
|
||||
case -EAGAIN:
|
||||
cond_resched();
|
||||
ret = 0;
|
||||
break;
|
||||
|
||||
default:
|
||||
WARN_ON_ONCE(1);
|
||||
ret = err;
|
||||
break;
|
||||
}
|
||||
|
||||
spin_lock(q->lock_ptr);
|
||||
raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
|
||||
@ -3041,10 +3058,8 @@ retry:
|
||||
* A unconditional UNLOCK_PI op raced against a waiter
|
||||
* setting the FUTEX_WAITERS bit. Try again.
|
||||
*/
|
||||
if (ret == -EAGAIN) {
|
||||
put_futex_key(&key);
|
||||
goto retry;
|
||||
}
|
||||
if (ret == -EAGAIN)
|
||||
goto pi_retry;
|
||||
/*
|
||||
* wake_futex_pi has detected invalid state. Tell user
|
||||
* space.
|
||||
@ -3059,9 +3074,19 @@ retry:
|
||||
* preserve the WAITERS bit not the OWNER_DIED one. We are the
|
||||
* owner.
|
||||
*/
|
||||
if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0)) {
|
||||
if ((ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))) {
|
||||
spin_unlock(&hb->lock);
|
||||
goto pi_faulted;
|
||||
switch (ret) {
|
||||
case -EFAULT:
|
||||
goto pi_faulted;
|
||||
|
||||
case -EAGAIN:
|
||||
goto pi_retry;
|
||||
|
||||
default:
|
||||
WARN_ON_ONCE(1);
|
||||
goto out_putkey;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@ -3075,6 +3100,11 @@ out_putkey:
|
||||
put_futex_key(&key);
|
||||
return ret;
|
||||
|
||||
pi_retry:
|
||||
put_futex_key(&key);
|
||||
cond_resched();
|
||||
goto retry;
|
||||
|
||||
pi_faulted:
|
||||
put_futex_key(&key);
|
||||
|
||||
@ -3435,6 +3465,7 @@ err_unlock:
|
||||
static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi)
|
||||
{
|
||||
u32 uval, uninitialized_var(nval), mval;
|
||||
int err;
|
||||
|
||||
/* Futex address must be 32bit aligned */
|
||||
if ((((unsigned long)uaddr) % sizeof(*uaddr)) != 0)
|
||||
@ -3444,42 +3475,57 @@ retry:
|
||||
if (get_user(uval, uaddr))
|
||||
return -1;
|
||||
|
||||
if ((uval & FUTEX_TID_MASK) == task_pid_vnr(curr)) {
|
||||
/*
|
||||
* Ok, this dying thread is truly holding a futex
|
||||
* of interest. Set the OWNER_DIED bit atomically
|
||||
* via cmpxchg, and if the value had FUTEX_WAITERS
|
||||
* set, wake up a waiter (if any). (We have to do a
|
||||
* futex_wake() even if OWNER_DIED is already set -
|
||||
* to handle the rare but possible case of recursive
|
||||
* thread-death.) The rest of the cleanup is done in
|
||||
* userspace.
|
||||
*/
|
||||
mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
|
||||
/*
|
||||
* We are not holding a lock here, but we want to have
|
||||
* the pagefault_disable/enable() protection because
|
||||
* we want to handle the fault gracefully. If the
|
||||
* access fails we try to fault in the futex with R/W
|
||||
* verification via get_user_pages. get_user() above
|
||||
* does not guarantee R/W access. If that fails we
|
||||
* give up and leave the futex locked.
|
||||
*/
|
||||
if (cmpxchg_futex_value_locked(&nval, uaddr, uval, mval)) {
|
||||
if ((uval & FUTEX_TID_MASK) != task_pid_vnr(curr))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* Ok, this dying thread is truly holding a futex
|
||||
* of interest. Set the OWNER_DIED bit atomically
|
||||
* via cmpxchg, and if the value had FUTEX_WAITERS
|
||||
* set, wake up a waiter (if any). (We have to do a
|
||||
* futex_wake() even if OWNER_DIED is already set -
|
||||
* to handle the rare but possible case of recursive
|
||||
* thread-death.) The rest of the cleanup is done in
|
||||
* userspace.
|
||||
*/
|
||||
mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
|
||||
|
||||
/*
|
||||
* We are not holding a lock here, but we want to have
|
||||
* the pagefault_disable/enable() protection because
|
||||
* we want to handle the fault gracefully. If the
|
||||
* access fails we try to fault in the futex with R/W
|
||||
* verification via get_user_pages. get_user() above
|
||||
* does not guarantee R/W access. If that fails we
|
||||
* give up and leave the futex locked.
|
||||
*/
|
||||
if ((err = cmpxchg_futex_value_locked(&nval, uaddr, uval, mval))) {
|
||||
switch (err) {
|
||||
case -EFAULT:
|
||||
if (fault_in_user_writeable(uaddr))
|
||||
return -1;
|
||||
goto retry;
|
||||
}
|
||||
if (nval != uval)
|
||||
|
||||
case -EAGAIN:
|
||||
cond_resched();
|
||||
goto retry;
|
||||
|
||||
/*
|
||||
* Wake robust non-PI futexes here. The wakeup of
|
||||
* PI futexes happens in exit_pi_state():
|
||||
*/
|
||||
if (!pi && (uval & FUTEX_WAITERS))
|
||||
futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
|
||||
default:
|
||||
WARN_ON_ONCE(1);
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
if (nval != uval)
|
||||
goto retry;
|
||||
|
||||
/*
|
||||
* Wake robust non-PI futexes here. The wakeup of
|
||||
* PI futexes happens in exit_pi_state():
|
||||
*/
|
||||
if (!pi && (uval & FUTEX_WAITERS))
|
||||
futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user