public-mirrors/linux
1
0
Fork 0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-11-19 05:06:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Browse source 
Pull locking fix from Thomas Gleixner:
 "A single commit, which makes the rtmutex.wait_lock an irq safe lock.

  This prevents a potential deadlock which can be triggered by the rcu
  boosting code from rcu_read_unlock()"

* 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  rtmutex: Make wait_lock irq safe
This commit is contained in:
Linus Torvalds 2016-01-31 15:29:37 -08:00
commit bbfb239a10
2 changed files with 81 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
kernel/futex.c
View file

@ -1191,7 +1191,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,
if (pi_state->owner != current) if (pi_state->owner != current)
return -EINVAL; return -EINVAL;
raw_spin_lock(&pi_state->pi_mutex.wait_lock); raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
/* /*
@ -1217,22 +1217,22 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,
else if (curval != uval) else if (curval != uval)
ret = -EINVAL; ret = -EINVAL;
if (ret) { if (ret) {
raw_spin_unlock(&pi_state->pi_mutex.wait_lock); raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
return ret; return ret;
} }
raw_spin_lock_irq(&pi_state->owner->pi_lock); raw_spin_lock(&pi_state->owner->pi_lock);
WARN_ON(list_empty(&pi_state->list)); WARN_ON(list_empty(&pi_state->list));
list_del_init(&pi_state->list); list_del_init(&pi_state->list);
raw_spin_unlock_irq(&pi_state->owner->pi_lock); raw_spin_unlock(&pi_state->owner->pi_lock);
raw_spin_lock_irq(&new_owner->pi_lock); raw_spin_lock(&new_owner->pi_lock);
WARN_ON(!list_empty(&pi_state->list)); WARN_ON(!list_empty(&pi_state->list));
list_add(&pi_state->list, &new_owner->pi_state_list); list_add(&pi_state->list, &new_owner->pi_state_list);
pi_state->owner = new_owner; pi_state->owner = new_owner;
raw_spin_unlock_irq(&new_owner->pi_lock); raw_spin_unlock(&new_owner->pi_lock);
raw_spin_unlock(&pi_state->pi_mutex.wait_lock); raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q); deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
@ -2127,11 +2127,11 @@ static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)
* we returned due to timeout or signal without taking the * we returned due to timeout or signal without taking the
* rt_mutex. Too late. * rt_mutex. Too late.
*/ */
raw_spin_lock(&q->pi_state->pi_mutex.wait_lock); raw_spin_lock_irq(&q->pi_state->pi_mutex.wait_lock);
owner = rt_mutex_owner(&q->pi_state->pi_mutex); owner = rt_mutex_owner(&q->pi_state->pi_mutex);
if (!owner) if (!owner)
owner = rt_mutex_next_owner(&q->pi_state->pi_mutex); owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);
raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock); raw_spin_unlock_irq(&q->pi_state->pi_mutex.wait_lock);
ret = fixup_pi_state_owner(uaddr, q, owner); ret = fixup_pi_state_owner(uaddr, q, owner);
goto out; goto out;
} }

135
kernel/locking/rtmutex.c
View file

@ -99,13 +99,14 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
* 2) Drop lock->wait_lock * 2) Drop lock->wait_lock
* 3) Try to unlock the lock with cmpxchg * 3) Try to unlock the lock with cmpxchg
*/ */
static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock) static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
unsigned long flags)
__releases(lock->wait_lock) __releases(lock->wait_lock)
{ {
struct task_struct *owner = rt_mutex_owner(lock); struct task_struct *owner = rt_mutex_owner(lock);
clear_rt_mutex_waiters(lock); clear_rt_mutex_waiters(lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
/* /*
* If a new waiter comes in between the unlock and the cmpxchg * If a new waiter comes in between the unlock and the cmpxchg
* we have two situations: * we have two situations:
@ -147,11 +148,12 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
/* /*
* Simple slow path only version: lock->owner is protected by lock->wait_lock. * Simple slow path only version: lock->owner is protected by lock->wait_lock.
*/ */
static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock) static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
unsigned long flags)
__releases(lock->wait_lock) __releases(lock->wait_lock)
{ {
lock->owner = NULL; lock->owner = NULL;
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
return true; return true;
} }
#endif #endif
@ -433,7 +435,6 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
int ret = 0, depth = 0; int ret = 0, depth = 0;
struct rt_mutex *lock; struct rt_mutex *lock;
bool detect_deadlock; bool detect_deadlock;
unsigned long flags;
bool requeue = true; bool requeue = true;
detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk); detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
@ -476,7 +477,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
/* /*
* [1] Task cannot go away as we did a get_task() before ! * [1] Task cannot go away as we did a get_task() before !
*/ */
raw_spin_lock_irqsave(&task->pi_lock, flags); raw_spin_lock_irq(&task->pi_lock);
/* /*
* [2] Get the waiter on which @task is blocked on. * [2] Get the waiter on which @task is blocked on.
@ -560,7 +561,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
* operations. * operations.
*/ */
if (!raw_spin_trylock(&lock->wait_lock)) { if (!raw_spin_trylock(&lock->wait_lock)) {
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock_irq(&task->pi_lock);
cpu_relax(); cpu_relax();
goto retry; goto retry;
} }
@ -591,7 +592,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
/* /*
* No requeue[7] here. Just release @task [8] * No requeue[7] here. Just release @task [8]
*/ */
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock(&task->pi_lock);
put_task_struct(task); put_task_struct(task);
/* /*
@ -599,14 +600,14 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
* If there is no owner of the lock, end of chain. * If there is no owner of the lock, end of chain.
*/ */
if (!rt_mutex_owner(lock)) { if (!rt_mutex_owner(lock)) {
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
return 0; return 0;
} }
/* [10] Grab the next task, i.e. owner of @lock */ /* [10] Grab the next task, i.e. owner of @lock */
task = rt_mutex_owner(lock); task = rt_mutex_owner(lock);
get_task_struct(task); get_task_struct(task);
raw_spin_lock_irqsave(&task->pi_lock, flags); raw_spin_lock(&task->pi_lock);
/* /*
* No requeue [11] here. We just do deadlock detection. * No requeue [11] here. We just do deadlock detection.
@ -621,8 +622,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
top_waiter = rt_mutex_top_waiter(lock); top_waiter = rt_mutex_top_waiter(lock);
/* [13] Drop locks */ /* [13] Drop locks */
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock(&task->pi_lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
/* If owner is not blocked, end of chain. */ /* If owner is not blocked, end of chain. */
if (!next_lock) if (!next_lock)
@ -643,7 +644,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
rt_mutex_enqueue(lock, waiter); rt_mutex_enqueue(lock, waiter);
/* [8] Release the task */ /* [8] Release the task */
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock(&task->pi_lock);
put_task_struct(task); put_task_struct(task);
/* /*
@ -661,14 +662,14 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
*/ */
if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
wake_up_process(rt_mutex_top_waiter(lock)->task); wake_up_process(rt_mutex_top_waiter(lock)->task);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
return 0; return 0;
} }
/* [10] Grab the next task, i.e. the owner of @lock */ /* [10] Grab the next task, i.e. the owner of @lock */
task = rt_mutex_owner(lock); task = rt_mutex_owner(lock);
get_task_struct(task); get_task_struct(task);
raw_spin_lock_irqsave(&task->pi_lock, flags); raw_spin_lock(&task->pi_lock);
/* [11] requeue the pi waiters if necessary */ /* [11] requeue the pi waiters if necessary */
if (waiter == rt_mutex_top_waiter(lock)) { if (waiter == rt_mutex_top_waiter(lock)) {
@ -722,8 +723,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
top_waiter = rt_mutex_top_waiter(lock); top_waiter = rt_mutex_top_waiter(lock);
/* [13] Drop the locks */ /* [13] Drop the locks */
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock(&task->pi_lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
/* /*
* Make the actual exit decisions [12], based on the stored * Make the actual exit decisions [12], based on the stored
@ -746,7 +747,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
goto again; goto again;
out_unlock_pi: out_unlock_pi:
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock_irq(&task->pi_lock);
out_put_task: out_put_task:
put_task_struct(task); put_task_struct(task);
@ -756,7 +757,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
/* /*
* Try to take an rt-mutex * Try to take an rt-mutex
* *
* Must be called with lock->wait_lock held. * Must be called with lock->wait_lock held and interrupts disabled
* *
* @lock: The lock to be acquired. * @lock: The lock to be acquired.
* @task: The task which wants to acquire the lock * @task: The task which wants to acquire the lock
@ -766,8 +767,6 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
struct rt_mutex_waiter *waiter) struct rt_mutex_waiter *waiter)
{ {
unsigned long flags;
/* /*
* Before testing whether we can acquire @lock, we set the * Before testing whether we can acquire @lock, we set the
* RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
@ -852,7 +851,7 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
* case, but conditionals are more expensive than a redundant * case, but conditionals are more expensive than a redundant
* store. * store.
*/ */
raw_spin_lock_irqsave(&task->pi_lock, flags); raw_spin_lock(&task->pi_lock);
task->pi_blocked_on = NULL; task->pi_blocked_on = NULL;
/* /*
* Finish the lock acquisition. @task is the new owner. If * Finish the lock acquisition. @task is the new owner. If
@ -861,7 +860,7 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
*/ */
if (rt_mutex_has_waiters(lock)) if (rt_mutex_has_waiters(lock))
rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock)); rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock(&task->pi_lock);
takeit: takeit:
/* We got the lock. */ /* We got the lock. */
@ -883,7 +882,7 @@ takeit:
* *
* Prepare waiter and propagate pi chain * Prepare waiter and propagate pi chain
* *
* This must be called with lock->wait_lock held. * This must be called with lock->wait_lock held and interrupts disabled
*/ */
static int task_blocks_on_rt_mutex(struct rt_mutex *lock, static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter, struct rt_mutex_waiter *waiter,
@ -894,7 +893,6 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
struct rt_mutex_waiter *top_waiter = waiter; struct rt_mutex_waiter *top_waiter = waiter;
struct rt_mutex *next_lock; struct rt_mutex *next_lock;
int chain_walk = 0, res; int chain_walk = 0, res;
unsigned long flags;
/* /*
* Early deadlock detection. We really don't want the task to * Early deadlock detection. We really don't want the task to
@ -908,7 +906,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
if (owner == task) if (owner == task)
return -EDEADLK; return -EDEADLK;
raw_spin_lock_irqsave(&task->pi_lock, flags); raw_spin_lock(&task->pi_lock);
__rt_mutex_adjust_prio(task); __rt_mutex_adjust_prio(task);
waiter->task = task; waiter->task = task;
waiter->lock = lock; waiter->lock = lock;
@ -921,12 +919,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
task->pi_blocked_on = waiter; task->pi_blocked_on = waiter;
raw_spin_unlock_irqrestore(&task->pi_lock, flags); raw_spin_unlock(&task->pi_lock);
if (!owner) if (!owner)
return 0; return 0;
raw_spin_lock_irqsave(&owner->pi_lock, flags); raw_spin_lock(&owner->pi_lock);
if (waiter == rt_mutex_top_waiter(lock)) { if (waiter == rt_mutex_top_waiter(lock)) {
rt_mutex_dequeue_pi(owner, top_waiter); rt_mutex_dequeue_pi(owner, top_waiter);
rt_mutex_enqueue_pi(owner, waiter); rt_mutex_enqueue_pi(owner, waiter);
@ -941,7 +939,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
/* Store the lock on which owner is blocked or NULL */ /* Store the lock on which owner is blocked or NULL */
next_lock = task_blocked_on_lock(owner); next_lock = task_blocked_on_lock(owner);
raw_spin_unlock_irqrestore(&owner->pi_lock, flags); raw_spin_unlock(&owner->pi_lock);
/* /*
* Even if full deadlock detection is on, if the owner is not * Even if full deadlock detection is on, if the owner is not
* blocked itself, we can avoid finding this out in the chain * blocked itself, we can avoid finding this out in the chain
@ -957,12 +955,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
*/ */
get_task_struct(owner); get_task_struct(owner);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
res = rt_mutex_adjust_prio_chain(owner, chwalk, lock, res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
next_lock, waiter, task); next_lock, waiter, task);
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irq(&lock->wait_lock);
return res; return res;
} }
@ -971,15 +969,14 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
* Remove the top waiter from the current tasks pi waiter tree and * Remove the top waiter from the current tasks pi waiter tree and
* queue it up. * queue it up.
* *
* Called with lock->wait_lock held. * Called with lock->wait_lock held and interrupts disabled.
*/ */
static void mark_wakeup_next_waiter(struct wake_q_head *wake_q, static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
struct rt_mutex *lock) struct rt_mutex *lock)
{ {
struct rt_mutex_waiter *waiter; struct rt_mutex_waiter *waiter;
unsigned long flags;
raw_spin_lock_irqsave(&current->pi_lock, flags); raw_spin_lock(&current->pi_lock);
waiter = rt_mutex_top_waiter(lock); waiter = rt_mutex_top_waiter(lock);
@ -1001,7 +998,7 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
*/ */
lock->owner = (void *) RT_MUTEX_HAS_WAITERS; lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
raw_spin_unlock_irqrestore(&current->pi_lock, flags); raw_spin_unlock(&current->pi_lock);
wake_q_add(wake_q, waiter->task); wake_q_add(wake_q, waiter->task);
} }
@ -1009,7 +1006,7 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
/* /*
* Remove a waiter from a lock and give up * Remove a waiter from a lock and give up
* *
* Must be called with lock->wait_lock held and * Must be called with lock->wait_lock held and interrupts disabled. I must
* have just failed to try_to_take_rt_mutex(). * have just failed to try_to_take_rt_mutex().
*/ */
static void remove_waiter(struct rt_mutex *lock, static void remove_waiter(struct rt_mutex *lock,
@ -1018,12 +1015,11 @@ static void remove_waiter(struct rt_mutex *lock,
bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock); struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex *next_lock; struct rt_mutex *next_lock;
unsigned long flags;
raw_spin_lock_irqsave(&current->pi_lock, flags); raw_spin_lock(&current->pi_lock);
rt_mutex_dequeue(lock, waiter); rt_mutex_dequeue(lock, waiter);
current->pi_blocked_on = NULL; current->pi_blocked_on = NULL;
raw_spin_unlock_irqrestore(&current->pi_lock, flags); raw_spin_unlock(&current->pi_lock);
/* /*
* Only update priority if the waiter was the highest priority * Only update priority if the waiter was the highest priority
@ -1032,7 +1028,7 @@ static void remove_waiter(struct rt_mutex *lock,
if (!owner || !is_top_waiter) if (!owner || !is_top_waiter)
return; return;
raw_spin_lock_irqsave(&owner->pi_lock, flags); raw_spin_lock(&owner->pi_lock);
rt_mutex_dequeue_pi(owner, waiter); rt_mutex_dequeue_pi(owner, waiter);
@ -1044,7 +1040,7 @@ static void remove_waiter(struct rt_mutex *lock,
/* Store the lock on which owner is blocked or NULL */ /* Store the lock on which owner is blocked or NULL */
next_lock = task_blocked_on_lock(owner); next_lock = task_blocked_on_lock(owner);
raw_spin_unlock_irqrestore(&owner->pi_lock, flags); raw_spin_unlock(&owner->pi_lock);
/* /*
* Don't walk the chain, if the owner task is not blocked * Don't walk the chain, if the owner task is not blocked
@ -1056,12 +1052,12 @@ static void remove_waiter(struct rt_mutex *lock,
/* gets dropped in rt_mutex_adjust_prio_chain()! */ /* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(owner); get_task_struct(owner);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
next_lock, NULL, current); next_lock, NULL, current);
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irq(&lock->wait_lock);
} }
/* /*
@ -1097,11 +1093,11 @@ void rt_mutex_adjust_pi(struct task_struct *task)
* __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
* @lock: the rt_mutex to take * @lock: the rt_mutex to take
* @state: the state the task should block in (TASK_INTERRUPTIBLE * @state: the state the task should block in (TASK_INTERRUPTIBLE
* or TASK_UNINTERRUPTIBLE) * or TASK_UNINTERRUPTIBLE)
* @timeout: the pre-initialized and started timer, or NULL for none * @timeout: the pre-initialized and started timer, or NULL for none
* @waiter: the pre-initialized rt_mutex_waiter * @waiter: the pre-initialized rt_mutex_waiter
* *
* lock->wait_lock must be held by the caller. * Must be called with lock->wait_lock held and interrupts disabled
*/ */
static int __sched static int __sched
__rt_mutex_slowlock(struct rt_mutex *lock, int state, __rt_mutex_slowlock(struct rt_mutex *lock, int state,
@ -1129,13 +1125,13 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state,
break; break;
} }
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
debug_rt_mutex_print_deadlock(waiter); debug_rt_mutex_print_deadlock(waiter);
schedule(); schedule();
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irq(&lock->wait_lock);
set_current_state(state); set_current_state(state);
} }
@ -1172,17 +1168,26 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
enum rtmutex_chainwalk chwalk) enum rtmutex_chainwalk chwalk)
{ {
struct rt_mutex_waiter waiter; struct rt_mutex_waiter waiter;
unsigned long flags;
int ret = 0; int ret = 0;
debug_rt_mutex_init_waiter(&waiter); debug_rt_mutex_init_waiter(&waiter);
RB_CLEAR_NODE(&waiter.pi_tree_entry); RB_CLEAR_NODE(&waiter.pi_tree_entry);
RB_CLEAR_NODE(&waiter.tree_entry); RB_CLEAR_NODE(&waiter.tree_entry);
raw_spin_lock(&lock->wait_lock); /*
* Technically we could use raw_spin_[un]lock_irq() here, but this can
* be called in early boot if the cmpxchg() fast path is disabled
* (debug, no architecture support). In this case we will acquire the
* rtmutex with lock->wait_lock held. But we cannot unconditionally
* enable interrupts in that early boot case. So we need to use the
* irqsave/restore variants.
*/
raw_spin_lock_irqsave(&lock->wait_lock, flags);
/* Try to acquire the lock again: */ /* Try to acquire the lock again: */
if (try_to_take_rt_mutex(lock, current, NULL)) { if (try_to_take_rt_mutex(lock, current, NULL)) {
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
return 0; return 0;
} }
@ -1211,7 +1216,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
*/ */
fixup_rt_mutex_waiters(lock); fixup_rt_mutex_waiters(lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
/* Remove pending timer: */ /* Remove pending timer: */
if (unlikely(timeout)) if (unlikely(timeout))
@ -1227,6 +1232,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
*/ */
static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
{ {
unsigned long flags;
int ret; int ret;
/* /*
@ -1238,10 +1244,10 @@ static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
return 0; return 0;
/* /*
* The mutex has currently no owner. Lock the wait lock and * The mutex has currently no owner. Lock the wait lock and try to
* try to acquire the lock. * acquire the lock. We use irqsave here to support early boot calls.
*/ */
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irqsave(&lock->wait_lock, flags);
ret = try_to_take_rt_mutex(lock, current, NULL); ret = try_to_take_rt_mutex(lock, current, NULL);
@ -1251,7 +1257,7 @@ static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
*/ */
fixup_rt_mutex_waiters(lock); fixup_rt_mutex_waiters(lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
return ret; return ret;
} }
@ -1263,7 +1269,10 @@ static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
struct wake_q_head *wake_q) struct wake_q_head *wake_q)
{ {
raw_spin_lock(&lock->wait_lock); unsigned long flags;
/* irqsave required to support early boot calls */
raw_spin_lock_irqsave(&lock->wait_lock, flags);
debug_rt_mutex_unlock(lock); debug_rt_mutex_unlock(lock);
@ -1302,10 +1311,10 @@ static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
*/ */
while (!rt_mutex_has_waiters(lock)) { while (!rt_mutex_has_waiters(lock)) {
/* Drops lock->wait_lock ! */ /* Drops lock->wait_lock ! */
if (unlock_rt_mutex_safe(lock) == true) if (unlock_rt_mutex_safe(lock, flags) == true)
return false; return false;
/* Relock the rtmutex and try again */ /* Relock the rtmutex and try again */
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irqsave(&lock->wait_lock, flags);
} }
/* /*
@ -1316,7 +1325,7 @@ static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
*/ */
mark_wakeup_next_waiter(wake_q, lock); mark_wakeup_next_waiter(wake_q, lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
/* check PI boosting */ /* check PI boosting */
return true; return true;
@ -1596,10 +1605,10 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
{ {
int ret; int ret;
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irq(&lock->wait_lock);
if (try_to_take_rt_mutex(lock, task, NULL)) { if (try_to_take_rt_mutex(lock, task, NULL)) {
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
return 1; return 1;
} }
@ -1620,7 +1629,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
if (unlikely(ret)) if (unlikely(ret))
remove_waiter(lock, waiter); remove_waiter(lock, waiter);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
debug_rt_mutex_print_deadlock(waiter); debug_rt_mutex_print_deadlock(waiter);
@ -1668,7 +1677,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
{ {
int ret; int ret;
raw_spin_lock(&lock->wait_lock); raw_spin_lock_irq(&lock->wait_lock);
set_current_state(TASK_INTERRUPTIBLE); set_current_state(TASK_INTERRUPTIBLE);
@ -1684,7 +1693,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
*/ */
fixup_rt_mutex_waiters(lock); fixup_rt_mutex_waiters(lock);
raw_spin_unlock(&lock->wait_lock); raw_spin_unlock_irq(&lock->wait_lock);
return ret; return ret;
} }