public-mirrors/linux
1
0
Fork 0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-09-18 22:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge tip:locking/core into tip:ras/core

Browse source 
Pick up helpers inlining work in order to address more noinstr fallout
in the MCE code.

Signed-off-by: Borislav Petkov <bp@suse.de>
This commit is contained in:
Borislav Petkov 2022-02-13 22:03:04 +01:00
commit c0f6799de2
14 changed files with 108 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
MAINTAINERS
View file

@ -3200,6 +3200,7 @@ ATOMIC INFRASTRUCTURE
M: Will Deacon <will@kernel.org> M: Will Deacon <will@kernel.org>
M: Peter Zijlstra <peterz@infradead.org> M: Peter Zijlstra <peterz@infradead.org>
R: Boqun Feng <boqun.feng@gmail.com> R: Boqun Feng <boqun.feng@gmail.com>
R: Mark Rutland <mark.rutland@arm.com>
L: linux-kernel@vger.kernel.org L: linux-kernel@vger.kernel.org
S: Maintained S: Maintained
F: arch/*/include/asm/atomic*.h F: arch/*/include/asm/atomic*.h

10
arch/x86/include/asm/cpumask.h
View file

@ -20,11 +20,21 @@ static __always_inline bool arch_cpu_online(int cpu)
{ {
return arch_test_bit(cpu, cpumask_bits(cpu_online_mask)); return arch_test_bit(cpu, cpumask_bits(cpu_online_mask));
} }
static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{
arch_clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
}
#else #else
static __always_inline bool arch_cpu_online(int cpu) static __always_inline bool arch_cpu_online(int cpu)
{ {
return cpu == 0; return cpu == 0;
} }
static __always_inline void arch_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{
return;
}
#endif #endif
#define arch_cpu_is_offline(cpu) unlikely(!arch_cpu_online(cpu)) #define arch_cpu_is_offline(cpu) unlikely(!arch_cpu_online(cpu))

2
arch/x86/include/asm/ptrace.h
View file

@ -137,7 +137,7 @@ static __always_inline int user_mode(struct pt_regs *regs)
#endif #endif
} }
static inline int v8086_mode(struct pt_regs *regs) static __always_inline int v8086_mode(struct pt_regs *regs)
{ {
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
return (regs->flags & X86_VM_MASK); return (regs->flags & X86_VM_MASK);

12
include/asm-generic/bitops/instrumented-atomic.h
View file

@ -23,7 +23,7 @@
* Note that @nr may be almost arbitrarily large; this function is not * Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity. * restricted to acting on a single-word quantity.
*/ */
static inline void set_bit(long nr, volatile unsigned long *addr) static __always_inline void set_bit(long nr, volatile unsigned long *addr)
{ {
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_set_bit(nr, addr); arch_set_bit(nr, addr);
@ -36,7 +36,7 @@ static inline void set_bit(long nr, volatile unsigned long *addr)
* *
* This is a relaxed atomic operation (no implied memory barriers). * This is a relaxed atomic operation (no implied memory barriers).
*/ */
static inline void clear_bit(long nr, volatile unsigned long *addr) static __always_inline void clear_bit(long nr, volatile unsigned long *addr)
{ {
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_clear_bit(nr, addr); arch_clear_bit(nr, addr);
@ -52,7 +52,7 @@ static inline void clear_bit(long nr, volatile unsigned long *addr)
* Note that @nr may be almost arbitrarily large; this function is not * Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity. * restricted to acting on a single-word quantity.
*/ */
static inline void change_bit(long nr, volatile unsigned long *addr) static __always_inline void change_bit(long nr, volatile unsigned long *addr)
{ {
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_change_bit(nr, addr); arch_change_bit(nr, addr);
@ -65,7 +65,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
* *
* This is an atomic fully-ordered operation (implied full memory barrier). * This is an atomic fully-ordered operation (implied full memory barrier).
*/ */
static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
{ {
kcsan_mb(); kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@ -79,7 +79,7 @@ static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
* *
* This is an atomic fully-ordered operation (implied full memory barrier). * This is an atomic fully-ordered operation (implied full memory barrier).
*/ */
static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
{ {
kcsan_mb(); kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@ -93,7 +93,7 @@ static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
* *
* This is an atomic fully-ordered operation (implied full memory barrier). * This is an atomic fully-ordered operation (implied full memory barrier).
*/ */
static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
{ {
kcsan_mb(); kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));

16
include/asm-generic/bitops/instrumented-non-atomic.h
View file

@ -22,7 +22,7 @@
* region of memory concurrently, the effect may be that only one operation * region of memory concurrently, the effect may be that only one operation
* succeeds. * succeeds.
*/ */
static inline void __set_bit(long nr, volatile unsigned long *addr) static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
{ {
instrument_write(addr + BIT_WORD(nr), sizeof(long)); instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___set_bit(nr, addr); arch___set_bit(nr, addr);
@ -37,7 +37,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr)
* region of memory concurrently, the effect may be that only one operation * region of memory concurrently, the effect may be that only one operation
* succeeds. * succeeds.
*/ */
static inline void __clear_bit(long nr, volatile unsigned long *addr) static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
{ {
instrument_write(addr + BIT_WORD(nr), sizeof(long)); instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___clear_bit(nr, addr); arch___clear_bit(nr, addr);
@ -52,13 +52,13 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr)
* region of memory concurrently, the effect may be that only one operation * region of memory concurrently, the effect may be that only one operation
* succeeds. * succeeds.
*/ */
static inline void __change_bit(long nr, volatile unsigned long *addr) static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
{ {
instrument_write(addr + BIT_WORD(nr), sizeof(long)); instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___change_bit(nr, addr); arch___change_bit(nr, addr);
} }
static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr) static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
{ {
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) { if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
/* /*
@ -90,7 +90,7 @@ static inline void __instrument_read_write_bitop(long nr, volatile unsigned long
* This operation is non-atomic. If two instances of this operation race, one * This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail. * can appear to succeed but actually fail.
*/ */
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
{ {
__instrument_read_write_bitop(nr, addr); __instrument_read_write_bitop(nr, addr);
return arch___test_and_set_bit(nr, addr); return arch___test_and_set_bit(nr, addr);
@ -104,7 +104,7 @@ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
* This operation is non-atomic. If two instances of this operation race, one * This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail. * can appear to succeed but actually fail.
*/ */
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
{ {
__instrument_read_write_bitop(nr, addr); __instrument_read_write_bitop(nr, addr);
return arch___test_and_clear_bit(nr, addr); return arch___test_and_clear_bit(nr, addr);
@ -118,7 +118,7 @@ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
* This operation is non-atomic. If two instances of this operation race, one * This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail. * can appear to succeed but actually fail.
*/ */
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
{ {
__instrument_read_write_bitop(nr, addr); __instrument_read_write_bitop(nr, addr);
return arch___test_and_change_bit(nr, addr); return arch___test_and_change_bit(nr, addr);
@ -129,7 +129,7 @@ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
* @nr: bit number to test * @nr: bit number to test
* @addr: Address to start counting from * @addr: Address to start counting from
*/ */
static inline bool test_bit(long nr, const volatile unsigned long *addr) static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
{ {
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
return arch_test_bit(nr, addr); return arch_test_bit(nr, addr);

38
include/linux/atomic/atomic-arch-fallback.h
View file

@ -151,7 +151,16 @@
static __always_inline int static __always_inline int
arch_atomic_read_acquire(const atomic_t *v) arch_atomic_read_acquire(const atomic_t *v)
{ {
return smp_load_acquire(&(v)->counter); int ret;
if (__native_word(atomic_t)) {
ret = smp_load_acquire(&(v)->counter);
} else {
ret = arch_atomic_read(v);
__atomic_acquire_fence();
}
return ret;
} }
#define arch_atomic_read_acquire arch_atomic_read_acquire #define arch_atomic_read_acquire arch_atomic_read_acquire
#endif #endif
@ -160,7 +169,12 @@ arch_atomic_read_acquire(const atomic_t *v)
static __always_inline void static __always_inline void
arch_atomic_set_release(atomic_t *v, int i) arch_atomic_set_release(atomic_t *v, int i)
{ {
smp_store_release(&(v)->counter, i); if (__native_word(atomic_t)) {
smp_store_release(&(v)->counter, i);
} else {
__atomic_release_fence();
arch_atomic_set(v, i);
}
} }
#define arch_atomic_set_release arch_atomic_set_release #define arch_atomic_set_release arch_atomic_set_release
#endif #endif
@ -1258,7 +1272,16 @@ arch_atomic_dec_if_positive(atomic_t *v)
static __always_inline s64 static __always_inline s64
arch_atomic64_read_acquire(const atomic64_t *v) arch_atomic64_read_acquire(const atomic64_t *v)
{ {
return smp_load_acquire(&(v)->counter); s64 ret;
if (__native_word(atomic64_t)) {
ret = smp_load_acquire(&(v)->counter);
} else {
ret = arch_atomic64_read(v);
__atomic_acquire_fence();
}
return ret;
} }
#define arch_atomic64_read_acquire arch_atomic64_read_acquire #define arch_atomic64_read_acquire arch_atomic64_read_acquire
#endif #endif
@ -1267,7 +1290,12 @@ arch_atomic64_read_acquire(const atomic64_t *v)
static __always_inline void static __always_inline void
arch_atomic64_set_release(atomic64_t *v, s64 i) arch_atomic64_set_release(atomic64_t *v, s64 i)
{ {
smp_store_release(&(v)->counter, i); if (__native_word(atomic64_t)) {
smp_store_release(&(v)->counter, i);
} else {
__atomic_release_fence();
arch_atomic64_set(v, i);
}
} }
#define arch_atomic64_set_release arch_atomic64_set_release #define arch_atomic64_set_release arch_atomic64_set_release
#endif #endif
@ -2358,4 +2386,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
#endif #endif
#endif /* _LINUX_ATOMIC_FALLBACK_H */ #endif /* _LINUX_ATOMIC_FALLBACK_H */
// cca554917d7ea73d5e3e7397dd70c484cad9b2c4 // 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae

18
include/linux/cpumask.h
View file

@ -102,7 +102,7 @@ extern atomic_t __num_online_cpus;
extern cpumask_t cpus_booted_once_mask; extern cpumask_t cpus_booted_once_mask;
static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
{ {
#ifdef CONFIG_DEBUG_PER_CPU_MAPS #ifdef CONFIG_DEBUG_PER_CPU_MAPS
WARN_ON_ONCE(cpu >= bits); WARN_ON_ONCE(cpu >= bits);
@ -110,7 +110,7 @@ static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
} }
/* verify cpu argument to cpumask_* operators */ /* verify cpu argument to cpumask_* operators */
static inline unsigned int cpumask_check(unsigned int cpu) static __always_inline unsigned int cpumask_check(unsigned int cpu)
{ {
cpu_max_bits_warn(cpu, nr_cpumask_bits); cpu_max_bits_warn(cpu, nr_cpumask_bits);
return cpu; return cpu;
@ -341,12 +341,12 @@ extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool
* @cpu: cpu number (< nr_cpu_ids) * @cpu: cpu number (< nr_cpu_ids)
* @dstp: the cpumask pointer * @dstp: the cpumask pointer
*/ */
static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
{ {
set_bit(cpumask_check(cpu), cpumask_bits(dstp)); set_bit(cpumask_check(cpu), cpumask_bits(dstp));
} }
static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
{ {
__set_bit(cpumask_check(cpu), cpumask_bits(dstp)); __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
} }
@ -357,12 +357,12 @@ static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
* @cpu: cpu number (< nr_cpu_ids) * @cpu: cpu number (< nr_cpu_ids)
* @dstp: the cpumask pointer * @dstp: the cpumask pointer
*/ */
static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{ {
clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
} }
static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{ {
__clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
} }
@ -374,7 +374,7 @@ static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
* *
* Returns 1 if @cpu is set in @cpumask, else returns 0 * Returns 1 if @cpu is set in @cpumask, else returns 0
*/ */
static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
{ {
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
} }
@ -388,7 +388,7 @@ static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
* *
* test_and_set_bit wrapper for cpumasks. * test_and_set_bit wrapper for cpumasks.
*/ */
static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
{ {
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
} }
@ -402,7 +402,7 @@ static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
* *
* test_and_clear_bit wrapper for cpumasks. * test_and_clear_bit wrapper for cpumasks.
*/ */
static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
{ {
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
} }

6
include/linux/local_lock_internal.h
View file

@ -44,9 +44,9 @@ static inline void local_lock_debug_init(local_lock_t *l)
} }
#else /* CONFIG_DEBUG_LOCK_ALLOC */ #else /* CONFIG_DEBUG_LOCK_ALLOC */
# define LOCAL_LOCK_DEBUG_INIT(lockname) # define LOCAL_LOCK_DEBUG_INIT(lockname)
static inline void local_lock_acquire(local_lock_t *l) { } # define local_lock_acquire(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
static inline void local_lock_release(local_lock_t *l) { } # define local_lock_release(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
static inline void local_lock_debug_init(local_lock_t *l) { } # define local_lock_debug_init(__ll) do { typecheck(local_lock_t *, __ll); } while (0)
#endif /* !CONFIG_DEBUG_LOCK_ALLOC */ #endif /* !CONFIG_DEBUG_LOCK_ALLOC */
#define INIT_LOCAL_LOCK(lockname) { LOCAL_LOCK_DEBUG_INIT(lockname) } #define INIT_LOCAL_LOCK(lockname) { LOCAL_LOCK_DEBUG_INIT(lockname) }

1
init/Kconfig
View file

@ -2054,6 +2054,7 @@ source "arch/Kconfig"
config RT_MUTEXES config RT_MUTEXES
bool bool
default y if PREEMPT_RT
config BASE_SMALL config BASE_SMALL
int int

29
kernel/locking/lockdep.c
View file

@ -6011,13 +6011,10 @@ static void zap_class(struct pending_free *pf, struct lock_class *class)
static void reinit_class(struct lock_class *class) static void reinit_class(struct lock_class *class)
{ {
void *const p = class;
const unsigned int offset = offsetof(struct lock_class, key);
WARN_ON_ONCE(!class->lock_entry.next); WARN_ON_ONCE(!class->lock_entry.next);
WARN_ON_ONCE(!list_empty(&class->locks_after)); WARN_ON_ONCE(!list_empty(&class->locks_after));
WARN_ON_ONCE(!list_empty(&class->locks_before)); WARN_ON_ONCE(!list_empty(&class->locks_before));
memset(p + offset, 0, sizeof(*class) - offset); memset_startat(class, 0, key);
WARN_ON_ONCE(!class->lock_entry.next); WARN_ON_ONCE(!class->lock_entry.next);
WARN_ON_ONCE(!list_empty(&class->locks_after)); WARN_ON_ONCE(!list_empty(&class->locks_after));
WARN_ON_ONCE(!list_empty(&class->locks_before)); WARN_ON_ONCE(!list_empty(&class->locks_before));
@ -6290,7 +6287,13 @@ void lockdep_reset_lock(struct lockdep_map *lock)
lockdep_reset_lock_reg(lock); lockdep_reset_lock_reg(lock);
} }
/* Unregister a dynamically allocated key. */ /*
* Unregister a dynamically allocated key.
*
* Unlike lockdep_register_key(), a search is always done to find a matching
* key irrespective of debug_locks to avoid potential invalid access to freed
* memory in lock_class entry.
*/
void lockdep_unregister_key(struct lock_class_key *key) void lockdep_unregister_key(struct lock_class_key *key)
{ {
struct hlist_head *hash_head = keyhashentry(key); struct hlist_head *hash_head = keyhashentry(key);
@ -6305,10 +6308,8 @@ void lockdep_unregister_key(struct lock_class_key *key)
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
if (!graph_lock()) lockdep_lock();
goto out_irq;
pf = get_pending_free();
hlist_for_each_entry_rcu(k, hash_head, hash_entry) { hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
if (k == key) { if (k == key) {
hlist_del_rcu(&k->hash_entry); hlist_del_rcu(&k->hash_entry);
@ -6316,11 +6317,13 @@ void lockdep_unregister_key(struct lock_class_key *key)
break; break;
} }
} }
WARN_ON_ONCE(!found); WARN_ON_ONCE(!found && debug_locks);
__lockdep_free_key_range(pf, key, 1); if (found) {
call_rcu_zapped(pf); pf = get_pending_free();
graph_unlock(); __lockdep_free_key_range(pf, key, 1);
out_irq: call_rcu_zapped(pf);
}
lockdep_unlock();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
/* Wait until is_dynamic_key() has finished accessing k->hash_entry. */ /* Wait until is_dynamic_key() has finished accessing k->hash_entry. */

5
kernel/locking/percpu-rwsem.c
View file

@ -7,6 +7,7 @@
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/sched/task.h> #include <linux/sched/task.h>
#include <linux/sched/debug.h>
#include <linux/errno.h> #include <linux/errno.h>
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
@ -162,7 +163,7 @@ static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader)
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
} }
bool __percpu_down_read(struct percpu_rw_semaphore *sem, bool try) bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
{ {
if (__percpu_down_read_trylock(sem)) if (__percpu_down_read_trylock(sem))
return true; return true;
@ -211,7 +212,7 @@ static bool readers_active_check(struct percpu_rw_semaphore *sem)
return true; return true;
} }
void percpu_down_write(struct percpu_rw_semaphore *sem) void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
{ {
might_sleep(); might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);

2
kernel/locking/rwsem.c
View file

@ -1048,7 +1048,7 @@ out_nolock:
/* /*
* Wait until we successfully acquire the write lock * Wait until we successfully acquire the write lock
*/ */
static struct rw_semaphore * static struct rw_semaphore __sched *
rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
{ {
long count; long count;

11
scripts/atomic/fallbacks/read_acquire
View file

@ -2,6 +2,15 @@ cat <<EOF
static __always_inline ${ret} static __always_inline ${ret}
arch_${atomic}_read_acquire(const ${atomic}_t *v) arch_${atomic}_read_acquire(const ${atomic}_t *v)
{ {
return smp_load_acquire(&(v)->counter); ${int} ret;
if (__native_word(${atomic}_t)) {
ret = smp_load_acquire(&(v)->counter);
} else {
ret = arch_${atomic}_read(v);
__atomic_acquire_fence();
}
return ret;
} }
EOF EOF

7
scripts/atomic/fallbacks/set_release
View file

@ -2,6 +2,11 @@ cat <<EOF
static __always_inline void static __always_inline void
arch_${atomic}_set_release(${atomic}_t *v, ${int} i) arch_${atomic}_set_release(${atomic}_t *v, ${int} i)
{ {
smp_store_release(&(v)->counter, i); if (__native_word(${atomic}_t)) {
smp_store_release(&(v)->counter, i);
} else {
__atomic_release_fence();
arch_${atomic}_set(v, i);
}
} }
EOF EOF