blob: 1834db20b861247579b4e783ce6a3a0ccf8b2ce4 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (c) 2021 Facebook
*/
#ifndef __MMAP_UNLOCK_WORK_H__
#define __MMAP_UNLOCK_WORK_H__
#include <linux/atomic.h>
#include <linux/err.h>
#include <linux/irq_work.h>
/* irq_work to run mmap_read_unlock() in irq_work */
struct mmap_unlock_irq_work {
struct irq_work irq_work;
struct mm_struct *mm;
atomic_t active;
};
DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
/*
* We cannot do mmap_read_unlock() when the irq is disabled, because of
* risk to deadlock with rq_lock. To look up vma when the irqs are
* disabled, we need to run mmap_read_unlock() in irq_work. We use a
* percpu variable to do the irq_work. The active flag reserves the slot
* before mmap_read_trylock() and until the irq_work callback consumes mm.
*/
static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void)
{
struct mmap_unlock_irq_work *work;
if (!irqs_disabled())
return NULL;
/*
* PREEMPT_RT does not allow to trylock mmap sem in interrupt
* disabled context. Force the fallback code.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT))
return ERR_PTR(-EBUSY);
work = this_cpu_ptr(&mmap_unlock_work);
if (irq_work_is_busy(&work->irq_work) ||
atomic_cmpxchg_acquire(&work->active, 0, 1))
return ERR_PTR(-EBUSY);
return work;
}
static inline void
bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work)
{
if (work)
atomic_set_release(&work->active, 0);
}
static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)
{
if (!work) {
mmap_read_unlock(mm);
} else {
work->mm = mm;
/* The lock will be released once we're out of interrupt
* context. Tell lockdep that we've released it now so
* it doesn't complain that we forgot to release it.
*/
rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
irq_work_queue(&work->irq_work);
}
}
#endif /* __MMAP_UNLOCK_WORK_H__ */
|
