1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
|
From 29515b215b9bbbad0368a5039ba6e53ed3fa7f25 Mon Sep 17 00:00:00 2001
From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Thu, 16 Feb 2012 13:33:12 -0800
Subject: [PATCH 06/11] i387: don't ever touch TS_USEDFPU directly, use helper
functions
commit 6d59d7a9f5b723a7ac1925c136e93ec83c0c3043 upstream.
This creates three helper functions that do the TS_USEDFPU accesses, and
makes everybody that used to do it by hand use those helpers instead.
In addition, there's a couple of helper functions for the "change both
CR0.TS and TS_USEDFPU at the same time" case, and the places that do
that together have been changed to use those. That means that we have
fewer random places that open-code this situation.
The intent is partly to clarify the code without actually changing any
semantics yet (since we clearly still have some hard to reproduce bug in
this area), but also to make it much easier to use another approach
entirely to caching the CR0.TS bit for software accesses.
Right now we use a bit in the thread-info 'status' variable (this patch
does not change that), but we might want to make it a full field of its
own or even make it a per-cpu variable.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
arch/x86/include/asm/i387.h | 75 +++++++++++++++++++++++++++++++-----------
arch/x86/kernel/traps.c | 2 +-
arch/x86/kernel/xsave.c | 2 +-
arch/x86/kvm/vmx.c | 2 +-
4 files changed, 58 insertions(+), 23 deletions(-)
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 55fb3aa..730d7be 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -280,6 +280,47 @@ static inline int restore_fpu_checking(struct task_struct *tsk)
}
/*
+ * Software FPU state helpers. Careful: these need to
+ * be preemption protection *and* they need to be
+ * properly paired with the CR0.TS changes!
+ */
+static inline int __thread_has_fpu(struct thread_info *ti)
+{
+ return ti->status & TS_USEDFPU;
+}
+
+/* Must be paired with an 'stts' after! */
+static inline void __thread_clear_has_fpu(struct thread_info *ti)
+{
+ ti->status &= ~TS_USEDFPU;
+}
+
+/* Must be paired with a 'clts' before! */
+static inline void __thread_set_has_fpu(struct thread_info *ti)
+{
+ ti->status |= TS_USEDFPU;
+}
+
+/*
+ * Encapsulate the CR0.TS handling together with the
+ * software flag.
+ *
+ * These generally need preemption protection to work,
+ * do try to avoid using these on their own.
+ */
+static inline void __thread_fpu_end(struct thread_info *ti)
+{
+ __thread_clear_has_fpu(ti);
+ stts();
+}
+
+static inline void __thread_fpu_begin(struct thread_info *ti)
+{
+ clts();
+ __thread_set_has_fpu(ti);
+}
+
+/*
* Signal frame handlers...
*/
extern int save_i387_xstate(void __user *buf);
@@ -287,23 +328,21 @@ extern int restore_i387_xstate(void __user *buf);
static inline void __unlazy_fpu(struct task_struct *tsk)
{
- if (task_thread_info(tsk)->status & TS_USEDFPU) {
+ if (__thread_has_fpu(task_thread_info(tsk))) {
__save_init_fpu(tsk);
- task_thread_info(tsk)->status &= ~TS_USEDFPU;
- stts();
+ __thread_fpu_end(task_thread_info(tsk));
} else
tsk->fpu_counter = 0;
}
static inline void __clear_fpu(struct task_struct *tsk)
{
- if (task_thread_info(tsk)->status & TS_USEDFPU) {
+ if (__thread_has_fpu(task_thread_info(tsk))) {
/* Ignore delayed exceptions from user space */
asm volatile("1: fwait\n"
"2:\n"
_ASM_EXTABLE(1b, 2b));
- task_thread_info(tsk)->status &= ~TS_USEDFPU;
- stts();
+ __thread_fpu_end(task_thread_info(tsk));
}
}
@@ -311,14 +350,14 @@ static inline void __clear_fpu(struct task_struct *tsk)
* Were we in an interrupt that interrupted kernel mode?
*
* We can do a kernel_fpu_begin/end() pair *ONLY* if that
- * pair does nothing at all: TS_USEDFPU must be clear (so
+ * pair does nothing at all: the thread must not have fpu (so
* that we don't try to save the FPU state), and TS must
* be set (so that the clts/stts pair does nothing that is
* visible in the interrupted kernel thread).
*/
static inline bool interrupted_kernel_fpu_idle(void)
{
- return !(current_thread_info()->status & TS_USEDFPU) &&
+ return !__thread_has_fpu(current_thread_info()) &&
(read_cr0() & X86_CR0_TS);
}
@@ -356,9 +395,9 @@ static inline void kernel_fpu_begin(void)
WARN_ON_ONCE(!irq_fpu_usable());
preempt_disable();
- if (me->status & TS_USEDFPU) {
+ if (__thread_has_fpu(me)) {
__save_init_fpu(me->task);
- me->status &= ~TS_USEDFPU;
+ __thread_clear_has_fpu(me);
/* We do 'stts()' in kernel_fpu_end() */
} else
clts();
@@ -422,24 +461,21 @@ static inline void irq_ts_restore(int TS_state)
*/
static inline int user_has_fpu(void)
{
- return current_thread_info()->status & TS_USEDFPU;
+ return __thread_has_fpu(current_thread_info());
}
static inline void user_fpu_end(void)
{
preempt_disable();
- current_thread_info()->status &= ~TS_USEDFPU;
- stts();
+ __thread_fpu_end(current_thread_info());
preempt_enable();
}
static inline void user_fpu_begin(void)
{
preempt_disable();
- if (!user_has_fpu()) {
- clts();
- current_thread_info()->status |= TS_USEDFPU;
- }
+ if (!user_has_fpu())
+ __thread_fpu_begin(current_thread_info());
preempt_enable();
}
@@ -448,11 +484,10 @@ static inline void user_fpu_begin(void)
*/
static inline void save_init_fpu(struct task_struct *tsk)
{
- WARN_ON_ONCE(!(task_thread_info(tsk)->status & TS_USEDFPU));
+ WARN_ON_ONCE(!__thread_has_fpu(task_thread_info(tsk)));
preempt_disable();
__save_init_fpu(tsk);
- task_thread_info(tsk)->status &= ~TS_USEDFPU;
- stts();
+ __thread_fpu_end(task_thread_info(tsk));
preempt_enable();
}
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 727e6c1..bb5445c 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -579,7 +579,7 @@ void __math_state_restore(void)
return;
}
- thread->status |= TS_USEDFPU; /* So we fnsave on switch_to() */
+ __thread_set_has_fpu(thread); /* clts in caller! */
tsk->fpu_counter++;
}
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 86f1f09..a0bcd0d 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -47,7 +47,7 @@ void __sanitize_i387_state(struct task_struct *tsk)
if (!fx)
return;
- BUG_ON(task_thread_info(tsk)->status & TS_USEDFPU);
+ BUG_ON(__thread_has_fpu(task_thread_info(tsk)));
xstate_bv = tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv;
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 579a0b5..b2c612d 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1456,7 +1456,7 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
#ifdef CONFIG_X86_64
wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
#endif
- if (current_thread_info()->status & TS_USEDFPU)
+ if (__thread_has_fpu(current_thread_info()))
clts();
load_gdt(&__get_cpu_var(host_gdt));
}
--
1.7.7.4
|
